From dad8170c287bdc453e0af98066f02def684feab4 Mon Sep 17 00:00:00 2001 From: yuranich Date: Fri, 20 Mar 2026 17:13:17 +0700 Subject: [PATCH 1/5] feat[notask]: add download profiler for registry blob performance diagnostics Made-with: Cursor --- .../client/bin/cli.js | 33 +- .../client/examples/profile-download.js | 59 ++++ .../client/lib/profiler.js | 296 ++++++++++++++++++ .../client/package.json | 3 + .../client/tests/unit/profiler.test.js | 236 ++++++++++++++ 5 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 packages/qvac-lib-registry-server/client/examples/profile-download.js create mode 100644 packages/qvac-lib-registry-server/client/lib/profiler.js create mode 100644 packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js diff --git a/packages/qvac-lib-registry-server/client/bin/cli.js b/packages/qvac-lib-registry-server/client/bin/cli.js index 7f77166dee..5e7ed0fcae 100755 --- a/packages/qvac-lib-registry-server/client/bin/cli.js +++ b/packages/qvac-lib-registry-server/client/bin/cli.js @@ -3,6 +3,7 @@ const { command, flag, arg, summary, header, footer, description } = require('paparam') const { QVACRegistryClient } = require('../index') +const { profileDownload } = require('../lib/profiler') const IdEnc = require('hypercore-id-encoding') const path = require('#path') @@ -191,6 +192,35 @@ const downloadCmd = command('download', } ) +const profileCmd = command('profile', + summary('Profile download performance for a model'), + description('Downloads a model to a temp directory while collecting UDX network stats, connection info, and hypercore metrics. Similar to hyperdrive-profiler but for registry blobs.'), + arg('', 'Model path'), + arg('[source]', 'Model source filter (e.g. hf, s3)'), + flag('--interval|-i [seconds]', 'Stats print interval in seconds (default: 5)'), + flag('--timeout|-t [ms]', 'Stream read timeout in ms (default: 120000)'), + async function (cmd) { + const rootFlags = getRootFlags(cmd) + const registryCoreKey = rootFlags.key || process.env.QVAC_REGISTRY_CORE_KEY + + if (!registryCoreKey) { + console.error('Registry core key is required. Set QVAC_REGISTRY_CORE_KEY or use --key.') + process.exit(1) + } + + const interval = cmd.flags.interval ? parseInt(cmd.flags.interval, 10) : 5 + const timeout = cmd.flags.timeout ? parseInt(cmd.flags.timeout, 10) : 120000 + + await profileDownload({ + registryCoreKey, + modelPath: cmd.args.path, + source: cmd.args.source || undefined, + intervalSec: interval, + timeout + }) + } +) + // --- Root command --- const cmd = command('qvac-registry', @@ -202,7 +232,8 @@ const cmd = command('qvac-registry', flag('--verbose|-v', 'Enable verbose/debug logging'), listCmd, getCmd, - downloadCmd + downloadCmd, + profileCmd ) cmd.parse() diff --git a/packages/qvac-lib-registry-server/client/examples/profile-download.js b/packages/qvac-lib-registry-server/client/examples/profile-download.js new file mode 100644 index 0000000000..d42265c567 --- /dev/null +++ b/packages/qvac-lib-registry-server/client/examples/profile-download.js @@ -0,0 +1,59 @@ +'use strict' + +const path = require('path') +require('dotenv').config({ path: path.resolve(__dirname, '../../.env') }) + +const { profileDownload } = require('../lib/profiler') + +const REGISTRY_CORE_KEY = process.env.QVAC_REGISTRY_CORE_KEY +const STATS_INTERVAL_SEC = parseInt(process.env.PROFILE_INTERVAL || '5', 10) + +function usage () { + console.log(` +Usage: node profile-download.js [model-path] [source] + + model-path Path of the model to download (omit to list available models) + source Source filter, e.g. "hf" or "s3" (default: first match) + +Environment: + QVAC_REGISTRY_CORE_KEY Registry view core key (required) + PROFILE_INTERVAL Stats print interval in seconds (default: 5) + +Examples: + node profile-download.js + node profile-download.js "ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-tiny.bin" + PROFILE_INTERVAL=2 node profile-download.js "ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-tiny.bin" hf +`) +} + +async function main () { + if (!REGISTRY_CORE_KEY) { + console.error('QVAC_REGISTRY_CORE_KEY is not set') + process.exit(1) + } + + const modelPath = process.argv[2] + const sourceFilter = process.argv[3] + + if (modelPath === '--help' || modelPath === '-h') { + usage() + process.exit(0) + } + + if (!modelPath) { + usage() + process.exit(0) + } + + await profileDownload({ + registryCoreKey: REGISTRY_CORE_KEY, + modelPath, + source: sourceFilter, + intervalSec: STATS_INTERVAL_SEC + }) +} + +main().catch((err) => { + console.error('Profiler failed:', err) + process.exit(1) +}) diff --git a/packages/qvac-lib-registry-server/client/lib/profiler.js b/packages/qvac-lib-registry-server/client/lib/profiler.js new file mode 100644 index 0000000000..4f373b2194 --- /dev/null +++ b/packages/qvac-lib-registry-server/client/lib/profiler.js @@ -0,0 +1,296 @@ +'use strict' + +const os = require('os') +const fs = require('fs') +const { performance } = require('perf_hooks') +const { pipeline } = require('stream/promises') + +const Corestore = require('corestore') +const Hyperswarm = require('hyperswarm') +const Hyperblobs = require('hyperblobs') +const HypercoreStats = require('hypercore-stats') +const HyperswarmStats = require('hyperswarm-stats') +const IdEnc = require('hypercore-id-encoding') +const byteSize = require('tiny-byte-size') +const path = require('#path') + +const { RegistryDatabase } = require('@qvac/registry-schema') + +/** + * Normalize a blob core key into a Buffer regardless of input format. + * Handles raw Buffer, { data: [...] } objects from HyperDB, and z32/hex strings. + */ +function decodeCoreKey (key) { + if (Buffer.isBuffer(key)) return key + if (typeof key === 'object' && key !== null && key.data) { + return Buffer.from(key.data) + } + return IdEnc.decode(key) +} + +/** + * Collect a snapshot of network, connection, and hypercore stats. + * All fields are plain values suitable for logging or formatting. + */ +function collectStats (swarmStats, hypercoreStats, blobCore, elapsedSec) { + const bytesRx = swarmStats.dhtStats.udxBytesReceived + const bytesTx = swarmStats.dhtStats.udxBytesTransmitted + + const peers = [] + for (const peer of blobCore.peers) { + peers.push({ + key: IdEnc.normalize(peer.remotePublicKey).slice(0, 12), + remoteLength: peer.remoteLength, + remoteContiguous: peer.remoteContiguousLength + }) + } + + return { + elapsedSec, + network: { + bytesRx, + bytesTx, + rxPerSec: elapsedSec > 0 ? bytesRx / elapsedSec : 0, + txPerSec: elapsedSec > 0 ? bytesTx / elapsedSec : 0, + packetsRx: swarmStats.dhtStats.udxPacketsReceived, + packetsTx: swarmStats.dhtStats.udxPacketsTransmitted, + packetsDropped: swarmStats.dhtStats.udxPacketsDropped + }, + connection: { + firewalled: swarmStats.dhtStats.isFirewalled, + blobPeers: blobCore.peers.length, + attempted: swarmStats.connects.client.attempted, + opened: swarmStats.connects.client.opened, + closed: swarmStats.connects.client.closed, + rtos: swarmStats.getRTOCountAcrossAllStreams(), + fastRecoveries: swarmStats.getFastRecoveriesAcrossAllStreams(), + retransmits: swarmStats.getRetransmitsAcrossAllStreams() + }, + hypercore: { + contiguousLength: blobCore.contiguousLength, + length: blobCore.length, + hotswaps: hypercoreStats.totalHotswaps + }, + peers + } +} + +/** + * Format a stats snapshot into a human-readable string. + */ +function formatStats (stats) { + const n = stats.network + const c = stats.connection + const h = stats.hypercore + + let lines = '--- ' + stats.elapsedSec.toFixed(1) + 's elapsed ---\n' + lines += 'Network (UDX)\n' + lines += ' Bytes received: ' + byteSize(n.bytesRx) + ' (' + byteSize(n.rxPerSec) + '/s)\n' + lines += ' Bytes transmitted: ' + byteSize(n.bytesTx) + ' (' + byteSize(n.txPerSec) + '/s)\n' + lines += ' Packets rx/tx: ' + n.packetsRx + ' / ' + n.packetsTx + '\n' + lines += ' Packets dropped: ' + n.packetsDropped + '\n' + lines += 'Connection\n' + lines += ' Firewalled: ' + c.firewalled + '\n' + lines += ' Blob peers: ' + c.blobPeers + '\n' + lines += ' Attempted: ' + c.attempted + '\n' + lines += ' Opened: ' + c.opened + '\n' + lines += ' Closed: ' + c.closed + '\n' + lines += ' Issues: rto=' + c.rtos + ' fast-recoveries=' + c.fastRecoveries + ' retransmits=' + c.retransmits + '\n' + lines += 'Hypercore\n' + lines += ' Blob core: ' + h.contiguousLength + ' / ' + h.length + ' (contiguous / length)\n' + lines += ' Hotswaps: ' + h.hotswaps + '\n' + + if (stats.peers.length > 0) { + lines += ' Peers:\n' + for (const p of stats.peers) { + lines += ' ' + p.key + '... remote=' + p.remoteContiguous + '/' + p.remoteLength + '\n' + } + } + + return lines +} + +/** + * Format a final download summary into a human-readable string. + */ +function formatSummary (opts) { + let lines = '='.repeat(50) + '\n' + lines += 'FINAL SUMMARY\n' + lines += '='.repeat(50) + '\n' + lines += 'Download\n' + lines += ' Model: ' + opts.modelPath + '\n' + lines += ' Size: ' + byteSize(opts.totalBytes) + ' (' + opts.totalBlocks + ' blocks)\n' + lines += ' Metadata: ' + opts.metadataSec.toFixed(2) + 's\n' + lines += ' Transfer: ' + opts.transferSec.toFixed(2) + 's\n' + lines += ' Avg speed: ' + byteSize(opts.avgSpeed) + '/s\n' + lines += ' Total: ' + opts.totalSec.toFixed(2) + 's\n' + return lines +} + +/** + * Profile a blob download from the registry, printing periodic stats. + * + * @param {object} opts + * @param {string} opts.registryCoreKey - Registry view core key (z32 or hex) + * @param {string} opts.modelPath - Model path in the registry + * @param {string} [opts.source] - Source filter (e.g. "hf", "s3") + * @param {number} [opts.intervalSec=5] - Stats print interval in seconds + * @param {number} [opts.timeout=120000] - Stream read timeout in ms + * @param {function} [opts.onStats] - Called with (formattedString, rawStats) on each interval + * @param {function} [opts.onLog] - Called with (message) for log output; defaults to console.log + * @returns {Promise} Summary with timing and stats + */ +async function profileDownload (opts) { + const { + registryCoreKey, + modelPath, + source, + intervalSec = 5, + timeout = 120000, + onStats, + onLog = console.log + } = opts + + if (!registryCoreKey) throw new Error('registryCoreKey is required') + if (!modelPath) throw new Error('modelPath is required') + + const tStart = performance.now() + const tmpdir = path.join(os.tmpdir(), 'qvac-profile-' + Date.now()) + fs.mkdirSync(tmpdir, { recursive: true }) + + const store = new Corestore(tmpdir) + await store.ready() + + const swarm = new Hyperswarm() + const hcStats = await HypercoreStats.fromCorestore(store, { cacheExpiryMs: 1000 }) + const swStats = new HyperswarmStats(swarm) + + swarm.on('connection', (conn, peerInfo) => { + const key = peerInfo?.publicKey ? IdEnc.normalize(peerInfo.publicKey).slice(0, 12) : 'unknown' + onLog(' [conn] peer ' + key + '... connected') + store.replicate(conn) + conn.on('error', (e) => onLog(' [conn] error: ' + e.message)) + conn.on('close', () => onLog(' [conn] peer ' + key + '... disconnected')) + }) + + const cleanupResources = async () => { + await swarm.destroy() + await store.close() + try { fs.rmSync(tmpdir, { recursive: true, force: true }) } catch {} + } + + try { + const viewKey = IdEnc.decode(registryCoreKey) + const viewCore = store.get({ key: viewKey }) + await viewCore.ready() + + onLog('View core key: ' + IdEnc.normalize(viewCore.key)) + + const foundPeers = viewCore.findingPeers() + swarm.join(viewCore.discoveryKey, { client: true, server: false }) + swarm.flush().then(() => foundPeers()) + + await viewCore.update() + const metadataSec = (performance.now() - tStart) / 1000 + + const db = new RegistryDatabase(viewCore, { extension: false }) + await db.ready() + + onLog('Metadata synced in ' + metadataSec.toFixed(2) + 's (' + viewCore.length + ' blocks)') + + const model = await db.getModel(modelPath, source || undefined) + if (!model) throw new Error('Model not found: ' + modelPath) + + const blob = model.blobBinding + if (!blob || !blob.coreKey) throw new Error('Model has no blob binding') + + onLog('\nProfiling download: ' + model.path) + onLog(' engine: ' + model.engine) + onLog(' source: ' + model.source) + onLog(' size: ' + byteSize(blob.byteLength) + ' (' + blob.blockLength + ' blocks)') + onLog('') + + const coreKeyBuf = decodeCoreKey(blob.coreKey) + const blobCore = store.get({ key: coreKeyBuf }) + await blobCore.ready() + const blobs = new Hyperblobs(blobCore) + await blobs.ready() + + onLog('Blob core key: ' + IdEnc.normalize(blobCore.key)) + onLog('Blob core discovery: ' + IdEnc.normalize(blobCore.discoveryKey)) + + const foundBlobPeers = blobCore.findingPeers() + swarm.join(blobCore.discoveryKey, { client: true, server: false }) + swarm.flush().then(() => foundBlobPeers()) + + await blobCore.update() + onLog('Blob core synced: length=' + blobCore.length) + onLog('') + + const outputFile = path.join(tmpdir, 'download.bin') + const tDownloadStart = performance.now() + + const emitStats = () => { + const elapsed = (performance.now() - tDownloadStart) / 1000 + const raw = collectStats(swStats, hcStats, blobCore, elapsed) + const formatted = formatStats(raw) + if (onStats) { + onStats(formatted, raw) + } else { + onLog(formatted) + } + return raw + } + + const statsInterval = setInterval(emitStats, intervalSec * 1000) + + const rangeDownload = blobCore.download({ + start: blob.blockOffset, + length: blob.blockLength + }) + + const readStream = blobs.createReadStream(blob, { wait: true, timeout }) + const writeStream = fs.createWriteStream(outputFile) + + try { + await pipeline(readStream, writeStream) + } finally { + rangeDownload.destroy() + clearInterval(statsInterval) + } + + const totalSec = (performance.now() - tStart) / 1000 + const transferSec = (performance.now() - tDownloadStart) / 1000 + const avgSpeed = transferSec > 0 ? blob.byteLength / transferSec : 0 + + const finalStats = emitStats() + + const summary = { + modelPath: model.path, + totalBytes: blob.byteLength, + totalBlocks: blob.blockLength, + metadataSec, + transferSec, + avgSpeed, + totalSec, + finalStats + } + + onLog(formatSummary(summary)) + + await blobs.close() + await blobCore.close() + + return summary + } finally { + await cleanupResources() + } +} + +module.exports = { + decodeCoreKey, + collectStats, + formatStats, + formatSummary, + profileDownload +} diff --git a/packages/qvac-lib-registry-server/client/package.json b/packages/qvac-lib-registry-server/client/package.json index f43a62171a..c807330da9 100644 --- a/packages/qvac-lib-registry-server/client/package.json +++ b/packages/qvac-lib-registry-server/client/package.json @@ -63,8 +63,11 @@ "devDependencies": { "brittle": "^3.4.0", "dotenv": "^17.2.3", + "hypercore-stats": "^2.4.0", + "hyperswarm-stats": "^1.3.0", "standard": "^17.1.0", "test-tmp": "^1.4.0", + "tiny-byte-size": "^1.1.0", "typescript": "^5.9.3" } } diff --git a/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js b/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js new file mode 100644 index 0000000000..1c4fdd250d --- /dev/null +++ b/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js @@ -0,0 +1,236 @@ +'use strict' + +const test = require('brittle') +const { decodeCoreKey, collectStats, formatStats, formatSummary } = require('../../lib/profiler') +const IdEnc = require('hypercore-id-encoding') + +test('profiler module exports', async t => { + t.ok(typeof decodeCoreKey === 'function', 'decodeCoreKey is a function') + t.ok(typeof collectStats === 'function', 'collectStats is a function') + t.ok(typeof formatStats === 'function', 'formatStats is a function') + t.ok(typeof formatSummary === 'function', 'formatSummary is a function') +}) + +// --- decodeCoreKey --- + +test('decodeCoreKey - Buffer passthrough', async t => { + const buf = Buffer.alloc(32, 0xab) + const result = decodeCoreKey(buf) + t.ok(Buffer.isBuffer(result), 'returns a Buffer') + t.ok(result.equals(buf), 'returns the same buffer') +}) + +test('decodeCoreKey - object with data array', async t => { + const original = Buffer.alloc(32, 0xcd) + const obj = { data: Array.from(original) } + const result = decodeCoreKey(obj) + t.ok(Buffer.isBuffer(result), 'returns a Buffer') + t.ok(result.equals(original), 'decodes correctly from data array') +}) + +test('decodeCoreKey - z32 string', async t => { + const buf = Buffer.alloc(32, 0xef) + const encoded = IdEnc.normalize(buf) + const result = decodeCoreKey(encoded) + t.ok(Buffer.isBuffer(result), 'returns a Buffer') + t.ok(result.equals(buf), 'decodes z32 string correctly') +}) + +// --- collectStats --- + +function createMockSwarmStats () { + return { + dhtStats: { + udxBytesReceived: 1048576, + udxBytesTransmitted: 4096, + udxPacketsReceived: 1000, + udxPacketsTransmitted: 50, + udxPacketsDropped: 2, + isFirewalled: false + }, + connects: { + client: { attempted: 3, opened: 2, closed: 1 } + }, + getRTOCountAcrossAllStreams: () => 0, + getFastRecoveriesAcrossAllStreams: () => 1, + getRetransmitsAcrossAllStreams: () => 3 + } +} + +function createMockHypercoreStats () { + return { totalHotswaps: 5 } +} + +function createMockBlobCore () { + const key = Buffer.alloc(32, 0x01) + return { + contiguousLength: 100, + length: 120, + peers: [{ + remotePublicKey: key, + remoteLength: 120, + remoteContiguousLength: 115 + }] + } +} + +test('collectStats - returns structured snapshot', async t => { + const sw = createMockSwarmStats() + const hc = createMockHypercoreStats() + const core = createMockBlobCore() + + const stats = collectStats(sw, hc, core, 10) + + t.is(stats.elapsedSec, 10) + + t.is(stats.network.bytesRx, 1048576) + t.is(stats.network.bytesTx, 4096) + t.is(stats.network.packetsRx, 1000) + t.is(stats.network.packetsTx, 50) + t.is(stats.network.packetsDropped, 2) + t.ok(Math.abs(stats.network.rxPerSec - 104857.6) < 0.01, 'rxPerSec') + t.ok(Math.abs(stats.network.txPerSec - 409.6) < 0.01, 'txPerSec') + + t.is(stats.connection.firewalled, false) + t.is(stats.connection.blobPeers, 1) + t.is(stats.connection.attempted, 3) + t.is(stats.connection.opened, 2) + t.is(stats.connection.closed, 1) + t.is(stats.connection.rtos, 0) + t.is(stats.connection.fastRecoveries, 1) + t.is(stats.connection.retransmits, 3) + + t.is(stats.hypercore.contiguousLength, 100) + t.is(stats.hypercore.length, 120) + t.is(stats.hypercore.hotswaps, 5) + + t.is(stats.peers.length, 1) + t.is(stats.peers[0].remoteLength, 120) + t.is(stats.peers[0].remoteContiguous, 115) +}) + +test('collectStats - handles zero elapsed', async t => { + const sw = createMockSwarmStats() + const hc = createMockHypercoreStats() + const core = createMockBlobCore() + + const stats = collectStats(sw, hc, core, 0) + t.is(stats.network.rxPerSec, 0, 'no division by zero') + t.is(stats.network.txPerSec, 0, 'no division by zero') +}) + +test('collectStats - handles no peers', async t => { + const sw = createMockSwarmStats() + const hc = createMockHypercoreStats() + const core = { contiguousLength: 0, length: 0, peers: [] } + + const stats = collectStats(sw, hc, core, 5) + t.is(stats.peers.length, 0) + t.is(stats.connection.blobPeers, 0) +}) + +// --- formatStats --- + +test('formatStats - contains expected sections', async t => { + const stats = { + elapsedSec: 12.5, + network: { + bytesRx: 500000, + bytesTx: 2000, + rxPerSec: 40000, + txPerSec: 160, + packetsRx: 500, + packetsTx: 20, + packetsDropped: 0 + }, + connection: { + firewalled: true, + blobPeers: 2, + attempted: 4, + opened: 3, + closed: 1, + rtos: 0, + fastRecoveries: 0, + retransmits: 0 + }, + hypercore: { + contiguousLength: 50, + length: 60, + hotswaps: 0 + }, + peers: [ + { + key: 'abc123456789', + remoteLength: 60, + remoteContiguous: 55 + } + ] + } + + const output = formatStats(stats) + + t.ok(output.includes('12.5s elapsed'), 'has elapsed') + t.ok(output.includes('Network (UDX)'), 'has network section') + t.ok(output.includes('Connection'), 'has connection section') + t.ok(output.includes('Hypercore'), 'has hypercore section') + t.ok(output.includes('Firewalled: true'), 'has firewall status') + t.ok(output.includes('Blob peers: 2'), 'has peer count') + t.ok(output.includes('50 / 60'), 'has contiguous/length') + t.ok(output.includes('abc123456789'), 'has peer key') + t.ok(output.includes('remote=55/60'), 'has peer remote info') +}) + +test('formatStats - no peers section when empty', async t => { + const stats = { + elapsedSec: 1, + network: { + bytesRx: 0, + bytesTx: 0, + rxPerSec: 0, + txPerSec: 0, + packetsRx: 0, + packetsTx: 0, + packetsDropped: 0 + }, + connection: { + firewalled: false, + blobPeers: 0, + attempted: 0, + opened: 0, + closed: 0, + rtos: 0, + fastRecoveries: 0, + retransmits: 0 + }, + hypercore: { + contiguousLength: 0, + length: 0, + hotswaps: 0 + }, + peers: [] + } + + const output = formatStats(stats) + t.absent(output.includes('Peers:'), 'no peers section') +}) + +// --- formatSummary --- + +test('formatSummary - contains expected fields', async t => { + const output = formatSummary({ + modelPath: 'test/model.gguf', + totalBytes: 1073741824, + totalBlocks: 16384, + metadataSec: 1.5, + transferSec: 45.2, + avgSpeed: 23756800, + totalSec: 46.7 + }) + + t.ok(output.includes('FINAL SUMMARY'), 'has header') + t.ok(output.includes('test/model.gguf'), 'has model path') + t.ok(output.includes('16384 blocks'), 'has block count') + t.ok(output.includes('1.50s'), 'has metadata time') + t.ok(output.includes('45.20s'), 'has transfer time') + t.ok(output.includes('46.70s'), 'has total time') +}) From a65447d6ab6044330112d899619275f85fa95fd2 Mon Sep 17 00:00:00 2001 From: yuranich Date: Fri, 20 Mar 2026 17:53:50 +0700 Subject: [PATCH 2/5] fix: move profiler deps from devDependencies to dependencies Made-with: Cursor --- packages/qvac-lib-registry-server/client/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/qvac-lib-registry-server/client/package.json b/packages/qvac-lib-registry-server/client/package.json index c807330da9..a16a943cd8 100644 --- a/packages/qvac-lib-registry-server/client/package.json +++ b/packages/qvac-lib-registry-server/client/package.json @@ -56,18 +56,18 @@ "hyperblobs": "^2.8.0", "hypercore-id-encoding": "^1.3.0", "hyperdb": "^4.16.1", + "hypercore-stats": "^2.4.0", "hyperswarm": "^4.14.0", + "hyperswarm-stats": "^1.3.0", "paparam": "^1.10.0", - "ready-resource": "^1.0.1" + "ready-resource": "^1.0.1", + "tiny-byte-size": "^1.1.0" }, "devDependencies": { "brittle": "^3.4.0", "dotenv": "^17.2.3", - "hypercore-stats": "^2.4.0", - "hyperswarm-stats": "^1.3.0", "standard": "^17.1.0", "test-tmp": "^1.4.0", - "tiny-byte-size": "^1.1.0", "typescript": "^5.9.3" } } From c97c65a0d00d5665388ee90a9ebd8dcd11f3f93e Mon Sep 17 00:00:00 2001 From: yuranich Date: Fri, 20 Mar 2026 17:56:32 +0700 Subject: [PATCH 3/5] doc: add profile command and example to client README Made-with: Cursor --- .../qvac-lib-registry-server/client/README.md | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/qvac-lib-registry-server/client/README.md b/packages/qvac-lib-registry-server/client/README.md index daac3792d2..7ae6d982ed 100644 --- a/packages/qvac-lib-registry-server/client/README.md +++ b/packages/qvac-lib-registry-server/client/README.md @@ -226,6 +226,47 @@ Downloading ... -> /absolute/path/ggml-tiny-q8_0.bin Download complete: 41.52 MB ``` +### Profile download performance + +Diagnose slow downloads by collecting UDX network stats, connection info, and hypercore metrics — similar to [hyperdrive-profiler](https://github.com/holepunchto/hyperdrive-profiler) but for registry Hyperblobs: + +```bash +$ qvac-registry profile \ + "ggerganov/whisper.cpp/resolve/5359861c739e955e79d9a303bcbc70fb988958b1/ggml-tiny.bin" hf + +--- 5.0s elapsed --- +Network (UDX) + Bytes received: 42.1MB (8.4MB/s) + Bytes transmitted: 128kB (25.6kB/s) + Packets rx/tx: 29034 / 1842 + Packets dropped: 0 +Connection + Firewalled: false + Blob peers: 2 + Issues: rto=0 fast-recoveries=0 retransmits=0 +Hypercore + Blob core: 665 / 665 (contiguous / length) + Hotswaps: 0 +... +================================================== +FINAL SUMMARY +================================================== +Download + Model: ggerganov/whisper.cpp/resolve/.../ggml-tiny.bin + Size: 73.5MB (1120 blocks) + Metadata: 2.15s + Transfer: 8.72s + Avg speed: 8.4MB/s + Total: 10.87s +``` + +Flags: + +``` +--interval|-i [seconds] Stats print interval (default: 5) +--timeout|-t [ms] Stream read timeout (default: 120000) +``` + ### JSON output All commands support `--json` for machine-readable output: @@ -250,6 +291,7 @@ See the `examples/` folder for complete working examples: - `download-model.js`: Download a single model to disk via metadata lookup - `download-blob.js`: Download a blob directly using known blob coordinates - `download-all-models.js`: Download all models in the registry +- `profile-download.js`: Profile download performance with network/connection/hypercore stats Run examples: @@ -259,6 +301,7 @@ node examples/example.js node examples/download-model.js node examples/download-blob.js node examples/download-all-models.js +node examples/profile-download.js "model/path" ``` ## Configuration From 1a4f83e95c66e9c17eda1d14cec6fe1e97350ef9 Mon Sep 17 00:00:00 2001 From: yuranich Date: Fri, 20 Mar 2026 21:19:39 +0700 Subject: [PATCH 4/5] fix: show full peer keys in profiler output for troubleshooting Made-with: Cursor --- .../qvac-lib-registry-server/client/lib/profiler.js | 10 +++++----- .../client/tests/unit/profiler.test.js | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/qvac-lib-registry-server/client/lib/profiler.js b/packages/qvac-lib-registry-server/client/lib/profiler.js index 4f373b2194..c4cf8bad6a 100644 --- a/packages/qvac-lib-registry-server/client/lib/profiler.js +++ b/packages/qvac-lib-registry-server/client/lib/profiler.js @@ -39,7 +39,7 @@ function collectStats (swarmStats, hypercoreStats, blobCore, elapsedSec) { const peers = [] for (const peer of blobCore.peers) { peers.push({ - key: IdEnc.normalize(peer.remotePublicKey).slice(0, 12), + key: IdEnc.normalize(peer.remotePublicKey), remoteLength: peer.remoteLength, remoteContiguous: peer.remoteContiguousLength }) @@ -103,7 +103,7 @@ function formatStats (stats) { if (stats.peers.length > 0) { lines += ' Peers:\n' for (const p of stats.peers) { - lines += ' ' + p.key + '... remote=' + p.remoteContiguous + '/' + p.remoteLength + '\n' + lines += ' ' + p.key + ' remote=' + p.remoteContiguous + '/' + p.remoteLength + '\n' } } @@ -166,11 +166,11 @@ async function profileDownload (opts) { const swStats = new HyperswarmStats(swarm) swarm.on('connection', (conn, peerInfo) => { - const key = peerInfo?.publicKey ? IdEnc.normalize(peerInfo.publicKey).slice(0, 12) : 'unknown' - onLog(' [conn] peer ' + key + '... connected') + const key = peerInfo?.publicKey ? IdEnc.normalize(peerInfo.publicKey) : 'unknown' + onLog(' [conn] peer ' + key + ' connected') store.replicate(conn) conn.on('error', (e) => onLog(' [conn] error: ' + e.message)) - conn.on('close', () => onLog(' [conn] peer ' + key + '... disconnected')) + conn.on('close', () => onLog(' [conn] peer ' + key + ' disconnected')) }) const cleanupResources = async () => { diff --git a/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js b/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js index 1c4fdd250d..b3152ee0ee 100644 --- a/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js +++ b/packages/qvac-lib-registry-server/client/tests/unit/profiler.test.js @@ -160,7 +160,7 @@ test('formatStats - contains expected sections', async t => { }, peers: [ { - key: 'abc123456789', + key: 'o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4', remoteLength: 60, remoteContiguous: 55 } @@ -176,7 +176,7 @@ test('formatStats - contains expected sections', async t => { t.ok(output.includes('Firewalled: true'), 'has firewall status') t.ok(output.includes('Blob peers: 2'), 'has peer count') t.ok(output.includes('50 / 60'), 'has contiguous/length') - t.ok(output.includes('abc123456789'), 'has peer key') + t.ok(output.includes('o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4o4'), 'has peer key') t.ok(output.includes('remote=55/60'), 'has peer remote info') }) From 0fdfc9863bcb9d4c616c0d82cf31255afc93e713 Mon Sep 17 00:00:00 2001 From: yuranich Date: Mon, 23 Mar 2026 18:09:34 +0700 Subject: [PATCH 5/5] fix: validate parseInt results for interval and timeout CLI flags Made-with: Cursor --- packages/qvac-lib-registry-server/client/bin/cli.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/qvac-lib-registry-server/client/bin/cli.js b/packages/qvac-lib-registry-server/client/bin/cli.js index 5e7ed0fcae..d965e8574e 100755 --- a/packages/qvac-lib-registry-server/client/bin/cli.js +++ b/packages/qvac-lib-registry-server/client/bin/cli.js @@ -211,6 +211,15 @@ const profileCmd = command('profile', const interval = cmd.flags.interval ? parseInt(cmd.flags.interval, 10) : 5 const timeout = cmd.flags.timeout ? parseInt(cmd.flags.timeout, 10) : 120000 + if (Number.isNaN(interval) || interval <= 0) { + console.error('--interval must be a positive number') + process.exit(1) + } + if (Number.isNaN(timeout) || timeout <= 0) { + console.error('--timeout must be a positive number') + process.exit(1) + } + await profileDownload({ registryCoreKey, modelPath: cmd.args.path,