QVAC-14392 chore: scope down DataLoader cleanup to packages/rag

packages/rag/NOTICE

@@ -29,23 +29,23 @@ JavaScript Dependencies
     https://github.com/holepunchto/bare-events
   bare-events@2.8.2
     https://github.com/holepunchto/bare-events
-  bare-fs@4.5.6
+  bare-fs@4.7.1
     https://github.com/holepunchto/bare-fs
   bare-inspect@3.1.4
     https://github.com/holepunchto/bare-inspect
   bare-module-resolve@1.12.1
     https://github.com/holepunchto/bare-module-resolve
-  bare-os@3.8.6
+  bare-os@3.9.0
     https://github.com/holepunchto/bare-os
   bare-path@3.0.0
     https://github.com/holepunchto/bare-path
-  bare-semver@1.0.2
+  bare-semver@1.0.3
     https://github.com/holepunchto/bare-semver
-  bare-stream@2.11.0
+  bare-stream@2.13.0
     https://github.com/holepunchto/bare-stream
   bare-type@1.1.0
     https://github.com/holepunchto/bare-type
-  bare-url@2.4.0
+  bare-url@2.4.2
     https://github.com/holepunchto/bare-url
   blind-relay@1.4.0
     https://github.com/holepunchto/blind-relay
@@ -55,6 +55,8 @@ JavaScript Dependencies
     https://github.com/holepunchto/events-universal
   hypercore-id-encoding@1.3.0
     https://github.com/holepunchto/hypercore-id-encoding
+  hyperdht-address@1.0.1
+    https://github.com/holepunchto/hyperdht-address
   hyperschema@1.20.1
     https://github.com/holepunchto/hyperschema
   noise-handshake@4.2.0
@@ -87,8 +89,8 @@ JavaScript Dependencies
 
   bogon@1.2.0
     https://github.com/mafintosh/bogon
-  dht-rpc@6.26.3
-    https://github.com/mafintosh/dht-rpc
+  dht-rpc@6.26.4
+    https://github.com/holepunchto/dht-rpc
   fast-fifo@1.3.2
     https://github.com/mafintosh/fast-fifo
   generate-object-property@2.0.0
@@ -97,16 +99,16 @@ JavaScript Dependencies
     https://github.com/mafintosh/generate-string
   hypercore-crypto@3.6.1
     https://github.com/mafintosh/hypercore-crypto
-  hyperdht@6.29.6
+  hyperdht@6.30.0
     https://github.com/holepunchto/hyperdht
   is-property@1.0.2
     https://github.com/mikolalysenko/is-property
   kademlia-routing-table@1.0.6
     https://github.com/mafintosh/kademlia-routing-table
   nat-sampler@1.0.1
     https://github.com/mafintosh/nat-sampler
-  protomux@3.10.1
-    https://github.com/mafintosh/protomux
+  protomux@3.10.3
+    https://github.com/holepunchto/protomux
   queue-tick@1.0.1
     https://github.com/mafintosh/queue-tick
   ready-resource@1.2.0

packages/rag/examples/chunking.js

@@ -1,30 +1,25 @@
-const { RAG, HyperDBAdapter } = require('../index')
+'use strict'
+
 const Corestore = require('corestore')
 const EmbedderPlugin = require('@qvac/embed-llamacpp')
-const HyperDriveDL = require('@qvac/dl-hyperdrive')
 const QvacLogger = require('@qvac/logging')
 
-const gteDriveKey = 'd1896d9259692818df95bd2480e90c2d057688a4f7c9b1ae13ac7f5ee379d03e'
+const { RAG, HyperDBAdapter } = require('../index')
+const { ensureModels } = require('./utils')
 
 const store = new Corestore('./local-store')
-const modelDir = './models'
 
 async function main () {
-  // Load the embedder using HyperDriveDL
-  const gteHdDL = new HyperDriveDL({
-    key: `hd://${gteDriveKey}`,
-    store
-  })
+  // Fetch the embedder model from the QVAC registry (cached on disk after first run).
+  const models = await ensureModels(['embedder'])
 
-  const embedderArgs = {
-    loader: gteHdDL,
-    opts: { stats: true },
+  const embedder = new EmbedderPlugin({
+    files: { model: [models.embedder.fullPath] },
+    config: { device: 'gpu', gpu_layers: '25' },
     logger: console,
-    diskPath: modelDir,
-    modelName: 'gte-large_fp16.gguf'
-  }
-  const embedder = new EmbedderPlugin(embedderArgs, '-ngl\t99\n-dev\tgpu\n--embd-separator\t⟪§§§EMBED_SEP§§§⟫\n-c\t4096')
-  await embedder.load(false)
+    opts: { stats: true }
+  })
+  await embedder.load()
 
   const embeddingFunction = async (text) => {
     const response = await embedder.run(text)
@@ -34,17 +29,14 @@ async function main () {
 
   const dbAdapter = new HyperDBAdapter({ store })
 
-  // Create logger for visibility
   const logger = new QvacLogger(console)
 
-  // Sample text for chunking demonstrations
   const sampleText = 'Artificial intelligence is transforming industries. Machine learning algorithms process vast amounts of data efficiently. Deep learning uses neural networks with multiple layers. Natural language processing enables computers to understand human text.'
 
   console.log('Original text:')
   console.log(`"${sampleText}"`)
   console.log(`\nLength: ${sampleText.length} characters, ${sampleText.split(' ').length} words\n`)
 
-  // 1. Default word-based chunking
   console.log('=== 1. Word Splitting (default) ===')
   const rag = new RAG({ embeddingFunction, dbAdapter, logger })
 
@@ -60,7 +52,6 @@ async function main () {
     console.log(`  Word Count: ${chunk.content.split(' ').filter(w => w).length}`)
   })
 
-  // 2. Character-based splitting
   console.log('\n=== 2. Character Splitting ===')
   const charResult = await rag.chunk(sampleText, {
     splitStrategy: 'character',
@@ -74,7 +65,6 @@ async function main () {
     console.log(`  Character Count: ${chunk.content.length}`)
   })
 
-  // 3. Sentence-based splitting
   console.log('\n=== 3. Sentence Splitting ===')
   const sentenceResult = await rag.chunk(sampleText, {
     splitStrategy: 'sentence',
@@ -87,7 +77,6 @@ async function main () {
     console.log(`  Chunk ${i + 1}: "${chunk.content.trim()}"`)
   })
 
-  // 4. Line-based splitting
   console.log('\n=== 4. Line Splitting ===')
   const multilineText = 'Line one: AI is transforming\nLine two: Machine learning processes data\nLine three: Deep learning uses networks\nLine four: NLP enables understanding'
   const lineResult = await rag.chunk(multilineText, {
@@ -101,7 +90,6 @@ async function main () {
     console.log(`  Chunk ${i + 1}: "${chunk.content}"`)
   })
 
-  // 5. Custom delimiter-based splitter
   console.log('\n=== 5. Custom Delimiter Splitter ===')
   const delimiterText = 'AI|Machine Learning|Deep Learning|NLP|Computer Vision|Robotics'
   const customDelimiterSplitter = (text) => text.split('|')
@@ -117,11 +105,8 @@ async function main () {
     console.log(`  Chunk ${i + 1}: "${chunk.content}"`)
   })
 
-  // 6. Custom whitespace-aware splitter
   console.log('\n=== 6. Custom Whitespace-Aware Splitter ===')
   const whitespaceSplitter = (text) => {
-    // Split by whitespace and filter empty strings
-    // Note: Tokens must match the original text for llm-splitter to work
     return text.split(/\s+/).filter(word => word.length > 0)
   }
 
@@ -136,7 +121,6 @@ async function main () {
     console.log(`  Chunk ${i + 1}: "${chunk.content}"`)
   })
 
-  // 7. Chunk strategy comparison
   console.log('\n=== 7. Chunk Strategy: paragraph vs character ===')
   const paragraphText = 'First paragraph here.\n\nSecond paragraph here.\n\nThird paragraph here.'
 
@@ -164,9 +148,8 @@ async function main () {
     console.log(`  Chunk ${i + 1}: "${chunk.content}"`)
   })
 
-  // Cleanup
-  await gteHdDL.close()
   await rag.close()
+  await embedder.unload()
   await store.close()
 }
 

packages/rag/examples/quickstart.js

@@ -1,37 +1,29 @@
+'use strict'
+
+const Corestore = require('corestore')
 const EmbedderPlugin = require('@qvac/embed-llamacpp')
 const LlmPlugin = require('@qvac/llm-llamacpp')
-const HyperDriveDL = require('@qvac/dl-hyperdrive')
-const knowledgeBase = require('./knowledge-base.json')
-const { RAG, HyperDBAdapter, QvacLlmAdapter } = require('../index')
-const Corestore = require('corestore')
 const QvacLogger = require('@qvac/logging')
 
-const llamaDriveKey = 'afa79ee07c0a138bb9f11bfaee771fb1bdfca8c82d961cff0474e49827bd1de3'
-const gteDriveKey = 'd1896d9259692818df95bd2480e90c2d057688a4f7c9b1ae13ac7f5ee379d03e'
+const { RAG, HyperDBAdapter, QvacLlmAdapter } = require('../index')
+const knowledgeBase = require('./knowledge-base.json')
+const { ensureModels } = require('./utils')
 
-const modelName = 'gte-large_fp16.gguf'
 const store = new Corestore('./store')
-
-const modelDir = './models'
-
 const query = 'Who won the individual title in LIV Golf UK by JCB in 2025?'
 
 async function main () {
-  // Load the embedder using HyperDriveDL
-  const gteHdDL = new HyperDriveDL({
-    key: `hd://${gteDriveKey}`,
-    store
-  })
+  // 1. Fetch embedder + LLM model files from the QVAC registry (cached on disk after first run).
+  const models = await ensureModels(['embedder', 'llm'])
 
-  const embedderArgs = {
-    loader: gteHdDL,
-    opts: { stats: true },
+  // 2. Construct embedder with the new files-based addon shape.
+  const embedder = new EmbedderPlugin({
+    files: { model: [models.embedder.fullPath] },
+    config: { device: 'gpu', gpu_layers: '99' },
     logger: console,
-    diskPath: modelDir,
-    modelName
-  }
-  const embedder = new EmbedderPlugin(embedderArgs, '-ngl\t99\n-dev\tgpu')
-  await embedder.load(false)
+    opts: { stats: true }
+  })
+  await embedder.load()
 
   const embeddingFunction = async (text) => {
     const response = await embedder.run(text)
@@ -44,39 +36,26 @@ async function main () {
     }
   }
 
-  // Load the LLM using HyperDriveDL
-  const llamaHdDL = new HyperDriveDL({
-    key: `hd://${llamaDriveKey}`,
-    store
-  })
-
-  const llmArgs = {
-    loader: llamaHdDL,
-    opts: { stats: true },
+  // 3. Construct LLM with the new files-based addon shape.
+  const llm = new LlmPlugin({
+    files: { model: [models.llm.fullPath] },
+    config: { device: 'gpu', gpu_layers: '99', ctx_size: '1024' },
     logger: console,
-    diskPath: modelDir,
-    modelName: 'Llama-3.2-1B-Instruct-Q4_0.gguf'
-  }
-  const llm = new LlmPlugin(llmArgs, { ctx_size: '1024', gpu_layers: '99', device: 'gpu' })
-  await llm.load(false)
+    opts: { stats: true }
+  })
+  await llm.load()
   const llmAdapter = new QvacLlmAdapter(llm)
 
-  // Initialize the database adapter
   const dbAdapter = new HyperDBAdapter({ store })
-
-  // Create logger for visibility
   const logger = new QvacLogger(console)
 
-  // Initialize the RAG pipeline
   const rag = new RAG({ embeddingFunction, dbAdapter, llm: llmAdapter, logger })
   await rag.ready()
 
   const knowledgeBaseMapped = knowledgeBase.map(kb => kb.text)
 
-  // Generate embeddings for the knowledge base and save them to the vector database
-  const docs = await rag.ingest(knowledgeBaseMapped, modelName)
+  const docs = await rag.ingest(knowledgeBaseMapped, models.embedder.filename)
 
-  // Generate a response to the user query
   const response = await rag.infer(query)
 
   let fullResponse = ''
@@ -88,17 +67,11 @@ async function main () {
 
   console.log(fullResponse)
 
-  // Delete the embeddings for the knowledge base
   await rag.deleteEmbeddings(docs.processed.map(doc => doc.id))
 
-  // Close the RAG pipeline
   await rag.close()
-
-  // Close HyperDriveDL instances
-  await llamaHdDL.close()
-  await gteHdDL.close()
-
-  // Close the store
+  await llm.unload()
+  await embedder.unload()
   await store.close()
 }
 

packages/rag/examples/utils.js

@@ -0,0 +1,73 @@
+'use strict'
+
+const fs = require('bare-fs')
+const path = require('bare-path')
+const { QVACRegistryClient } = require('@qvac/registry-client')
+
+const DEFAULT_DISK_PATH = './models'
+
+const RAG_MODELS = {
+  embedder: {
+    path: 'ChristianAzinn/gte-large-gguf/blob/f9fa5479908e72c2a8b9d6ba112911cd1e51be53/gte-large_fp16.gguf',
+    source: 'hf',
+    filename: 'gte-large_fp16.gguf'
+  },
+  llm: {
+    path: 'unsloth/Llama-3.2-1B-Instruct-GGUF/blob/b69aef112e9f895e6f98d7ae0949f72ff09aa401/Llama-3.2-1B-Instruct-Q4_0.gguf',
+    source: 'hf',
+    filename: 'Llama-3.2-1B-Instruct-Q4_0.gguf'
+  }
+}
+
+async function ensureModels (keys, diskPath) {
+  diskPath = diskPath || DEFAULT_DISK_PATH
+
+  const requested = keys.map(key => {
+    const model = RAG_MODELS[key]
+    if (!model) {
+      throw new Error(`Unknown model key: ${key}. Available keys: ${Object.keys(RAG_MODELS).join(', ')}`)
+    }
+    return { key, ...model, fullPath: path.join(diskPath, model.filename) }
+  })
+
+  const missing = requested.filter(m => !fs.existsSync(m.fullPath))
+
+  if (missing.length === 0) {
+    console.log('Models already cached locally.')
+    return toResult(requested, diskPath)
+  }
+
+  fs.mkdirSync(diskPath, { recursive: true })
+
+  console.log('Downloading models from QVAC registry...')
+  const client = new QVACRegistryClient()
+
+  try {
+    await client.ready()
+
+    for (const m of missing) {
+      console.log(`  Downloading ${m.filename}...`)
+      await client.downloadModel(m.path, m.source, {
+        outputFile: m.fullPath,
+        timeout: 300000
+      })
+      console.log(`  Downloaded: ${m.filename}`)
+    }
+
+    console.log('Models ready.')
+  } finally {
+    await client.close()
+  }
+
+  return toResult(requested, diskPath)
+}
+
+function toResult (requested, diskPath) {
+  const out = {}
+  for (const m of requested) {
+    out[m.key] = { filename: m.filename, dir: diskPath, fullPath: m.fullPath }
+  }
+  return out
+}
+
+module.exports = { ensureModels, RAG_MODELS, DEFAULT_DISK_PATH }