Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion packages/rag/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ Each pluggable adapter has specific dependency requirements. Choose the adapters

```bash
npm install corestore hyperdb hyperschema

# Browser/RN environments without Node-style crypto.createHash
npm install crypto-browserify
```

**`BaseDBAdapter`** - Custom database interface
Expand Down Expand Up @@ -143,7 +146,7 @@ npm install @qvac/rag

**Installation Strategy:**

- **Minimal production bundle**: Only 3 core dependencies (`@qvac/error`, `ready-resource`, `uuid-random`)
- **Minimal production bundle**: Small core dependency set; adapter dependencies are loaded only when their code paths are used
- **Tests work out of the box**: Adapter deps included in `devDependencies` for seamless testing
- **Production efficiency**: Use `npm install --omit=dev` to exclude testing dependencies
- **Pick and choose**: Install only the adapter dependencies you actually need
Expand Down
2 changes: 1 addition & 1 deletion packages/rag/examples/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ async function ensureModels (keys, diskPath) {
if (!model) {
throw new Error(`Unknown model key: ${key}. Available keys: ${Object.keys(RAG_MODELS).join(', ')}`)
}
return { key, ...model, fullPath: path.join(diskPath, model.filename) }
return { key, ...model, fullPath: path.resolve(diskPath, model.filename) }
})

const missing = requested.filter(m => !fs.existsSync(m.fullPath))
Expand Down
19 changes: 18 additions & 1 deletion packages/rag/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,20 @@
"NOTICE"
],
"types": "index.d.ts",
"imports": {
"#crypto": {
"bare": "bare-crypto",
"node": "node:crypto",
"react-native": "./src/shims/crypto.js",
"default": "./src/shims/crypto.js"
},
"#fetch": {
"bare": "bare-fetch",
"node": "./src/shims/fetch.js",
"react-native": "./src/shims/fetch.js",
"default": "./src/shims/fetch.js"
}
},
"devDependencies": {
"@qvac/embed-llamacpp": "^0.14.0",
"@qvac/llm-llamacpp": "^0.16.0",
Expand All @@ -44,12 +58,12 @@
"dependencies": {
"@qvac/error": "^0.1.1",
"ready-resource": "^1.1.2",
"uuid-random": "^1.3.2",
"zod": "^4.1.13"
},
"peerDependencies": {
"bare-crypto": "^1.13.4",
"bare-fetch": "^2.5.0",
"crypto-browserify": "^3.12.1",
"hyperdb": "^4.19.0",
"hyperdht": "^6.23.0",
"hyperschema": "^1.13.0",
Expand All @@ -62,6 +76,9 @@
"bare-fetch": {
"optional": true
},
"crypto-browserify": {
"optional": true
},
"hyperdb": {
"optional": true
},
Expand Down
18 changes: 1 addition & 17 deletions packages/rag/src/adapters/database/HyperDBAdapter.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,7 @@ const {
} = require('../../utils/helper')
const QvacLogger = require('@qvac/logging')

let qvacCrypto
try {
qvacCrypto = require('crypto')
} catch (e) {
try {
qvacCrypto = require('bare-crypto')
} catch (e2) {
if (typeof global !== 'undefined' && global.crypto && global.crypto.createHash) {
qvacCrypto = global.crypto
} else {
throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'No crypto implementation found. Please ensure a crypto module is available in your environment.'
})
}
}
}
const qvacCrypto = require('#crypto')

class HyperDBAdapter extends BaseDBAdapter {
/**
Expand Down
10 changes: 7 additions & 3 deletions packages/rag/src/adapters/llm/HttpLlmAdapter.js
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ class HttpLlmAdapter extends BaseLlmAdapter {
*/
async _makeHttpRequest (requestBody) {
try {
const fetch = await import('bare-fetch').then(module => module.default || module)
const fetch = await import('#fetch').then(module => module.default || module)

const response = await fetch(this.httpConfig.apiUrl, {
method: this.httpConfig.method,
Expand All @@ -94,8 +94,12 @@ class HttpLlmAdapter extends BaseLlmAdapter {

return response.json()
} catch (error) {
if ((error.code === 'MODULE_NOT_FOUND' || error.code === 'ERR_MODULE_NOT_FOUND') && error.message.includes('bare-fetch')) {
throw new QvacErrorRAG({ code: ERR_CODES.DEPENDENCY_REQUIRED, adds: 'bare-fetch is required for HttpLlmAdapter.', cause: error })
if ((error.code === 'MODULE_NOT_FOUND' || error.code === 'ERR_MODULE_NOT_FOUND') && (error.message.includes('bare-fetch') || error.message.includes('#fetch'))) {
throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'Fetch unavailable: #fetch could not resolve. Bare: install bare-fetch; otherwise ensure globalThis.fetch exists and your bundler supports package imports.',
cause: error
})
}
throw error
}
Expand Down
25 changes: 25 additions & 0 deletions packages/rag/src/shims/crypto.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
'use strict'

const { QvacErrorRAG, ERR_CODES } = require('../errors')

function ensureCrypto () {
const crypto = typeof globalThis !== 'undefined' ? globalThis.crypto : null
if (crypto && crypto !== module.exports && typeof crypto.createHash === 'function') {
return crypto
}
throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'No Node-style crypto implementation available. This code path requires globalThis.crypto.createHash, including HyperDB document hashing. Bare: install bare-crypto. Node: node:crypto is used by the package import map. Browser/RN: install and configure crypto-browserify, or another createHash-compatible polyfill, before using APIs that depend on #crypto.'
})
}

const cryptoShim = new Proxy({}, {
get (_target, prop) {
return ensureCrypto()[prop]
},
has (_target, prop) {
try { return prop in ensureCrypto() } catch { return false }
}
})

module.exports = cryptoShim
20 changes: 20 additions & 0 deletions packages/rag/src/shims/fetch.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
'use strict'

const { QvacErrorRAG, ERR_CODES } = require('../errors')

function ensureFetch () {
if (typeof globalThis !== 'undefined' && typeof globalThis.fetch === 'function') {
return globalThis.fetch.bind(globalThis)
}
throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'No fetch implementation found. Please ensure a Fetch-compatible globalThis.fetch is available. Bare: install bare-fetch. Node 18+ and browser/RN environments normally provide fetch globally.'
})
}

function fetchProxy (...args) {
return ensureFetch()(...args)
}

module.exports = fetchProxy
module.exports.default = fetchProxy
62 changes: 48 additions & 14 deletions packages/rag/src/utils/helper.js
Original file line number Diff line number Diff line change
@@ -1,19 +1,9 @@
'use strict'

const { QvacErrorRAG, ERR_CODES } = require('../errors')
// Set up crypto polyfill for uuid-random
try {
const crypto = require('bare-crypto')
global.crypto = crypto
} catch (e2) {
if (typeof global === 'undefined' || (typeof global !== 'undefined' && !global.crypto)) {
throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'No crypto implementation found. Please ensure a crypto module is available in your environment.'
})
}
}
const uuid = require('uuid-random')

const UUID_BYTES = 16
const BYTE_TO_HEX = Array.from({ length: 256 }, (_, i) => (i + 0x100).toString(16).slice(1))

/**
* Calculate the cosine similarity between two vectors.
Expand Down Expand Up @@ -98,7 +88,51 @@ function normalizeDocs (docs) {
* @returns {string} A unique identifier.
*/
function generateId () {
return uuid()
const bytes = randomBytes(UUID_BYTES)
bytes[6] = (bytes[6] & 0x0f) | 0x40
bytes[8] = (bytes[8] & 0x3f) | 0x80

return BYTE_TO_HEX[bytes[0]] + BYTE_TO_HEX[bytes[1]] +
BYTE_TO_HEX[bytes[2]] + BYTE_TO_HEX[bytes[3]] + '-' +
BYTE_TO_HEX[bytes[4]] + BYTE_TO_HEX[bytes[5]] + '-' +
BYTE_TO_HEX[bytes[6]] + BYTE_TO_HEX[bytes[7]] + '-' +
BYTE_TO_HEX[bytes[8]] + BYTE_TO_HEX[bytes[9]] + '-' +
BYTE_TO_HEX[bytes[10]] + BYTE_TO_HEX[bytes[11]] +
BYTE_TO_HEX[bytes[12]] + BYTE_TO_HEX[bytes[13]] +
BYTE_TO_HEX[bytes[14]] + BYTE_TO_HEX[bytes[15]]
}

function randomBytes (size) {
try {
const crypto = typeof globalThis !== 'undefined' ? globalThis.crypto : null
if (crypto && typeof crypto.getRandomValues === 'function') {
return crypto.getRandomValues(new Uint8Array(size))
}
} catch {}

try {
const cryptoMod = require('#crypto')
if (cryptoMod && typeof cryptoMod.randomBytes === 'function') {
return toUint8Array(cryptoMod.randomBytes(size))
}
if (cryptoMod && typeof cryptoMod.getRandomValues === 'function') {
return cryptoMod.getRandomValues(new Uint8Array(size))
}
} catch {}

throw new QvacErrorRAG({
code: ERR_CODES.DEPENDENCY_REQUIRED,
adds: 'No secure random byte source available for UUID generation. Provide globalThis.crypto.getRandomValues or a #crypto implementation with randomBytes/getRandomValues.'
})
}

function toUint8Array (bytes) {
if (bytes instanceof Uint8Array) return bytes
if (typeof ArrayBuffer !== 'undefined' && bytes instanceof ArrayBuffer) return new Uint8Array(bytes)
if (typeof ArrayBuffer !== 'undefined' && ArrayBuffer.isView(bytes)) {
return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
}
return Uint8Array.from(bytes)
}

/**
Expand Down
17 changes: 17 additions & 0 deletions packages/rag/test/unit/helper.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,23 @@ test('generateId: should generate multiple unique IDs', t => {
t.is(ids.size, count, 'All generated IDs should be unique')
})

test('generateId: should fall back to #crypto when global crypto is unusable', t => {
const original = globalThis.crypto
const cryptoShim = require('../../src/shims/crypto')
const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i

globalThis.crypto = cryptoShim
try {
t.ok(uuidPattern.test(generateId()), 'Generated ID should match UUID v4 pattern')
} finally {
if (original === undefined) {
delete globalThis.crypto
} else {
globalThis.crypto = original
}
}
})

test('normalizeDocs: comprehensive mixed scenario', t => {
const docs = [
'String document 1',
Expand Down
62 changes: 62 additions & 0 deletions packages/rag/test/unit/shim-crypto.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
'use strict'

const test = require('brittle')
const cryptoShim = require('../../src/shims/crypto')
const { QvacErrorRAG, ERR_CODES } = require('../../src/errors')

test('crypto shim: throws QvacErrorRAG when no crypto implementation is available', t => {
const original = globalThis.crypto
// Force the shim's resolver to find no implementation.
delete globalThis.crypto

try {
const probe = cryptoShim.createHash
t.fail(`Expected accessing a property on the shim to throw, got ${typeof probe}`)
} catch (err) {
t.ok(err instanceof QvacErrorRAG, 'Error should be instance of QvacErrorRAG')
t.is(err.code, ERR_CODES.DEPENDENCY_REQUIRED, 'Error code should be DEPENDENCY_REQUIRED')
t.ok(err.message.includes('crypto-browserify'), 'Error should mention crypto-browserify')
} finally {
if (original !== undefined) globalThis.crypto = original
}
})

test('crypto shim: delegates property access to globalThis.crypto when available', t => {
const original = globalThis.crypto
const stub = {
createHash: () => 'stub',
anything: 'value'
}
globalThis.crypto = stub

try {
t.is(typeof cryptoShim.createHash, 'function', 'createHash should be delegated as a function')
t.is(cryptoShim.createHash(), 'stub', 'createHash invocation should return stubbed value')
t.is(cryptoShim.anything, 'value', 'arbitrary properties should be delegated to the stub')
} finally {
if (original === undefined) {
delete globalThis.crypto
} else {
globalThis.crypto = original
}
}
})

test('crypto shim: rejects self-referential global crypto', t => {
const original = globalThis.crypto
globalThis.crypto = cryptoShim

try {
const probe = cryptoShim.createHash
t.fail(`Expected accessing a property on the self-referential shim to throw, got ${typeof probe}`)
} catch (err) {
t.ok(err instanceof QvacErrorRAG, 'Error should be instance of QvacErrorRAG')
t.is(err.code, ERR_CODES.DEPENDENCY_REQUIRED, 'Error code should be DEPENDENCY_REQUIRED')
} finally {
if (original === undefined) {
delete globalThis.crypto
} else {
globalThis.crypto = original
}
}
})
50 changes: 50 additions & 0 deletions packages/rag/test/unit/shim-fetch.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
'use strict'

const test = require('brittle')
const fetchShim = require('../../src/shims/fetch')
const { QvacErrorRAG, ERR_CODES } = require('../../src/errors')

test('fetch shim: throws QvacErrorRAG when no fetch implementation is available', async t => {
const original = globalThis.fetch
// Force the shim's resolver to find no implementation.
delete globalThis.fetch

try {
await fetchShim('https://example.test')
t.fail('Expected calling the shim to throw')
} catch (err) {
t.ok(err instanceof QvacErrorRAG, 'Error should be instance of QvacErrorRAG')
t.is(err.code, ERR_CODES.DEPENDENCY_REQUIRED, 'Error code should be DEPENDENCY_REQUIRED')
t.ok(err.message.includes('globalThis.fetch'), 'Error should mention globalThis.fetch')
} finally {
if (original !== undefined) globalThis.fetch = original
}
})

test('fetch shim: delegates calls to globalThis.fetch when available', async t => {
const original = globalThis.fetch
let receivedArgs
globalThis.fetch = async function stub (...args) {
receivedArgs = args
return { ok: true, url: args[0] }
}

try {
const result = await fetchShim('https://example.test', { method: 'GET' })
t.ok(result.ok, 'Proxy should return the stub response')
t.is(result.url, 'https://example.test', 'Proxy should pass through positional args')
t.is(receivedArgs[0], 'https://example.test', 'First arg forwarded to stub')
t.alike(receivedArgs[1], { method: 'GET' }, 'Second arg forwarded to stub')
} finally {
if (original === undefined) {
delete globalThis.fetch
} else {
globalThis.fetch = original
}
}
})

test('fetch shim: exposes a default export that aliases the same function', t => {
t.is(typeof fetchShim, 'function', 'Module export should be a function')
t.is(fetchShim.default, fetchShim, 'default property should reference the same function')
})
Loading