diff --git a/yarn-project/kv-store/src/indexeddb/map.ts b/yarn-project/kv-store/src/indexeddb/map.ts index 0c3a604d1ccc..41d0fb7fe7b9 100644 --- a/yarn-project/kv-store/src/indexeddb/map.ts +++ b/yarn-project/kv-store/src/indexeddb/map.ts @@ -1,22 +1,22 @@ import type { IDBPDatabase, IDBPObjectStore } from 'idb'; import type { Key, Range } from '../interfaces/common.js'; -import type { AztecAsyncMultiMap } from '../interfaces/map.js'; +import type { AztecAsyncMap } from '../interfaces/map.js'; import type { AztecIDBSchema } from './store.js'; /** * A map backed by IndexedDB. */ -export class IndexedDBAztecMap implements AztecAsyncMultiMap { +export class IndexedDBAztecMap implements AztecAsyncMap { protected name: string; - #container: string; + protected container: string; #_db?: IDBPObjectStore; #rootDB: IDBPDatabase; constructor(rootDB: IDBPDatabase, mapName: string) { this.name = mapName; - this.#container = `map:${mapName}`; + this.container = `map:${mapName}`; this.#rootDB = rootDB; } @@ -29,38 +29,22 @@ export class IndexedDBAztecMap implements AztecAsyncMultiMap { - const data = await this.db.get(this.#slot(key)); + const data = await this.db.get(this.slot(key)); return data?.value as V; } - async *getValuesAsync(key: K): AsyncIterableIterator { - const index = this.db.index('keyCount'); - const rangeQuery = IDBKeyRange.bound( - [this.#container, this.#normalizeKey(key), 0], - [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER], - false, - false, - ); - for await (const cursor of index.iterate(rangeQuery)) { - yield cursor.value.value as V; - } - } - async hasAsync(key: K): Promise { const result = (await this.getAsync(key)) !== undefined; return result; } async set(key: K, val: V): Promise { - const count = await this.db - .index('key') - .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)])); await this.db.put({ value: val, - container: this.#container, - key: this.#normalizeKey(key), - keyCount: count + 1, - slot: this.#slot(key, count), + container: this.container, + key: this.normalizeKey(key), + keyCount: 1, + slot: this.slot(key), }); } @@ -77,30 +61,14 @@ export class IndexedDBAztecMap implements AztecAsyncMultiMap { - await this.db.delete(this.#slot(key)); - } - - async deleteValue(key: K, val: V): Promise { - const index = this.db.index('keyCount'); - const rangeQuery = IDBKeyRange.bound( - [this.#container, this.#normalizeKey(key), 0], - [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER], - false, - false, - ); - for await (const cursor of index.iterate(rangeQuery)) { - if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) { - await cursor.delete(); - return; - } - } + await this.db.delete(this.slot(key)); } async *entriesAsync(range: Range = {}): AsyncIterableIterator<[K, V]> { const index = this.db.index('key'); const rangeQuery = IDBKeyRange.bound( - [this.#container, range.start ?? ''], - [this.#container, range.end ?? '\uffff'], + [this.container, range.start ?? ''], + [this.container, range.end ?? '\uffff'], !!range.reverse, !range.reverse, ); @@ -131,12 +99,12 @@ export class IndexedDBAztecMap implements AztecAsyncMultiMap 1 ? denormalizedKey : key) as K; } - #normalizeKey(key: K): string { + protected normalizeKey(key: K): string { const arrayKey = Array.isArray(key) ? key : [key]; return arrayKey.join(','); } - #slot(key: K, index: number = 0): string { - return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`; + protected slot(key: K, index: number = 0): string { + return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`; } } diff --git a/yarn-project/kv-store/src/indexeddb/multi_map.test.ts b/yarn-project/kv-store/src/indexeddb/multi_map.test.ts new file mode 100644 index 000000000000..b80079395746 --- /dev/null +++ b/yarn-project/kv-store/src/indexeddb/multi_map.test.ts @@ -0,0 +1,7 @@ +import { describeAztecMultiMap } from '../interfaces/multi_map_test_suite.js'; +import { mockLogger } from '../interfaces/utils.js'; +import { AztecIndexedDBStore } from './store.js'; + +describe('IndexedDBMap', () => { + describeAztecMultiMap('AztecMultiMap', async () => await AztecIndexedDBStore.open(mockLogger, undefined, true)); +}); diff --git a/yarn-project/kv-store/src/indexeddb/multi_map.ts b/yarn-project/kv-store/src/indexeddb/multi_map.ts new file mode 100644 index 000000000000..de7829671a6d --- /dev/null +++ b/yarn-project/kv-store/src/indexeddb/multi_map.ts @@ -0,0 +1,53 @@ +import type { Key } from '../interfaces/common.js'; +import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js'; +import { IndexedDBAztecMap } from './map.js'; + +/** + * A multi map backed by IndexedDB. + */ +export class IndexedDBAztecMultiMap + extends IndexedDBAztecMap + implements AztecAsyncMultiMap +{ + override async set(key: K, val: V): Promise { + const count = await this.db + .index('key') + .count(IDBKeyRange.bound([this.container, this.normalizeKey(key)], [this.container, this.normalizeKey(key)])); + await this.db.put({ + value: val, + container: this.container, + key: this.normalizeKey(key), + keyCount: count + 1, + slot: this.slot(key, count), + }); + } + + async *getValuesAsync(key: K): AsyncIterableIterator { + const index = this.db.index('keyCount'); + const rangeQuery = IDBKeyRange.bound( + [this.container, this.normalizeKey(key), 0], + [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER], + false, + false, + ); + for await (const cursor of index.iterate(rangeQuery)) { + yield cursor.value.value as V; + } + } + + async deleteValue(key: K, val: V): Promise { + const index = this.db.index('keyCount'); + const rangeQuery = IDBKeyRange.bound( + [this.container, this.normalizeKey(key), 0], + [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER], + false, + false, + ); + for await (const cursor of index.iterate(rangeQuery)) { + if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) { + await cursor.delete(); + return; + } + } + } +} diff --git a/yarn-project/kv-store/src/indexeddb/store.ts b/yarn-project/kv-store/src/indexeddb/store.ts index d921a5f0cb97..d0d895357cd9 100644 --- a/yarn-project/kv-store/src/indexeddb/store.ts +++ b/yarn-project/kv-store/src/indexeddb/store.ts @@ -5,12 +5,14 @@ import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb'; import type { AztecAsyncArray } from '../interfaces/array.js'; import type { Key, StoreSize } from '../interfaces/common.js'; import type { AztecAsyncCounter } from '../interfaces/counter.js'; -import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js'; +import type { AztecAsyncMap } from '../interfaces/map.js'; +import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js'; import type { AztecAsyncSet } from '../interfaces/set.js'; import type { AztecAsyncSingleton } from '../interfaces/singleton.js'; import type { AztecAsyncKVStore } from '../interfaces/store.js'; import { IndexedDBAztecArray } from './array.js'; import { IndexedDBAztecMap } from './map.js'; +import { IndexedDBAztecMultiMap } from './multi_map.js'; import { IndexedDBAztecSet } from './set.js'; import { IndexedDBAztecSingleton } from './singleton.js'; @@ -34,7 +36,11 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore { #name: string; #containers = new Set< - IndexedDBAztecArray | IndexedDBAztecMap | IndexedDBAztecSet | IndexedDBAztecSingleton + | IndexedDBAztecArray + | IndexedDBAztecMap + | IndexedDBAztecMultiMap + | IndexedDBAztecSet + | IndexedDBAztecSingleton >(); constructor(rootDB: IDBPDatabase, public readonly isEphemeral: boolean, log: Logger, name: string) { @@ -119,7 +125,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore { * @returns A new AztecMultiMap */ openMultiMap(name: string): AztecAsyncMultiMap { - const multimap = new IndexedDBAztecMap(this.#rootDB, name); + const multimap = new IndexedDBAztecMultiMap(this.#rootDB, name); this.#containers.add(multimap); return multimap; } diff --git a/yarn-project/kv-store/src/interfaces/index.ts b/yarn-project/kv-store/src/interfaces/index.ts index 1eeb611d7005..f8cf603c2c7d 100644 --- a/yarn-project/kv-store/src/interfaces/index.ts +++ b/yarn-project/kv-store/src/interfaces/index.ts @@ -4,4 +4,5 @@ export * from './counter.js'; export * from './singleton.js'; export * from './store.js'; export * from './set.js'; +export * from './multi_map.js'; export type { Range, StoreSize } from './common.js'; diff --git a/yarn-project/kv-store/src/interfaces/map.ts b/yarn-project/kv-store/src/interfaces/map.ts index 4a2169656f5a..e498185df0ea 100644 --- a/yarn-project/kv-store/src/interfaces/map.ts +++ b/yarn-project/kv-store/src/interfaces/map.ts @@ -62,40 +62,6 @@ export interface AztecMap extends AztecBaseMap { clear(): Promise; } -export interface AztecMapWithSize extends AztecMap { - /** - * Gets the size of the map. - * @returns The size of the map - */ - size(): number; -} - -/** - * A map backed by a persistent store that can have multiple values for a single key. - */ -export interface AztecMultiMap extends AztecMap { - /** - * Gets all the values at the given key. - * @param key - The key to get the values from - */ - getValues(key: K): IterableIterator; - - /** - * Deletes a specific value at the given key. - * @param key - The key to delete the value at - * @param val - The value to delete - */ - deleteValue(key: K, val: V): Promise; -} - -export interface AztecMultiMapWithSize extends AztecMultiMap { - /** - * Gets the size of the map. - * @returns The size of the map - */ - size(): number; -} - /** * A map backed by a persistent store. */ @@ -131,21 +97,3 @@ export interface AztecAsyncMap extends AztecBaseMap { */ keysAsync(range?: Range): AsyncIterableIterator; } - -/** - * A map backed by a persistent store that can have multiple values for a single key. - */ -export interface AztecAsyncMultiMap extends AztecAsyncMap { - /** - * Gets all the values at the given key. - * @param key - The key to get the values from - */ - getValuesAsync(key: K): AsyncIterableIterator; - - /** - * Deletes a specific value at the given key. - * @param key - The key to delete the value at - * @param val - The value to delete - */ - deleteValue(key: K, val: V): Promise; -} diff --git a/yarn-project/kv-store/src/interfaces/map_test_suite.ts b/yarn-project/kv-store/src/interfaces/map_test_suite.ts index c6419890d3a6..f3d6694aa189 100644 --- a/yarn-project/kv-store/src/interfaces/map_test_suite.ts +++ b/yarn-project/kv-store/src/interfaces/map_test_suite.ts @@ -3,7 +3,7 @@ import { toArray } from '@aztec/foundation/iterable'; import { expect } from 'chai'; import type { Key, Range } from './common.js'; -import type { AztecAsyncMap, AztecAsyncMultiMap, AztecMap, AztecMultiMap } from './map.js'; +import type { AztecAsyncMap, AztecMap } from './map.js'; import type { AztecAsyncKVStore, AztecKVStore } from './store.js'; import { isSyncStore } from './utils.js'; @@ -14,11 +14,11 @@ export function describeAztecMap( ) { describe(testName, () => { let store: AztecKVStore | AztecAsyncKVStore; - let map: AztecMultiMap | AztecAsyncMultiMap; + let map: AztecMap | AztecAsyncMap; beforeEach(async () => { store = await getStore(); - map = store.openMultiMap('test'); + map = store.openMap('test'); }); afterEach(async () => { @@ -27,32 +27,26 @@ export function describeAztecMap( async function get(key: Key, sut: AztecAsyncMap | AztecMap = map) { return isSyncStore(store) && !forceAsync - ? (sut as AztecMultiMap).get(key) - : await (sut as AztecAsyncMultiMap).getAsync(key); + ? (sut as AztecMap).get(key) + : await (sut as AztecAsyncMap).getAsync(key); } async function entries() { return isSyncStore(store) && !forceAsync - ? await toArray((map as AztecMultiMap).entries()) - : await toArray((map as AztecAsyncMultiMap).entriesAsync()); + ? await toArray((map as AztecMap).entries()) + : await toArray((map as AztecAsyncMap).entriesAsync()); } async function values() { return isSyncStore(store) && !forceAsync - ? await toArray((map as AztecMultiMap).values()) - : await toArray((map as AztecAsyncMultiMap).valuesAsync()); + ? await toArray((map as AztecMap).values()) + : await toArray((map as AztecAsyncMap).valuesAsync()); } async function keys(range?: Range, sut: AztecAsyncMap | AztecMap = map) { return isSyncStore(store) && !forceAsync - ? await toArray((sut as AztecMultiMap).keys(range)) - : await toArray((sut as AztecAsyncMultiMap).keysAsync(range)); - } - - async function getValues(key: Key) { - return isSyncStore(store) && !forceAsync - ? await toArray((map as AztecMultiMap).getValues(key)) - : await toArray((map as AztecAsyncMultiMap).getValuesAsync(key)); + ? await toArray((sut as AztecMap).keys(range)) + : await toArray((sut as AztecAsyncMap).keysAsync(range)); } it('should be able to set and get values', async () => { @@ -64,6 +58,13 @@ export function describeAztecMap( expect(await get('quux')).to.equal(undefined); }); + it('should be able to overwrite values', async () => { + await map.set('foo', 'bar'); + await map.set('foo', 'baz'); + + expect(await get('foo')).to.equal('baz'); + }); + it('should be able to set values if they do not exist', async () => { expect(await map.setIfNotExists('foo', 'bar')).to.equal(true); expect(await map.setIfNotExists('foo', 'baz')).to.equal(false); @@ -109,22 +110,6 @@ export function describeAztecMap( expect(await keys()).to.deep.equal(['baz', 'foo']); }); - it('should be able to get multiple values for a single key', async () => { - await map.set('foo', 'bar'); - await map.set('foo', 'baz'); - - expect(await getValues('foo')).to.deep.equal(['bar', 'baz']); - }); - - it('should be able to delete individual values for a single key', async () => { - await map.set('foo', 'bar'); - await map.set('foo', 'baz'); - - await map.deleteValue('foo', 'bar'); - - expect(await getValues('foo')).to.deep.equal(['baz']); - }); - it('supports range queries', async () => { await map.set('a', 'a'); await map.set('b', 'b'); diff --git a/yarn-project/kv-store/src/interfaces/multi_map.ts b/yarn-project/kv-store/src/interfaces/multi_map.ts new file mode 100644 index 000000000000..4e3d7e922c7f --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/multi_map.ts @@ -0,0 +1,38 @@ +import type { Key } from './common.js'; +import type { AztecAsyncMap, AztecMap } from './map.js'; + +/** + * A map backed by a persistent store that can have multiple values for a single key. + */ +export interface AztecMultiMap extends AztecMap { + /** + * Gets all the values at the given key. + * @param key - The key to get the values from + */ + getValues(key: K): IterableIterator; + + /** + * Deletes a specific value at the given key. + * @param key - The key to delete the value at + * @param val - The value to delete + */ + deleteValue(key: K, val: V): Promise; +} + +/** + * A map backed by a persistent store that can have multiple values for a single key. + */ +export interface AztecAsyncMultiMap extends AztecAsyncMap { + /** + * Gets all the values at the given key. + * @param key - The key to get the values from + */ + getValuesAsync(key: K): AsyncIterableIterator; + + /** + * Deletes a specific value at the given key. + * @param key - The key to delete the value at + * @param val - The value to delete + */ + deleteValue(key: K, val: V): Promise; +} diff --git a/yarn-project/kv-store/src/interfaces/multi_map_test_suite.ts b/yarn-project/kv-store/src/interfaces/multi_map_test_suite.ts new file mode 100644 index 000000000000..8e23a11d4eb7 --- /dev/null +++ b/yarn-project/kv-store/src/interfaces/multi_map_test_suite.ts @@ -0,0 +1,143 @@ +import { toArray } from '@aztec/foundation/iterable'; + +import { expect } from 'chai'; + +import type { Key, Range } from './common.js'; +import type { AztecAsyncMultiMap, AztecMultiMap } from './multi_map.js'; +import type { AztecAsyncKVStore, AztecKVStore } from './store.js'; +import { isSyncStore } from './utils.js'; + +export function describeAztecMultiMap( + testName: string, + getStore: () => AztecKVStore | Promise, + forceAsync: boolean = false, +) { + describe(testName, () => { + let store: AztecKVStore | AztecAsyncKVStore; + let multiMap: AztecMultiMap | AztecAsyncMultiMap; + + beforeEach(async () => { + store = await getStore(); + multiMap = store.openMultiMap('test'); + }); + + afterEach(async () => { + await store.delete(); + }); + + async function get(key: Key, sut: AztecAsyncMultiMap | AztecMultiMap = multiMap) { + return isSyncStore(store) && !forceAsync + ? (sut as AztecMultiMap).get(key) + : await (sut as AztecAsyncMultiMap).getAsync(key); + } + + async function entries() { + return isSyncStore(store) && !forceAsync + ? await toArray((multiMap as AztecMultiMap).entries()) + : await toArray((multiMap as AztecAsyncMultiMap).entriesAsync()); + } + + async function values() { + return isSyncStore(store) && !forceAsync + ? await toArray((multiMap as AztecMultiMap).values()) + : await toArray((multiMap as AztecAsyncMultiMap).valuesAsync()); + } + + async function keys(range?: Range, sut: AztecAsyncMultiMap | AztecMultiMap = multiMap) { + return isSyncStore(store) && !forceAsync + ? await toArray((sut as AztecMultiMap).keys(range)) + : await toArray((sut as AztecAsyncMultiMap).keysAsync(range)); + } + + async function getValues(key: Key) { + return isSyncStore(store) && !forceAsync + ? await toArray((multiMap as AztecMultiMap).getValues(key)) + : await toArray((multiMap as AztecAsyncMultiMap).getValuesAsync(key)); + } + + it('should be able to set and get values', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('baz', 'qux'); + + expect(await get('foo')).to.equal('bar'); + expect(await get('baz')).to.equal('qux'); + expect(await get('quux')).to.equal(undefined); + }); + + it('should be able to set values if they do not exist', async () => { + expect(await multiMap.setIfNotExists('foo', 'bar')).to.equal(true); + expect(await multiMap.setIfNotExists('foo', 'baz')).to.equal(false); + + expect(await get('foo')).to.equal('bar'); + }); + + it('should be able to delete values', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('baz', 'qux'); + + await multiMap.delete('foo'); + + expect(await get('foo')).to.equal(undefined); + expect(await get('baz')).to.equal('qux'); + }); + + it('should be able to iterate over entries when there are no keys', async () => { + expect(await entries()).to.deep.equal([]); + }); + + it('should be able to iterate over entries', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('baz', 'qux'); + + expect(await entries()).to.deep.equal([ + ['baz', 'qux'], + ['foo', 'bar'], + ]); + }); + + it('should be able to iterate over values', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('baz', 'quux'); + + expect(await values()).to.deep.equal(['quux', 'bar']); + }); + + it('should be able to iterate over keys', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('baz', 'qux'); + + expect(await keys()).to.deep.equal(['baz', 'foo']); + }); + + it('should be able to get multiple values for a single key', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('foo', 'baz'); + + expect(await getValues('foo')).to.deep.equal(['bar', 'baz']); + }); + + it('should be able to delete individual values for a single key', async () => { + await multiMap.set('foo', 'bar'); + await multiMap.set('foo', 'baz'); + + await multiMap.deleteValue('foo', 'bar'); + + expect(await getValues('foo')).to.deep.equal(['baz']); + }); + + it('supports range queries', async () => { + await multiMap.set('a', 'a'); + await multiMap.set('b', 'b'); + await multiMap.set('c', 'c'); + await multiMap.set('d', 'd'); + + expect(await keys({ start: 'b', end: 'c' })).to.deep.equal(['b']); + expect(await keys({ start: 'b' })).to.deep.equal(['b', 'c', 'd']); + expect(await keys({ end: 'c' })).to.deep.equal(['a', 'b']); + expect(await keys({ start: 'b', end: 'c', reverse: true })).to.deep.equal(['c']); + expect(await keys({ start: 'b', limit: 1 })).to.deep.equal(['b']); + expect(await keys({ start: 'b', reverse: true })).to.deep.equal(['d', 'c']); + expect(await keys({ end: 'b', reverse: true })).to.deep.equal(['b', 'a']); + }); + }); +} diff --git a/yarn-project/kv-store/src/interfaces/store.ts b/yarn-project/kv-store/src/interfaces/store.ts index b45dcec7996f..8f35d33e1947 100644 --- a/yarn-project/kv-store/src/interfaces/store.ts +++ b/yarn-project/kv-store/src/interfaces/store.ts @@ -1,14 +1,8 @@ import type { AztecArray, AztecAsyncArray } from './array.js'; import type { Key, StoreSize } from './common.js'; import type { AztecAsyncCounter, AztecCounter } from './counter.js'; -import type { - AztecAsyncMap, - AztecAsyncMultiMap, - AztecMap, - AztecMapWithSize, - AztecMultiMap, - AztecMultiMapWithSize, -} from './map.js'; +import type { AztecAsyncMap, AztecMap } from './map.js'; +import type { AztecAsyncMultiMap, AztecMultiMap } from './multi_map.js'; import type { AztecAsyncSet, AztecSet } from './set.js'; import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js'; @@ -36,20 +30,6 @@ export interface AztecKVStore { */ openMultiMap(name: string): AztecMultiMap; - /** - * Creates a new multi-map with size. - * @param name - The name of the multi-map - * @returns The multi-map - */ - openMultiMapWithSize(name: string): AztecMultiMapWithSize; - - /** - * Creates a new map with size. - * @param name - The name of the map - * @returns The map - */ - openMapWithSize(name: string): AztecMapWithSize; - /** * Creates a new array. * @param name - The name of the array diff --git a/yarn-project/kv-store/src/lmdb-v2/map.test.ts b/yarn-project/kv-store/src/lmdb-v2/map.test.ts deleted file mode 100644 index c49726ceabea..000000000000 --- a/yarn-project/kv-store/src/lmdb-v2/map.test.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { describeAztecMap } from '../interfaces/map_test_suite.js'; -import { openTmpStore } from './factory.js'; - -describeAztecMap('LMDBMap', () => openTmpStore('test'), true); diff --git a/yarn-project/kv-store/src/lmdb-v2/map.ts b/yarn-project/kv-store/src/lmdb-v2/map.ts index 7a451c68f787..36d8f39e3e87 100644 --- a/yarn-project/kv-store/src/lmdb-v2/map.ts +++ b/yarn-project/kv-store/src/lmdb-v2/map.ts @@ -1,7 +1,7 @@ import { Encoder } from 'msgpackr'; import type { Key, Range } from '../interfaces/common.js'; -import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js'; +import type { AztecAsyncMap } from '../interfaces/map.js'; import type { ReadTransaction } from './read_transaction.js'; // eslint-disable-next-line import/no-cycle import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js'; @@ -113,122 +113,3 @@ export class LMDBMap implements AztecAsyncMap { } } } - -export class LMDBMultiMap implements AztecAsyncMultiMap { - private prefix: string; - private encoder = new Encoder(); - constructor(private store: AztecLMDBStoreV2, name: string) { - this.prefix = `multimap:${name}`; - } - - /** - * Sets the value at the given key. - * @param key - The key to set the value at - * @param val - The value to set - */ - set(key: K, val: V): Promise { - return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val))); - } - - /** - * Sets the value at the given key if it does not already exist. - * @param key - The key to set the value at - * @param val - The value to set - */ - setIfNotExists(key: K, val: V): Promise { - return execInWriteTx(this.store, async tx => { - const exists = !!(await this.getAsync(key)); - if (!exists) { - await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)); - return true; - } - return false; - }); - } - - /** - * Deletes the value at the given key. - * @param key - The key to delete the value at - */ - delete(key: K): Promise { - return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key))); - } - - getAsync(key: K): Promise { - return execInReadTx(this.store, async tx => { - const val = await tx.getIndex(serializeKey(this.prefix, key)); - return val.length > 0 ? this.encoder.unpack(val[0]) : undefined; - }); - } - - hasAsync(key: K): Promise { - return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0); - } - - /** - * Iterates over the map's key-value entries in the key's natural order - * @param range - The range of keys to iterate over - */ - async *entriesAsync(range?: Range): AsyncIterableIterator<[K, V]> { - const reverse = range?.reverse ?? false; - const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix); - const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined; - - let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx(); - const shouldClose = !tx; - tx ??= this.store.getReadTx(); - - try { - for await (const [key, vals] of tx.iterateIndex( - reverse ? endKey! : startKey, - reverse ? startKey : endKey, - reverse, - range?.limit, - )) { - const deserializedKey = deserializeKey(this.prefix, key); - if (!deserializedKey) { - break; - } - - for (const val of vals) { - yield [deserializedKey, this.encoder.unpack(val)]; - } - } - } finally { - if (shouldClose) { - tx.close(); - } - } - } - - /** - * Iterates over the map's values in the key's natural order - * @param range - The range of keys to iterate over - */ - async *valuesAsync(range?: Range): AsyncIterableIterator { - for await (const [_, value] of this.entriesAsync(range)) { - yield value; - } - } - - /** - * Iterates over the map's keys in the key's natural order - * @param range - The range of keys to iterate over - */ - async *keysAsync(range?: Range): AsyncIterableIterator { - for await (const [key, _] of this.entriesAsync(range)) { - yield key; - } - } - - deleteValue(key: K, val: V | undefined): Promise { - return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val))); - } - - async *getValuesAsync(key: K): AsyncIterableIterator { - const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key))); - for (const value of values) { - yield this.encoder.unpack(value); - } - } -} diff --git a/yarn-project/kv-store/src/lmdb-v2/multi_map.test.ts b/yarn-project/kv-store/src/lmdb-v2/multi_map.test.ts new file mode 100644 index 000000000000..be68041a7283 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb-v2/multi_map.test.ts @@ -0,0 +1,4 @@ +import { describeAztecMultiMap } from '../interfaces/multi_map_test_suite.js'; +import { openTmpStore } from './factory.js'; + +describeAztecMultiMap('LMDBMultiMap', () => openTmpStore('test'), true); diff --git a/yarn-project/kv-store/src/lmdb-v2/multi_map.ts b/yarn-project/kv-store/src/lmdb-v2/multi_map.ts new file mode 100644 index 000000000000..b6d863929e73 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb-v2/multi_map.ts @@ -0,0 +1,126 @@ +import { Encoder } from 'msgpackr/pack'; + +import type { Key, Range } from '../interfaces/common.js'; +import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js'; +import type { ReadTransaction } from './read_transaction.js'; +import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js'; +import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js'; + +export class LMDBMultiMap implements AztecAsyncMultiMap { + private prefix: string; + private encoder = new Encoder(); + constructor(private store: AztecLMDBStoreV2, name: string) { + this.prefix = `multimap:${name}`; + } + + /** + * Sets the value at the given key. + * @param key - The key to set the value at + * @param val - The value to set + */ + set(key: K, val: V): Promise { + return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val))); + } + + /** + * Sets the value at the given key if it does not already exist. + * @param key - The key to set the value at + * @param val - The value to set + */ + setIfNotExists(key: K, val: V): Promise { + return execInWriteTx(this.store, async tx => { + const exists = !!(await this.getAsync(key)); + if (!exists) { + await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)); + return true; + } + return false; + }); + } + + /** + * Deletes the value at the given key. + * @param key - The key to delete the value at + */ + delete(key: K): Promise { + return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key))); + } + + getAsync(key: K): Promise { + return execInReadTx(this.store, async tx => { + const val = await tx.getIndex(serializeKey(this.prefix, key)); + return val.length > 0 ? this.encoder.unpack(val[0]) : undefined; + }); + } + + hasAsync(key: K): Promise { + return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0); + } + + /** + * Iterates over the map's key-value entries in the key's natural order + * @param range - The range of keys to iterate over + */ + async *entriesAsync(range?: Range): AsyncIterableIterator<[K, V]> { + const reverse = range?.reverse ?? false; + const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix); + const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined; + + let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx(); + const shouldClose = !tx; + tx ??= this.store.getReadTx(); + + try { + for await (const [key, vals] of tx.iterateIndex( + reverse ? endKey! : startKey, + reverse ? startKey : endKey, + reverse, + range?.limit, + )) { + const deserializedKey = deserializeKey(this.prefix, key); + if (!deserializedKey) { + break; + } + + for (const val of vals) { + yield [deserializedKey, this.encoder.unpack(val)]; + } + } + } finally { + if (shouldClose) { + tx.close(); + } + } + } + + /** + * Iterates over the map's values in the key's natural order + * @param range - The range of keys to iterate over + */ + async *valuesAsync(range?: Range): AsyncIterableIterator { + for await (const [_, value] of this.entriesAsync(range)) { + yield value; + } + } + + /** + * Iterates over the map's keys in the key's natural order + * @param range - The range of keys to iterate over + */ + async *keysAsync(range?: Range): AsyncIterableIterator { + for await (const [key, _] of this.entriesAsync(range)) { + yield key; + } + } + + deleteValue(key: K, val: V | undefined): Promise { + return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val))); + } + + async *getValuesAsync(key: K): AsyncIterableIterator { + const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key))); + for (const value of values) { + yield this.encoder.unpack(value); + } + } +} diff --git a/yarn-project/kv-store/src/lmdb-v2/store.ts b/yarn-project/kv-store/src/lmdb-v2/store.ts index c4aa30cf2104..6c17193c54f4 100644 --- a/yarn-project/kv-store/src/lmdb-v2/store.ts +++ b/yarn-project/kv-store/src/lmdb-v2/store.ts @@ -8,14 +8,15 @@ import { rm } from 'fs/promises'; import type { AztecAsyncArray } from '../interfaces/array.js'; import type { Key, StoreSize } from '../interfaces/common.js'; import type { AztecAsyncCounter } from '../interfaces/counter.js'; -import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js'; +import type { AztecAsyncMap } from '../interfaces/map.js'; +import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js'; import type { AztecAsyncSet } from '../interfaces/set.js'; import type { AztecAsyncSingleton } from '../interfaces/singleton.js'; import type { AztecAsyncKVStore } from '../interfaces/store.js'; // eslint-disable-next-line import/no-cycle import { LMDBArray } from './array.js'; // eslint-disable-next-line import/no-cycle -import { LMDBMap, LMDBMultiMap } from './map.js'; +import { LMDBMap } from './map.js'; import { Database, type LMDBMessageChannel, @@ -23,6 +24,7 @@ import { type LMDBRequestBody, type LMDBResponseBody, } from './message.js'; +import { LMDBMultiMap } from './multi_map.js'; import { ReadTransaction } from './read_transaction.js'; // eslint-disable-next-line import/no-cycle import { LMDBSingleValue } from './singleton.js'; diff --git a/yarn-project/kv-store/src/lmdb/map.test.ts b/yarn-project/kv-store/src/lmdb/map.test.ts index bcc5fa12febf..224750df8e91 100644 --- a/yarn-project/kv-store/src/lmdb/map.test.ts +++ b/yarn-project/kv-store/src/lmdb/map.test.ts @@ -1,6 +1,3 @@ -import { expect } from 'chai'; - -import type { AztecMapWithSize, AztecMultiMapWithSize } from '../interfaces/map.js'; import { describeAztecMap } from '../interfaces/map_test_suite.js'; import { openTmpStore } from './index.js'; @@ -9,53 +6,3 @@ describe('LMDBMap', () => { describeAztecMap('Async AztecMap', () => Promise.resolve(openTmpStore(true)), true); }); - -describe('AztecMultiMapWithSize', () => { - let map: AztecMultiMapWithSize; - let map2: AztecMultiMapWithSize; - - beforeEach(() => { - const store = openTmpStore(true); - map = store.openMultiMapWithSize('test'); - map2 = store.openMultiMapWithSize('test2'); - }); - - it('should be able to delete values', async () => { - await map.set('foo', 'bar'); - await map.set('foo', 'baz'); - - await map2.set('foo', 'bar'); - await map2.set('foo', 'baz'); - - expect(map.size()).to.equal(2); - expect(map2.size()).to.equal(2); - - await map.deleteValue('foo', 'bar'); - - expect(map.size()).to.equal(1); - expect(map.get('foo')).to.equal('baz'); - - expect(map2.size()).to.equal(2); - }); -}); - -describe('AztecMapWithSize', () => { - let map: AztecMapWithSize; - - beforeEach(() => { - const store = openTmpStore(true); - map = store.openMapWithSize('test'); - }); - - it('should be able to delete values', async () => { - await map.set('foo', 'bar'); - await map.set('fizz', 'buzz'); - - expect(map.size()).to.equal(2); - - await map.delete('foo'); - - expect(map.size()).to.equal(1); - expect(map.get('fizz')).to.equal('buzz'); - }); -}); diff --git a/yarn-project/kv-store/src/lmdb/map.ts b/yarn-project/kv-store/src/lmdb/map.ts index 9b9843e4e405..148d91361557 100644 --- a/yarn-project/kv-store/src/lmdb/map.ts +++ b/yarn-project/kv-store/src/lmdb/map.ts @@ -1,7 +1,7 @@ import type { Database, RangeOptions } from 'lmdb'; import type { Key, Range } from '../interfaces/common.js'; -import type { AztecAsyncMultiMap, AztecMapWithSize, AztecMultiMap } from '../interfaces/map.js'; +import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js'; /** The slot where a key-value entry would be stored */ type MapValueSlot = ['map', string, 'slot', K]; @@ -9,7 +9,7 @@ type MapValueSlot = ['map', string, 'slot', K]; /** * A map backed by LMDB. */ -export class LmdbAztecMap implements AztecMultiMap, AztecAsyncMultiMap { +export class LmdbAztecMap implements AztecMap, AztecAsyncMap { protected db: Database<[K, V], MapValueSlot>; protected name: string; @@ -39,26 +39,6 @@ export class LmdbAztecMap implements AztecMultiMap, Azte return Promise.resolve(this.get(key)); } - *getValues(key: K): IterableIterator { - const transaction = this.db.useReadTransaction(); - try { - const values = this.db.getValues(this.slot(key), { - transaction, - }); - for (const value of values) { - yield value?.[1]; - } - } finally { - transaction.done(); - } - } - - async *getValuesAsync(key: K): AsyncIterableIterator { - for (const value of this.getValues(key)) { - yield value; - } - } - has(key: K): boolean { return this.db.doesExist(this.slot(key)); } @@ -90,10 +70,6 @@ export class LmdbAztecMap implements AztecMultiMap, Azte await this.db.remove(this.slot(key)); } - async deleteValue(key: K, val: V): Promise { - await this.db.remove(this.slot(key), [key, val]); - } - *entries(range: Range = {}): IterableIterator<[K, V]> { const transaction = this.db.useReadTransaction(); @@ -184,65 +160,3 @@ export class LmdbAztecMap implements AztecMultiMap, Azte } } } - -export class LmdbAztecMapWithSize - extends LmdbAztecMap - implements AztecMapWithSize, AztecAsyncMultiMap -{ - #sizeCache?: number; - - constructor(rootDb: Database, mapName: string) { - super(rootDb, mapName); - } - - override async set(key: K, val: V): Promise { - await this.db.childTransaction(() => { - const exists = this.db.doesExist(this.slot(key)); - this.db.putSync(this.slot(key), [key, val], { - appendDup: true, - }); - if (!exists) { - this.#sizeCache = undefined; // Invalidate cache - } - }); - } - - override async delete(key: K): Promise { - await this.db.childTransaction(async () => { - const exists = this.db.doesExist(this.slot(key)); - if (exists) { - await this.db.remove(this.slot(key)); - this.#sizeCache = undefined; // Invalidate cache - } - }); - } - - override async deleteValue(key: K, val: V): Promise { - await this.db.childTransaction(async () => { - const exists = this.db.doesExist(this.slot(key)); - if (exists) { - await this.db.remove(this.slot(key), [key, val]); - this.#sizeCache = undefined; // Invalidate cache - } - }); - } - - /** - * Gets the size of the map by counting entries. - * @returns The number of entries in the map - */ - size(): number { - if (this.#sizeCache === undefined) { - this.#sizeCache = this.db.getCount({ - start: this.startSentinel, - end: this.endSentinel, - }); - } - return this.#sizeCache; - } - - // Reset cache on clear/drop operations - clearCache() { - this.#sizeCache = undefined; - } -} diff --git a/yarn-project/kv-store/src/lmdb/multi_map.test.ts b/yarn-project/kv-store/src/lmdb/multi_map.test.ts new file mode 100644 index 000000000000..77fad77c98ef --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/multi_map.test.ts @@ -0,0 +1,8 @@ +import { describeAztecMultiMap } from '../interfaces/multi_map_test_suite.js'; +import { openTmpStore } from './index.js'; + +describe('LMDBMultiMap', () => { + describeAztecMultiMap('Sync AztecMultiMap', () => openTmpStore(true)); + + describeAztecMultiMap('Async AztecMultiMap', () => Promise.resolve(openTmpStore(true)), true); +}); diff --git a/yarn-project/kv-store/src/lmdb/multi_map.ts b/yarn-project/kv-store/src/lmdb/multi_map.ts new file mode 100644 index 000000000000..fad497032028 --- /dev/null +++ b/yarn-project/kv-store/src/lmdb/multi_map.ts @@ -0,0 +1,35 @@ +import type { Key } from '../interfaces/common.js'; +import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js'; +import { LmdbAztecMap } from './map.js'; + +/** + * A map backed by LMDB. + */ +export class LmdbAztecMultiMap + extends LmdbAztecMap + implements AztecMultiMap, AztecAsyncMultiMap +{ + *getValues(key: K): IterableIterator { + const transaction = this.db.useReadTransaction(); + try { + const values = this.db.getValues(this.slot(key), { + transaction, + }); + for (const value of values) { + yield value?.[1]; + } + } finally { + transaction.done(); + } + } + + async *getValuesAsync(key: K): AsyncIterableIterator { + for (const value of this.getValues(key)) { + yield value; + } + } + + async deleteValue(key: K, val: V): Promise { + await this.db.remove(this.slot(key), [key, val]); + } +} diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 18e458bb2582..5a5a85c704b8 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -9,20 +9,15 @@ import { join } from 'path'; import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js'; import type { Key, StoreSize } from '../interfaces/common.js'; import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js'; -import type { - AztecAsyncMap, - AztecAsyncMultiMap, - AztecMap, - AztecMapWithSize, - AztecMultiMap, - AztecMultiMapWithSize, -} from '../interfaces/map.js'; +import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js'; +import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js'; import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js'; import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js'; import type { AztecAsyncKVStore, AztecKVStore } from '../interfaces/store.js'; import { LmdbAztecArray } from './array.js'; import { LmdbAztecCounter } from './counter.js'; -import { LmdbAztecMap, LmdbAztecMapWithSize } from './map.js'; +import { LmdbAztecMap } from './map.js'; +import { LmdbAztecMultiMap } from './multi_map.js'; import { LmdbAztecSet } from './set.js'; import { LmdbAztecSingleton } from './singleton.js'; @@ -119,29 +114,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore { * @returns A new AztecMultiMap */ openMultiMap(name: string): AztecMultiMap & AztecAsyncMultiMap { - return new LmdbAztecMap(this.#multiMapData, name); + return new LmdbAztecMultiMap(this.#multiMapData, name); } openCounter(name: string): AztecCounter & AztecAsyncCounter { return new LmdbAztecCounter(this.#data, name); } - /** - * Creates a new AztecMultiMapWithSize in the store. A multi-map with size stores multiple values for a single key automatically. - * @param name - Name of the map - * @returns A new AztecMultiMapWithSize - */ - openMultiMapWithSize(name: string): AztecMultiMapWithSize { - return new LmdbAztecMapWithSize(this.#multiMapData, name); - } - - /** - * Creates a new AztecMapWithSize in the store. - * @param name - Name of the map - * @returns A new AztecMapWithSize - */ - openMapWithSize(name: string): AztecMapWithSize { - return new LmdbAztecMapWithSize(this.#data, name); - } /** * Creates a new AztecArray in the store.