-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: SQLite-based package cache (#26608)
Co-authored-by: Michael Kriese <[email protected]>
- Loading branch information
Showing
7 changed files
with
379 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,55 @@ | ||
import { withDir } from 'tmp-promise'; | ||
import { GlobalConfig } from '../../../config/global'; | ||
import { SqlitePackageCache } from './sqlite'; | ||
|
||
function withSqlite<T>( | ||
fn: (sqlite: SqlitePackageCache) => Promise<T>, | ||
): Promise<T> { | ||
return withDir( | ||
async ({ path }) => { | ||
GlobalConfig.set({ cacheDir: path }); | ||
const sqlite = await SqlitePackageCache.init(path); | ||
const res = await fn(sqlite); | ||
await sqlite.cleanup(); | ||
return res; | ||
}, | ||
{ unsafeCleanup: true }, | ||
); | ||
} | ||
|
||
describe('util/cache/package/sqlite', () => { | ||
it('should get undefined', async () => { | ||
const res = await withSqlite((sqlite) => sqlite.get('foo', 'bar')); | ||
expect(res).toBeUndefined(); | ||
}); | ||
|
||
it('should set and get', async () => { | ||
const res = await withSqlite(async (sqlite) => { | ||
await sqlite.set('foo', 'bar', { foo: 'foo' }); | ||
await sqlite.set('foo', 'bar', { bar: 'bar' }); | ||
await sqlite.set('foo', 'bar', { baz: 'baz' }); | ||
return sqlite.get('foo', 'bar'); | ||
}); | ||
expect(res).toEqual({ baz: 'baz' }); | ||
}); | ||
|
||
it('reopens', async () => { | ||
const res = await withDir( | ||
async ({ path }) => { | ||
GlobalConfig.set({ cacheDir: path }); | ||
|
||
const client1 = await SqlitePackageCache.init(path); | ||
await client1.set('foo', 'bar', 'baz'); | ||
await client1.cleanup(); | ||
|
||
const client2 = await SqlitePackageCache.init(path); | ||
const res = await client2.get('foo', 'bar'); | ||
await client2.cleanup(); | ||
return res; | ||
}, | ||
{ unsafeCleanup: true }, | ||
); | ||
|
||
expect(res).toBe('baz'); | ||
}); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
import { promisify } from 'node:util'; | ||
import zlib, { constants } from 'node:zlib'; | ||
import Sqlite from 'better-sqlite3'; | ||
import type { Database, Statement } from 'better-sqlite3'; | ||
import { exists } from 'fs-extra'; | ||
import * as upath from 'upath'; | ||
import { logger } from '../../../logger'; | ||
import { ensureDir } from '../../fs'; | ||
|
||
const brotliCompress = promisify(zlib.brotliCompress); | ||
const brotliDecompress = promisify(zlib.brotliDecompress); | ||
|
||
function compress(input: unknown): Promise<Buffer> { | ||
const jsonStr = JSON.stringify(input); | ||
return brotliCompress(jsonStr, { | ||
params: { | ||
[constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, | ||
[constants.BROTLI_PARAM_QUALITY]: 3, | ||
}, | ||
}); | ||
} | ||
|
||
async function decompress<T>(input: Buffer): Promise<T> { | ||
const buf = await brotliDecompress(input); | ||
const jsonStr = buf.toString('utf8'); | ||
return JSON.parse(jsonStr) as T; | ||
} | ||
|
||
export class SqlitePackageCache { | ||
private readonly upsertStatement: Statement<unknown[]>; | ||
private readonly getStatement: Statement<unknown[]>; | ||
private readonly deleteExpiredRows: Statement<unknown[]>; | ||
private readonly countStatement: Statement<unknown[]>; | ||
|
||
static async init(cacheDir: string): Promise<SqlitePackageCache> { | ||
const sqliteDir = upath.join(cacheDir, 'renovate/renovate-cache-sqlite'); | ||
await ensureDir(sqliteDir); | ||
const sqliteFile = upath.join(sqliteDir, 'db.sqlite'); | ||
|
||
if (await exists(sqliteFile)) { | ||
logger.debug(`Using SQLite package cache: ${sqliteFile}`); | ||
} else { | ||
logger.debug(`Creating SQLite package cache: ${sqliteFile}`); | ||
} | ||
|
||
const client = new Sqlite(sqliteFile); | ||
const res = new SqlitePackageCache(client); | ||
return res; | ||
} | ||
|
||
private constructor(private client: Database) { | ||
client.pragma('journal_mode = WAL'); | ||
client.pragma("encoding = 'UTF-8'"); | ||
|
||
client | ||
.prepare( | ||
` | ||
CREATE TABLE IF NOT EXISTS package_cache ( | ||
namespace TEXT NOT NULL, | ||
key TEXT NOT NULL, | ||
expiry INTEGER NOT NULL, | ||
data BLOB NOT NULL, | ||
PRIMARY KEY (namespace, key) | ||
) | ||
`, | ||
) | ||
.run(); | ||
client | ||
.prepare('CREATE INDEX IF NOT EXISTS expiry ON package_cache (expiry)') | ||
.run(); | ||
client | ||
.prepare( | ||
'CREATE INDEX IF NOT EXISTS namespace_key ON package_cache (namespace, key)', | ||
) | ||
.run(); | ||
|
||
this.upsertStatement = client.prepare(` | ||
INSERT INTO package_cache (namespace, key, data, expiry) | ||
VALUES (@namespace, @key, @data, unixepoch() + @ttlSeconds) | ||
ON CONFLICT (namespace, key) DO UPDATE SET | ||
data = @data, | ||
expiry = unixepoch() + @ttlSeconds | ||
`); | ||
|
||
this.getStatement = client | ||
.prepare( | ||
` | ||
SELECT data FROM package_cache | ||
WHERE | ||
namespace = @namespace AND key = @key AND expiry > unixepoch() | ||
`, | ||
) | ||
.pluck(true); | ||
|
||
this.deleteExpiredRows = client.prepare(` | ||
DELETE FROM package_cache | ||
WHERE expiry <= unixepoch() | ||
`); | ||
|
||
this.countStatement = client | ||
.prepare('SELECT COUNT(*) FROM package_cache') | ||
.pluck(true); | ||
} | ||
|
||
async set( | ||
namespace: string, | ||
key: string, | ||
value: unknown, | ||
ttlMinutes = 5, | ||
): Promise<void> { | ||
const data = await compress(value); | ||
const ttlSeconds = ttlMinutes * 60; | ||
this.upsertStatement.run({ namespace, key, data, ttlSeconds }); | ||
return Promise.resolve(); | ||
} | ||
|
||
async get<T = unknown>( | ||
namespace: string, | ||
key: string, | ||
): Promise<T | undefined> { | ||
const data = this.getStatement.get({ namespace, key }) as | ||
| Buffer | ||
| undefined; | ||
|
||
if (!data) { | ||
return undefined; | ||
} | ||
|
||
return await decompress<T>(data); | ||
} | ||
|
||
private cleanupExpired(): void { | ||
const start = Date.now(); | ||
const totalCount = this.countStatement.get() as number; | ||
const { changes: deletedCount } = this.deleteExpiredRows.run(); | ||
const finish = Date.now(); | ||
const durationMs = finish - start; | ||
logger.debug( | ||
`SQLite package cache: deleted ${deletedCount} of ${totalCount} entries in ${durationMs}ms`, | ||
); | ||
} | ||
|
||
cleanup(): Promise<void> { | ||
this.cleanupExpired(); | ||
this.client.close(); | ||
return Promise.resolve(); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.