feat: SQLite-based package cache (#26608)

Co-authored-by: Michael Kriese <[email protected]>

docs/usage/self-hosted-experimental.md

@@ -167,3 +167,8 @@ If set, Renovate will enable `forcePathStyle` when instantiating the AWS S3 clie
 > Whether to force path style URLs for S3 objects (e.g., `https://s3.amazonaws.com//` instead of `https://.s3.amazonaws.com/`)
 
 Source: [AWS S3 documentation - Interface BucketEndpointInputConfig](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-s3/interfaces/bucketendpointinputconfig.html)
+
+## `RENOVATE_X_SQLITE_PACKAGE_CACHE`
+
+If set, Renovate will use SQLite as the backend for the package cache.
+Don't combine with `redisUrl`, Redis would be preferred over SQlite.

lib/util/cache/package/index.spec.ts

@@ -2,8 +2,13 @@ import { cleanup, get, init, set } from '.';
 
 jest.mock('./file');
 jest.mock('./redis');
+jest.mock('./sqlite');
 
 describe('util/cache/package/index', () => {
+  beforeEach(() => {
+    delete process.env.RENOVATE_X_SQLITE_PACKAGE_CACHE;
+  });
+
   it('returns undefined if not initialized', async () => {
     expect(await get('test', 'missing-key')).toBeUndefined();
     expect(await set('test', 'some-key', 'some-value', 5)).toBeUndefined();
@@ -28,4 +33,14 @@ describe('util/cache/package/index', () => {
     expect(await get('some-namespace', 'unknown-key')).toBeUndefined();
     expect(await cleanup({ redisUrl: 'some-url' })).toBeUndefined();
   });
+
+  it('sets and gets sqlite', async () => {
+    process.env.RENOVATE_X_SQLITE_PACKAGE_CACHE = 'true';
+    await init({ cacheDir: 'some-dir' });
+    expect(
+      await set('some-namespace', 'some-key', 'some-value', 1),
+    ).toBeUndefined();
+    expect(await get('some-namespace', 'unknown-key')).toBeUndefined();
+    expect(await cleanup({ redisUrl: 'some-url' })).toBeUndefined();
+  });
 });

lib/util/cache/package/index.ts

@@ -2,6 +2,7 @@ import type { AllConfig } from '../../../config/types';
 import * as memCache from '../memory';
 import * as fileCache from './file';
 import * as redisCache from './redis';
+import { SqlitePackageCache } from './sqlite';
 import type { PackageCache } from './types';
 
 let cacheProxy: PackageCache | undefined;
@@ -60,13 +61,22 @@ export async function init(config: AllConfig): Promise<void> {
       get: redisCache.get,
       set: redisCache.set,
     };
-  } else if (config.cacheDir) {
+    return;
+  }
+
+  if (process.env.RENOVATE_X_SQLITE_PACKAGE_CACHE) {
+    cacheProxy = await SqlitePackageCache.init(config.cacheDir!);
+    return;
+  }
+
+  if (config.cacheDir) {
     fileCache.init(config.cacheDir);
     cacheProxy = {
       get: fileCache.get,
       set: fileCache.set,
       cleanup: fileCache.cleanup,
     };
+    return;
   }
 }
 

lib/util/cache/package/sqlite.spec.ts

@@ -0,0 +1,55 @@
+import { withDir } from 'tmp-promise';
+import { GlobalConfig } from '../../../config/global';
+import { SqlitePackageCache } from './sqlite';
+
+function withSqlite<T>(
+  fn: (sqlite: SqlitePackageCache) => Promise<T>,
+): Promise<T> {
+  return withDir(
+    async ({ path }) => {
+      GlobalConfig.set({ cacheDir: path });
+      const sqlite = await SqlitePackageCache.init(path);
+      const res = await fn(sqlite);
+      await sqlite.cleanup();
+      return res;
+    },
+    { unsafeCleanup: true },
+  );
+}
+
+describe('util/cache/package/sqlite', () => {
+  it('should get undefined', async () => {
+    const res = await withSqlite((sqlite) => sqlite.get('foo', 'bar'));
+    expect(res).toBeUndefined();
+  });
+
+  it('should set and get', async () => {
+    const res = await withSqlite(async (sqlite) => {
+      await sqlite.set('foo', 'bar', { foo: 'foo' });
+      await sqlite.set('foo', 'bar', { bar: 'bar' });
+      await sqlite.set('foo', 'bar', { baz: 'baz' });
+      return sqlite.get('foo', 'bar');
+    });
+    expect(res).toEqual({ baz: 'baz' });
+  });
+
+  it('reopens', async () => {
+    const res = await withDir(
+      async ({ path }) => {
+        GlobalConfig.set({ cacheDir: path });
+
+        const client1 = await SqlitePackageCache.init(path);
+        await client1.set('foo', 'bar', 'baz');
+        await client1.cleanup();
+
+        const client2 = await SqlitePackageCache.init(path);
+        const res = await client2.get('foo', 'bar');
+        await client2.cleanup();
+        return res;
+      },
+      { unsafeCleanup: true },
+    );
+
+    expect(res).toBe('baz');
+  });
+});

lib/util/cache/package/sqlite.ts

@@ -0,0 +1,148 @@
+import { promisify } from 'node:util';
+import zlib, { constants } from 'node:zlib';
+import Sqlite from 'better-sqlite3';
+import type { Database, Statement } from 'better-sqlite3';
+import { exists } from 'fs-extra';
+import * as upath from 'upath';
+import { logger } from '../../../logger';
+import { ensureDir } from '../../fs';
+
+const brotliCompress = promisify(zlib.brotliCompress);
+const brotliDecompress = promisify(zlib.brotliDecompress);
+
+function compress(input: unknown): Promise<Buffer> {
+  const jsonStr = JSON.stringify(input);
+  return brotliCompress(jsonStr, {
+    params: {
+      [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
+      [constants.BROTLI_PARAM_QUALITY]: 3,
+    },
+  });
+}
+
+async function decompress<T>(input: Buffer): Promise<T> {
+  const buf = await brotliDecompress(input);
+  const jsonStr = buf.toString('utf8');
+  return JSON.parse(jsonStr) as T;
+}
+
+export class SqlitePackageCache {
+  private readonly upsertStatement: Statement<unknown[]>;
+  private readonly getStatement: Statement<unknown[]>;
+  private readonly deleteExpiredRows: Statement<unknown[]>;
+  private readonly countStatement: Statement<unknown[]>;
+
+  static async init(cacheDir: string): Promise<SqlitePackageCache> {
+    const sqliteDir = upath.join(cacheDir, 'renovate/renovate-cache-sqlite');
+    await ensureDir(sqliteDir);
+    const sqliteFile = upath.join(sqliteDir, 'db.sqlite');
+
+    if (await exists(sqliteFile)) {
+      logger.debug(`Using SQLite package cache: ${sqliteFile}`);
+    } else {
+      logger.debug(`Creating SQLite package cache: ${sqliteFile}`);
+    }
+
+    const client = new Sqlite(sqliteFile);
+    const res = new SqlitePackageCache(client);
+    return res;
+  }
+
+  private constructor(private client: Database) {
+    client.pragma('journal_mode = WAL');
+    client.pragma("encoding = 'UTF-8'");
+
+    client
+      .prepare(
+        `
+          CREATE TABLE IF NOT EXISTS package_cache (
+            namespace TEXT NOT NULL,
+            key TEXT NOT NULL,
+            expiry INTEGER NOT NULL,
+            data BLOB NOT NULL,
+            PRIMARY KEY (namespace, key)
+          )
+        `,
+      )
+      .run();
+    client
+      .prepare('CREATE INDEX IF NOT EXISTS expiry ON package_cache (expiry)')
+      .run();
+    client
+      .prepare(
+        'CREATE INDEX IF NOT EXISTS namespace_key ON package_cache (namespace, key)',
+      )
+      .run();
+
+    this.upsertStatement = client.prepare(`
+      INSERT INTO package_cache (namespace, key, data, expiry)
+      VALUES (@namespace, @key, @data, unixepoch() + @ttlSeconds)
+      ON CONFLICT (namespace, key) DO UPDATE SET
+        data = @data,
+        expiry = unixepoch() + @ttlSeconds
+    `);
+
+    this.getStatement = client
+      .prepare(
+        `
+          SELECT data FROM package_cache
+          WHERE
+            namespace = @namespace AND key = @key AND expiry > unixepoch()
+        `,
+      )
+      .pluck(true);
+
+    this.deleteExpiredRows = client.prepare(`
+      DELETE FROM package_cache
+      WHERE expiry <= unixepoch()
+    `);
+
+    this.countStatement = client
+      .prepare('SELECT COUNT(*) FROM package_cache')
+      .pluck(true);
+  }
+
+  async set(
+    namespace: string,
+    key: string,
+    value: unknown,
+    ttlMinutes = 5,
+  ): Promise<void> {
+    const data = await compress(value);
+    const ttlSeconds = ttlMinutes * 60;
+    this.upsertStatement.run({ namespace, key, data, ttlSeconds });
+    return Promise.resolve();
+  }
+
+  async get<T = unknown>(
+    namespace: string,
+    key: string,
+  ): Promise<T | undefined> {
+    const data = this.getStatement.get({ namespace, key }) as
+      | Buffer
+      | undefined;
+
+    if (!data) {
+      return undefined;
+    }
+
+    return await decompress<T>(data);
+  }
+
+  private cleanupExpired(): void {
+    const start = Date.now();
+    const totalCount = this.countStatement.get() as number;
+    const { changes: deletedCount } = this.deleteExpiredRows.run();
+    const finish = Date.now();
+    const durationMs = finish - start;
+    logger.debug(
+      `SQLite package cache: deleted ${deletedCount} of ${totalCount} entries in ${durationMs}ms`,
+    );
+  }
+
+  cleanup(): Promise<void> {
+    this.cleanupExpired();
+    this.client.close();
+    return Promise.resolve();
+  }
+}

package.json

@@ -168,6 +168,7 @@
     "@renovatebot/pep440": "3.0.17",
     "@renovatebot/ruby-semver": "3.0.23",
     "@sindresorhus/is": "4.6.0",
+    "@types/better-sqlite3": "7.6.8",
     "@types/ms": "0.7.34",
     "@types/tmp": "0.2.6",
     "@yarnpkg/core": "4.0.2",
@@ -177,6 +178,7 @@
     "auth-header": "1.0.0",
     "aws4": "1.12.0",
     "azure-devops-node-api": "12.3.0",
+    "better-sqlite3": "9.2.2",
     "bunyan": "1.8.15",
     "cacache": "18.0.2",
     "cacheable-lookup": "5.0.4",