feat(datasource): add debian datasource (#30071)

Co-authored-by: Sebastian Poxhofer <[email protected]>
Co-authored-by: Rhys Arkins <[email protected]>
Co-authored-by: Michael Kriese <[email protected]>
Co-authored-by: Sergei Zharinov <[email protected]>
Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>
Co-authored-by: Aleksandr Mezin <[email protected]>
Co-authored-by: Jasmin Müller <[email protected]>

lib/modules/datasource/api.ts

@@ -15,6 +15,7 @@ import { CrateDatasource } from './crate';
 import { CustomDatasource } from './custom';
 import { DartDatasource } from './dart';
 import { DartVersionDatasource } from './dart-version';
+import { DebDatasource } from './deb';
 import { DenoDatasource } from './deno';
 import { DockerDatasource } from './docker';
 import { DotnetVersionDatasource } from './dotnet-version';
@@ -84,6 +85,7 @@ api.set(CrateDatasource.id, new CrateDatasource());
 api.set(CustomDatasource.id, new CustomDatasource());
 api.set(DartDatasource.id, new DartDatasource());
 api.set(DartVersionDatasource.id, new DartVersionDatasource());
+api.set(DebDatasource.id, new DebDatasource());
 api.set(DenoDatasource.id, new DenoDatasource());
 api.set(DockerDatasource.id, new DockerDatasource());
 api.set(DotnetVersionDatasource.id, new DotnetVersionDatasource());

lib/modules/datasource/deb/__fixtures__/Packages

@@ -0,0 +1,73 @@
+Package: album
+Version: 4.15-1
+Installed-Size: 288
+Maintainer: Salvo 'LtWorf' Tomaselli <[email protected]>
+Architecture: all
+Depends: perl:any, imagemagick
+Recommends: album-data
+Suggests: httpd, jhead, libav-tools
+Description: HTML photo album generator with theme support
+Homepage: http://marginalhacks.com/Hacks/album
+Description-md5: 3eaaefa453087570fb45ac51eeccbe7c
+Tag: implemented-in::perl, interface::commandline, interface::web,
+ role::program, scope::application, use::browsing, use::organizing,
+ web::application, works-with-format::html, works-with::image,
+ works-with::image:raster, works-with::text
+Section: non-free/web
+Priority: optional
+Filename: pool/non-free/a/album/album_4.15-1_all.deb
+Size: 89962
+MD5sum: 7ed9561371bc198dab9f88741b3386ae
+SHA256: 336d96db2998e9a80d00fef36ad38756463cfbd516d1f89b97ecdff22f2b6ec1
+
+Package: album-data
+Version: 4.05-7.2
+Installed-Size: 7936
+Maintainer: Salvo 'LtWorf' Tomaselli <[email protected]>
+Architecture: all
+Depends: album
+Description: themes, plugins and translations for album
+Homepage: http://marginalhacks.com/Hacks/album
+Description-md5: 34adea76df6b2c02712e3838461edbb2
+Tag: role::app-data
+Section: non-free/web
+Priority: optional
+Filename: pool/non-free/a/album-data/album-data_4.05-7.2_all.deb
+Size: 5469888
+MD5sum: bfecda545171260a755ab5b4acd97aa6
+SHA256: 8e16c340d46e53752d5916c7f8aca665aa81a78614bcae0ad7b9decf555c114e
+
+Package: album-data
+Version: 4.05-7.3
+Installed-Size: 7936
+Maintainer: Salvo 'LtWorf' Tomaselli <[email protected]>
+Architecture: all
+Depends: album
+Description: themes, plugins and translations for album
+Homepage: http://marginalhacks.com/Hacks/album
+Description-md5: 34adea76df6b2c02712e3838461edbb2
+Tag: role::app-data
+Section: non-free/web
+Priority: optional
+Filename: pool/non-free/a/album-data/album-data_4.05-7.2_all.deb
+Size: 5469888
+MD5sum: bfecda545171260a755ab5b4acd97aa6
+SHA256: 8e16c340d46e53752d5916c7f8aca665aa81a78614bcae0ad7b9decf555c114e
+
+Package: alien-arena-data
+Version: 7.71.3+ds-1
+Installed-Size: 1918877
+Maintainer: Debian Games Team <[email protected]>
+Architecture: all
+Depends: fonts-freefont-ttf, fonts-aenigma
+Enhances: alien-arena (>= 7.71.3+ds), alien-arena-server (>= 7.71.3+ds)
+Description: Game data files for Alien Arena
+Homepage: https://martianbackup.com
+Description-md5: f930829d2a1207940bee317dc2015735
+Tag: game::fps, role::app-data, use::gameplaying
+Section: non-free/games
+Priority: optional
+Filename: pool/non-free/a/alien-arena-data/alien-arena-data_7.71.3+ds-1_all.deb
+Size: 766655844
+MD5sum: 447e0d3a42973dbf3d87df28359501d2
+SHA256: a3fdaf2b0b9e969149642300f0781f7cfdd3ed33f49be38afe58e89531a21b70

lib/modules/datasource/deb/checksum.spec.ts

@@ -0,0 +1,57 @@
+import type { DirectoryResult } from 'tmp-promise';
+import { dir } from 'tmp-promise';
+import { Fixtures } from '../../../../test/fixtures';
+import { GlobalConfig } from '../../../config/global';
+import { outputCacheFile } from '../../../util/fs';
+import { computeFileChecksum, parseChecksumsFromInRelease } from './checksum';
+
+const fixtureInRelease = Fixtures.getBinary(`InRelease`).toString();
+
+describe('modules/datasource/deb/checksum', () => {
+  let cacheDir: DirectoryResult | null;
+
+  beforeEach(async () => {
+    const cacheDir = await dir({ unsafeCleanup: true });
+    GlobalConfig.set({ cacheDir: cacheDir.path });
+  });
+
+  afterEach(async () => {
+    await cacheDir?.cleanup();
+    cacheDir = null;
+  });
+
+  describe('parseChecksumsFromInRelease', () => {
+    it('parses the checksum for the specified package', () => {
+      const expectedHash =
+        'bf77b15e68c5bfd7267c76a34172021de8f10f861f41ebda7b39d1390dd4bf9a';
+      expect(
+        parseChecksumsFromInRelease(
+          fixtureInRelease,
+          'contrib/binary-amd64/Packages.gz',
+        ),
+      ).toBe(expectedHash);
+
+      expect(
+        parseChecksumsFromInRelease(
+          fixtureInRelease,
+          'non-existing/binary-amd64/Packages.gz',
+        ),
+      ).toBeNull();
+    });
+  });
+
+  describe('computeFileChecksum', () => {
+    it('computes the checksum of a file', async () => {
+      await outputCacheFile('file.txt', 'bar');
+
+      const expectedHash =
+        'fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9';
+
+      expect(await computeFileChecksum('file.txt')).toBe(expectedHash);
+    });
+
+    it('should fail if there is an error in the stream', async () => {
+      await expect(computeFileChecksum('file.txt')).rejects.toThrow();
+    });
+  });
+});

lib/modules/datasource/deb/checksum.ts

@@ -0,0 +1,40 @@
+import { createCacheReadStream } from '../../../util/fs';
+import { hashStream } from '../../../util/hash';
+import { escapeRegExp, newlineRegex, regEx } from '../../../util/regex';
+
+/**
+ * Parses the SHA256 checksum for a specified package path from the InRelease content.
+ *
+ * @param inReleaseContent - content of the InRelease file
+ * @param packagePath - path of the package file (e.g., 'contrib/binary-amd64/Packages.gz')
+ * @returns The SHA256 checksum if found, otherwise undefined
+ */
+export function parseChecksumsFromInRelease(
+  inReleaseContent: string,
+  packagePath: string,
+): string | null {
+  const lines = inReleaseContent.split(newlineRegex);
+  const regex = regEx(
+    `([a-f0-9]{64})\\s+\\d+\\s+${escapeRegExp(packagePath)}$`,
+  );
+
+  for (const line of lines) {
+    const match = regex.exec(line);
+    if (match) {
+      return match[1];
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Computes the SHA256 checksum of a specified file.
+ *
+ * @param filePath - path of the file
+ * @returns resolves to the SHA256 checksum
+ */
+export function computeFileChecksum(filePath: string): Promise<string> {
+  const stream = createCacheReadStream(filePath);
+  return hashStream(stream, 'sha256');
+}

lib/modules/datasource/deb/common.ts

@@ -0,0 +1,17 @@
+import type { PackageDescription } from './types';
+
+/**
+ * This specifies the directory where the extracted and downloaded packages files are stored relative to cacheDir.
+ * The folder will be created automatically if it doesn't exist.
+ */
+export const cacheSubDir: string = 'deb';
+
+export const requiredPackageKeys: Array<keyof PackageDescription> = [
+  'Package',
+  'Version',
+];
+
+export const packageKeys: Array<keyof PackageDescription> = [
+  ...requiredPackageKeys,
+  'Homepage',
+];

lib/modules/datasource/deb/file.spec.ts

@@ -0,0 +1,35 @@
+import type { DirectoryResult } from 'tmp-promise';
+import { dir } from 'tmp-promise';
+import upath from 'upath';
+import { Fixtures } from '../../../../test/fixtures';
+import { fs } from '../../../../test/util';
+import { GlobalConfig } from '../../../config/global';
+import { extract } from './file';
+
+const fixturePackagesArchivePath = Fixtures.getPath(`Packages.gz`);
+
+describe('modules/datasource/deb/file', () => {
+  let cacheDir: DirectoryResult | null;
+  let extractedPackageFile: string;
+
+  beforeEach(async () => {
+    cacheDir = await dir({ unsafeCleanup: true });
+    GlobalConfig.set({ cacheDir: cacheDir.path });
+
+    const extractionFolder = await fs.ensureCacheDir('file');
+    extractedPackageFile = upath.join(extractionFolder, `package.txt`);
+  });
+
+  afterEach(async () => {
+    await cacheDir?.cleanup();
+    cacheDir = null;
+  });
+
+  describe('extract', () => {
+    it('should throw error for unsupported compression', async () => {
+      await expect(
+        extract(fixturePackagesArchivePath, 'xz', extractedPackageFile),
+      ).rejects.toThrow('Unsupported compression standard');
+    });
+  });
+});

lib/modules/datasource/deb/file.ts

@@ -0,0 +1,37 @@
+import { createUnzip } from 'zlib';
+import * as fs from '../../../util/fs';
+
+/**
+ * Extracts the specified compressed file to the output file.
+ *
+ * @param compressedFile - The path to the compressed file.
+ * @param compression - The compression method used (currently only 'gz' is supported).
+ * @param outputFile - The path where the extracted content will be stored.
+ * @throws Will throw an error if the compression method is unknown.
+ */
+export async function extract(
+  compressedFile: string,
+  compression: string,
+  outputFile: string,
+): Promise<void> {
+  if (compression === 'gz') {
+    const source = fs.createCacheReadStream(compressedFile);
+    const destination = fs.createCacheWriteStream(outputFile);
+    await fs.pipeline(source, createUnzip(), destination);
+  } else {
+    throw new Error(`Unsupported compression standard '${compression}'`);
+  }
+}
+
+/**
+ * Checks if the file exists and retrieves its creation time.
+ *
+ * @param filePath - The path to the file.
+ * @returns The creation time if the file exists, otherwise undefined.
+ */
+export async function getFileCreationTime(
+  filePath: string,
+): Promise<Date | undefined> {
+  const stats = await fs.statCacheFile(filePath);
+  return stats?.ctime;
+}