fix: allow specifying the vcpkg version + fix: fix check for apt package installs + feat: allow parallel apt-get calls

.gitignore

@@ -17,3 +17,5 @@ exe/
 *.log
 *.exe
 .cache/
+
+coverage

biome.jsonc

@@ -7,7 +7,8 @@
             "**/dist/**",
             "dev/cpp_vcpkg_project/**/*",
             "**/.venv/",
-            "**/.*cache/"
+            "**/.*cache/",
+            "**/coverage/"
         ],
         "ignoreUnknown": true
     },

src/llvm/llvm.ts

@@ -23,13 +23,15 @@ export async function setupLLVM(version: string, setupDir: string, arch: string)
 }
 
 async function setupLLVMWithoutActivation_raw(version: string, setupDir: string, arch: string) {
-  // install LLVM and its dependencies in parallel
-  const [installationInfo, _1, _2] = await Promise.all([
+  // install LLVM
+  const [installationInfo, _1] = await Promise.all([
     setupLLVMOnly(version, setupDir, arch),
-    setupLLVMDeps(arch),
     addLLVMLoggingMatcher(),
   ])
 
+  // install LLVM dependencies
+  await setupLLVMDeps(arch)
+
   return installationInfo
 }
 const setupLLVMWithoutActivation = memoize(setupLLVMWithoutActivation_raw, { isPromise: true })

src/llvm/llvm_installer.ts

@@ -4,7 +4,7 @@ import { execa } from "execa"
 import { chmod, readFile, writeFile } from "fs/promises"
 import { DEFAULT_TIMEOUT } from "../installTool"
 import { addPath } from "../utils/env/addEnv"
-import { hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack"
+import { aptTimeout, hasNala, isPackageRegexInstalled, setupAptPack } from "../utils/setup/setupAptPack"
 import type { InstallationInfo } from "../utils/setup/setupBin"
 
 export enum LLVMPackages {
@@ -70,16 +70,16 @@ function nonInteractiveScript(script: string) {
   // make the scirpt non-interactive and fix broken packages
   return script.replace(
     /add-apt-repository "\${REPO_NAME}"/g,
-    // eslint-disable-next-line no-template-curly-in-string
-    "add-apt-repository -y \"${REPO_NAME}\"",
+    `add-apt-repository -y -n "\${REPO_NAME}"
+apt-get update -o ${aptTimeout} -y`,
   )
 }
 
 async function removeConflictingPackages(givenScript: string) {
   // fix conflicts between libclang-rt and libclang
   let script = givenScript.replace(
     /apt-get install -y/g,
-    "apt-get install -o Dpkg::Options::=\"--force-overwrite\" -y --fix-broken",
+    `apt-get install -o Dpkg::Options::="--force-overwrite" -o ${aptTimeout}  -y --fix-broken`,
   )
 
   // check if these are installed and if so, remove them from the script as they conflict

src/utils/setup/setupAptPack.ts

@@ -13,6 +13,9 @@ import type { InstallationInfo } from "./setupBin"
 let didUpdate: boolean = false
 let didInit: boolean = false
 
+// wait up to 300 seconds if the apt-get lock is held
+export const aptTimeout = "DPkg::Lock::Timeout=300"
+
 export type AptPackage = {
   name: string
   version?: string
@@ -44,13 +47,9 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
   // Add the repos if needed
   await addRepositories(apt, packages)
 
-  // Qualify the packages into full package name/version
-  let qualifiedPacks = await Promise.all(packages.map((pack) => getAptArg(pack.name, pack.version)))
-
-  // find the packages that are not installed
-  qualifiedPacks = await Promise.all(qualifiedPacks.filter(async (pack) => !(await isPackageInstalled(pack))))
+  const needToInstall = await filterAndQualifyAptPackages(packages)
 
-  if (qualifiedPacks.length === 0) {
+  if (needToInstall.length === 0) {
     info("All packages are already installed")
     return { binDir: "/usr/bin/" }
   }
@@ -63,13 +62,13 @@ export async function setupAptPack(packages: AptPackage[], update = false): Prom
 
   // Install
   try {
-    execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks])
+    execRootSync(apt, ["install", "--fix-broken", "-y", ...needToInstall])
   } catch (err) {
     if ("stderr" in (err as ExecaError)) {
       const stderr = (err as ExecaError).stderr
       if (retryErrors.some((error) => stderr.includes(error))) {
-        warning(`Failed to install packages ${qualifiedPacks}. Retrying...`)
-        execRootSync(apt, ["install", "--fix-broken", "-y", ...qualifiedPacks])
+        warning(`Failed to install packages ${needToInstall}. Retrying...`)
+        execRootSync(apt, ["install", "--fix-broken", "-y", "-o", aptTimeout, ...needToInstall])
       }
     } else {
       throw err
@@ -86,17 +85,32 @@ export enum AptPackageType {
   None = 3,
 }
 
+/**
+ * Filter out the packages that are already installed and qualify the packages into a full package name/version
+ */
+async function filterAndQualifyAptPackages(packages: AptPackage[]) {
+  return (await Promise.all(packages.map(qualifiedNeededAptPackage)))
+    .filter((pack) => pack !== undefined)
+}
+
+async function qualifiedNeededAptPackage(pack: AptPackage) {
+  // Qualify the packages into full package name/version
+  const qualified = await getAptArg(pack.name, pack.version)
+  // filter out the packages that are already installed
+  return (await isPackageInstalled(qualified)) ? undefined : qualified
+}
+
 async function addRepositories(apt: string, packages: AptPackage[]) {
   const allRepositories = [...new Set(packages.flatMap((pack) => pack.repositories ?? []))]
   if (allRepositories.length !== 0) {
     if (!didInit) {
       await initApt(apt)
       didInit = true
     }
-    await installAddAptRepo()
+    await installAddAptRepo(apt)
     for (const repo of allRepositories) {
       // eslint-disable-next-line no-await-in-loop
-      execRootSync("add-apt-repository", ["-y", repo])
+      execRootSync("add-apt-repository", ["-y", "--no-update", repo])
     }
     updateRepos(apt)
     didUpdate = true
@@ -163,7 +177,7 @@ export function hasNala() {
   return which.sync("nala", { nothrow: true }) !== null
 }
 
-function getApt() {
+export function getApt() {
   let apt: string
   if (hasNala()) {
     apt = "nala"
@@ -174,14 +188,14 @@ function getApt() {
 }
 
 function updateRepos(apt: string) {
-  execRootSync(apt, apt !== "nala" ? ["update", "-y"] : ["update"])
+  execRootSync(apt, apt !== "nala" ? ["update", "-y", "-o", aptTimeout] : ["update", "-o", aptTimeout])
 }
 
-async function installAddAptRepo() {
+async function installAddAptRepo(apt: string) {
   if (await isPackageInstalled("software-properties-common")) {
     return
   }
-  execRootSync("apt-get", ["install", "-y", "--fix-broken", "software-properties-common"])
+  execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, "software-properties-common"])
 }
 
 /** Install gnupg and certificates (usually missing from docker containers) */
@@ -192,10 +206,14 @@ async function initApt(apt: string) {
     didUpdate = true
   }
 
-  const toInstall = ["ca-certificates", "gnupg", "apt-utils"].filter(async (pack) => !(await isPackageInstalled(pack)))
+  const toInstall = await filterAndQualifyAptPackages([
+    { name: "ca-certificates" },
+    { name: "gnupg" },
+    { name: "apt-utils" },
+  ])
 
   if (toInstall.length !== 0) {
-    execRootSync(apt, ["install", "-y", "--fix-broken", ...toInstall])
+    execRootSync(apt, ["install", "-y", "--fix-broken", "-o", aptTimeout, ...toInstall])
   }
 
   const promises: Promise<string | void>[] = [

src/vcpkg/__tests__/vcpkg.test.ts

@@ -1,20 +1,28 @@
-import { setupTmpDir, testBin } from "../../utils/tests/test-helpers"
+import { cleanupTmpDir, setupTmpDir, testBin } from "../../utils/tests/test-helpers"
 import { setupVcpkg } from "../vcpkg"
 
 jest.setTimeout(300000)
-async function testVcpkg(directory: string) {
-  const { binDir } = await setupVcpkg("", directory, "")
-  await testBin("vcpkg", ["--version"], binDir)
-  return binDir
-}
 
 describe("setup-vcpkg", () => {
   let directory: string
-  beforeAll(async () => {
+  beforeEach(async () => {
     directory = await setupTmpDir("vcpkg")
   })
 
   it("should setup vcpkg", async () => {
-    await testVcpkg(directory)
+    console.log(!("true" in ["", "true"]))
+    const { binDir } = await setupVcpkg("", directory, "")
+    await testBin("vcpkg", ["--version"], binDir)
+    return binDir
+  })
+
+  it("should setup vcpkg with specific version", async () => {
+    const { binDir } = await setupVcpkg("e590c2b30c08caf1dd8d612ec602a003f9784b7d", directory, "")
+    await testBin("vcpkg", ["--version"], binDir)
+    return binDir
+  })
+
+  afterEach(async () => {
+    await cleanupTmpDir(directory)
   })
 })

src/vcpkg/vcpkg.ts

@@ -1,5 +1,5 @@
 import { grantUserWriteAccess } from "admina"
-import { notice } from "ci-log"
+import { info, notice } from "ci-log"
 import { execaSync } from "execa"
 import { pathExists } from "path-exists"
 import { addShExt, addShRelativePrefix, dirname, join } from "patha"
@@ -16,7 +16,7 @@ import { setupPacmanPack } from "../utils/setup/setupPacmanPack"
 let hasVCPKG = false
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export async function setupVcpkg(_version: string, setupDir: string, _arch: string): Promise<InstallationInfo> {
+export async function setupVcpkg(version: string, setupDir: string, _arch: string): Promise<InstallationInfo> {
   if (!hasVCPKG || which.sync("vcpkg", { nothrow: true }) === null) {
     if (process.platform === "linux") {
       // vcpkg download and extraction dependencies
@@ -50,12 +50,23 @@ export async function setupVcpkg(_version: string, setupDir: string, _arch: stri
       }
     }
 
+    // clone if not already exists
     if (!(await pathExists(join(setupDir, addShExt("bootstrap-vcpkg", ".bat"))))) {
       execaSync("git", ["clone", "https://github.com/microsoft/vcpkg"], { cwd: dirname(setupDir), stdio: "inherit" })
     } else {
-      notice(`Vcpkg folder already exists at ${setupDir}. This might mean that ~/vcpkg is restored from the cache.`)
+      notice(`Vcpkg folder already exists at ${setupDir}. Skipping the clone`)
     }
 
+    // if version specified, checkout the version
+    if (version !== "" && version !== "true") {
+      info(`Checking out vcpkg version ${version}`)
+      execaSync("git", ["checkout", version], {
+        cwd: setupDir,
+        stdio: "inherit",
+      })
+    }
+
+    // bootstrap vcpkg
     execaSync(addShExt(addShRelativePrefix("bootstrap-vcpkg"), ".bat"), {
       cwd: setupDir,
       shell: true,