Make integration tests more stable in CI

integrations/utils.ts

@@ -1,10 +1,11 @@
 import dedent from 'dedent'
 import fastGlob from 'fast-glob'
-import { exec, spawn } from 'node:child_process'
+import { exec, execFile, spawn, type ChildProcess } from 'node:child_process'
 import fs from 'node:fs/promises'
+import { createServer } from 'node:net'
 import { platform, tmpdir } from 'node:os'
 import path from 'node:path'
-import { stripVTControlCharacters } from 'node:util'
+import { promisify, stripVTControlCharacters } from 'node:util'
 import { RawSourceMap, SourceMapConsumer } from 'source-map-js'
 import { test as defaultTest, type ExpectStatic } from 'vitest'
 import { createLineTable } from '../packages/tailwindcss/src/source-maps/line-table'
@@ -64,12 +65,15 @@ interface TestFlags {
   only?: boolean
   skip?: boolean
   debug?: boolean
+  concurrent?: boolean
 }
 
 type SpawnActor = { predicate: (message: string) => boolean; resolve: () => void }
 
 export const IS_WINDOWS = platform() === 'win32'
 
+const execFileAsync = promisify(execFile)
+
 const TEST_TIMEOUT = IS_WINDOWS ? 120000 : 60000
 const ASSERTION_TIMEOUT = IS_WINDOWS ? 10000 : 5000
 
@@ -82,7 +86,7 @@ export function test(
   name: string,
   config: TestConfig,
   testCallback: TestCallback,
-  { only = false, skip = false, debug = false }: TestFlags = {},
+  { only = false, skip = false, debug = false, concurrent = false }: TestFlags = {},
 ) {
   return defaultTest(
     name,
@@ -91,7 +95,7 @@ export function test(
       retry: process.env.CI ? 2 : 0,
       only: only || (!process.env.CI && debug),
       skip,
-      concurrent: true,
+      concurrent,
     },
     async (options) => {
       let rootDir = debug ? path.join(REPO_ROOT, '.debug') : TMP_ROOT
@@ -170,6 +174,7 @@ export function test(
           if (debug) console.log(`>& ${command}`)
           let child = spawn(command, {
             cwd,
+            detached: !IS_WINDOWS,
             shell: true,
             ...childProcessOptions,
             env: {
@@ -178,19 +183,22 @@ export function test(
             },
           })
 
-          function dispose() {
-            if (!child.kill()) {
-              child.kill('SIGKILL')
-            }
+          let disposed = false
+
+          async function dispose() {
+            if (disposed) return disposePromise
+            disposed = true
+
+            await killProcessTree(child)
 
-            let timer = setTimeout(
-              () =>
-                rejectDisposal?.(new Error(`spawned process (${command}) did not exit in time`)),
-              ASSERTION_TIMEOUT,
+            let timer = setTimeout(() => {
+              forceKillProcessTree(child)
+              rejectDisposal?.(new Error(`spawned process (${command}) did not exit in time`))
+            }, ASSERTION_TIMEOUT)
+            disposePromise.then(
+              () => clearTimeout(timer),
+              () => clearTimeout(timer),
             )
-            disposePromise.finally(() => {
-              clearTimeout(timer)
-            })
             return disposePromise
           }
           disposables.push(dispose)
@@ -428,11 +436,18 @@ export function test(
       let disposables: (() => Promise<void>)[] = []
 
       async function dispose() {
-        await Promise.all(disposables.map((dispose) => dispose()))
+        let results = await Promise.allSettled(disposables.map((dispose) => dispose()))
 
         if (!debug) {
           await gracefullyRemove(root)
         }
+
+        let errors = results.flatMap((result) =>
+          result.status === 'rejected' ? [result.reason] : [],
+        )
+        if (errors.length > 0) {
+          throw new AggregateError(errors, 'Failed to clean up spawned processes')
+        }
       }
 
       options.onTestFinished(dispose)
@@ -461,6 +476,9 @@ test.only = (name: string, config: TestConfig, testCallback: TestCallback) => {
 test.skip = (name: string, config: TestConfig, testCallback: TestCallback) => {
   return test(name, config, testCallback, { skip: true })
 }
+test.concurrent = (name: string, config: TestConfig, testCallback: TestCallback) => {
+  return test(name, config, testCallback, { concurrent: true })
+}
 test.debug = (name: string, config: TestConfig, testCallback: TestCallback) => {
   return test(name, config, testCallback, { debug: true })
 }
@@ -607,6 +625,65 @@ export async function fetchStyles(base: string, path = '/'): Promise<string> {
   }, '')
 }
 
+export async function getRandomPort() {
+  return new Promise<number>((resolve, reject) => {
+    let server = createServer()
+    server.unref()
+    server.on('error', reject)
+    server.listen(0, '127.0.0.1', () => {
+      let address = server.address()
+      server.close(() => {
+        if (address && typeof address === 'object') {
+          resolve(address.port)
+        } else {
+          reject(new Error('Unable to allocate random port'))
+        }
+      })
+    })
+  })
+}
+
+async function killProcessTree(child: ChildProcess) {
+  if (child.exitCode !== null || child.signalCode !== null || child.pid === undefined) {
+    return
+  }
+
+  if (IS_WINDOWS) {
+    await execFileAsync('taskkill', ['/pid', String(child.pid), '/T', '/F'], {
+      timeout: ASSERTION_TIMEOUT,
+      windowsHide: true,
+    }).catch(() => {})
+    return
+  }
+
+  try {
+    process.kill(-child.pid, 'SIGTERM')
+  } catch (error: any) {
+    if (error?.code !== 'ESRCH') {
+      child.kill()
+    }
+  }
+}
+
+function forceKillProcessTree(child: ChildProcess) {
+  if (child.exitCode !== null || child.signalCode !== null || child.pid === undefined) {
+    return
+  }
+
+  if (IS_WINDOWS) {
+    execFile('taskkill', ['/pid', String(child.pid), '/T', '/F'], { windowsHide: true }, () => {})
+    return
+  }
+
+  try {
+    process.kill(-child.pid, 'SIGKILL')
+  } catch (error: any) {
+    if (error?.code !== 'ESRCH') {
+      child.kill('SIGKILL')
+    }
+  }
+}
+
 async function gracefullyRemove(dir: string) {
   // Skip removing the directory in CI because it can stall on Windows
   if (!process.env.CI) {

integrations/vite/nuxt.test.ts

@@ -1,4 +1,14 @@
-import { candidate, css, fetchStyles, html, json, retryAssertion, test, ts } from '../utils'
+import {
+  candidate,
+  css,
+  fetchStyles,
+  getRandomPort,
+  html,
+  json,
+  retryAssertion,
+  test,
+  ts,
+} from '../utils'
 
 const SETUP = {
   fs: {
@@ -84,7 +94,8 @@ test('build', SETUP, async ({ spawn, exec, expect }) => {
   await exec('pnpm nuxt build')
   // The Nuxt preview server does not automatically assign a free port if 3000
   // is taken, so we use a random port instead.
-  let process = await spawn(`pnpm nuxt preview --port 8724`, {
+  let port = await getRandomPort()
+  let process = await spawn(`pnpm nuxt preview --port ${port}`, {
     env: {
       TEST: 'false',
       NODE_ENV: 'development',