ci: add apt timeout

.github/workflows/pr-ci.yml

@@ -714,52 +714,43 @@ jobs:
           image: kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a
           skipClusterLogsExport: true
 
-      - name: cache apt packages (podman/runc)
+      - name: cache Homebrew downloads (podman)
         if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux'
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
-          path: ${{ runner.temp }}/apt-archives
-          key: apt-podman-runc-${{ runner.os }}-v1
+          path: ${{ runner.temp }}/homebrew-cache
+          key: brew-podman-${{ runner.os }}-${{ runner.arch }}-v1
 
-      - name: cache apt packages (podman/runc)
+      - name: install podman (Homebrew)
         if: (matrix.install-podman == 'rootless' || matrix.install-podman == 'rootful') && runner.os == 'Linux'
+        timeout-minutes: 10
+        env:
+          HOMEBREW_CACHE: ${{ runner.temp }}/homebrew-cache
+          HOMEBREW_NO_AUTO_UPDATE: "1"
+          HOMEBREW_NO_INSTALL_CLEANUP: "1"
         run: |
-          for i in $(seq 1 60); do
-            if sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; then
-              echo "waiting for dpkg lock to be released ($i/60)"
-              sleep 2
-            else
-              break
-            fi
-          done
-          mkdir -p "${{ runner.temp }}/apt-archives"
-          sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" update
-          sudo apt-get -o Dir::Cache::Archives="${{ runner.temp }}/apt-archives" install -y podman runc
-
-          sudo rm -f "${{ runner.temp }}/apt-archives/lock"
-          sudo rm -rf "${{ runner.temp }}/apt-archives/partial"
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH"
+          brew install podman
 
-          sudo mkdir -p /etc/containers
-          printf '[engine]\nruntime = "runc"\n' | sudo tee /etc/containers/containers.conf
+          echo "Defaults secure_path=\"$HOMEBREW_PREFIX/bin:$HOMEBREW_PREFIX/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"" \
+            | sudo tee /etc/sudoers.d/homebrew-path >/dev/null
+          sudo visudo -cf /etc/sudoers.d/homebrew-path
 
       - name: Install Podman (Linux rootless)
         if: matrix.install-podman == 'rootless' && runner.os == 'Linux'
-        env:
-          APT_CACHE_DIR: ${{ runner.temp }}/apt-archives
         run: |
           podman info
           podman run --rm busybox@sha256:fd8d9aa63ba2f0982b5304e1ee8d3b90a210bc1ffb5314d980eb6962f1a9715d echo "podman runtime preflight OK"
 
       - name: Install Podman (Linux rootful)
         if: matrix.install-podman == 'rootful' && runner.os == 'Linux'
-        env:
-          APT_CACHE_DIR: ${{ runner.temp }}/apt-archives
         run: |
-          sudo systemctl enable --now podman.socket
+          sudo mkdir -p /run/podman
+          sudo -b sh -c 'podman system service --time=0 unix:///run/podman/podman.sock > /tmp/podman-service.log 2>&1'
           if ! timeout 30 bash -c 'until sudo podman info >/dev/null 2>&1; do sleep 1; done'; then
-            echo "::error::podman.socket did not become ready within 30s"
-            sudo systemctl status podman.socket --no-pager || true
-            sudo journalctl -u podman.socket --no-pager -n 100 || true
+            echo "::error::podman service did not become ready within 30s"
+            cat /tmp/podman-service.log || true
             exit 1
           fi
           echo "DOCKER_HOST=unix:///run/podman/podman.sock" >> "$GITHUB_ENV"

cmd/internal/agentworkspace/up.go

@@ -331,7 +331,7 @@ func (w *workspaceInitializer) initialize(ctx context.Context) error {
 		log.Warnf("failed to set up docker/git credentials (continuing without them): %v", err)
 	}
 
-	dockerErrChan := w.installDockerAsync()
+	dockerErrChan := w.installDockerAsync(ctx)
 
 	if err := w.prepareWorkspaceContent(ctx); err != nil {
 		return err
@@ -400,7 +400,7 @@ type dockerInstallResult struct {
 	err  error
 }
 
-func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult {
+func (w *workspaceInitializer) installDockerAsync(ctx context.Context) <-chan dockerInstallResult {
 	resultChan := make(chan dockerInstallResult, 1)
 
 	go func() {
@@ -410,14 +410,14 @@ func (w *workspaceInitializer) installDockerAsync() <-chan dockerInstallResult {
 			return
 		}
 
-		dockerPath, err := w.ensureDockerInstalled()
+		dockerPath, err := w.ensureDockerInstalled(ctx)
 		resultChan <- dockerInstallResult{path: dockerPath, err: err}
 	}()
 
 	return resultChan
 }
 
-func (w *workspaceInitializer) ensureDockerInstalled() (string, error) {
+func (w *workspaceInitializer) ensureDockerInstalled(ctx context.Context) (string, error) {
 	dockerCmd := w.getDockerCommand()
 
 	if command.Exists(dockerCmd) {
@@ -444,7 +444,7 @@ func (w *workspaceInitializer) ensureDockerInstalled() (string, error) {
 	}
 
 	log.Debug("attempting to install docker")
-	dockerPath, err := installDocker()
+	dockerPath, err := installDocker(ctx)
 	log.Debugf("docker installation path=%q, err=%v", dockerPath, err)
 	return dockerPath, err
 }
@@ -784,11 +784,11 @@ func prepareImage(workspaceDir, image string) error {
 
 // installDocker installs Docker and returns the path to the docker binary.
 // This function assumes docker does not already exist - the caller should check first.
-func installDocker() (dockerPath string, err error) {
+func installDocker(ctx context.Context) (dockerPath string, err error) {
 	writer := log.Writer(log.LevelInfo)
 	defer func() { _ = writer.Close() }()
 	log.Debug("installing Docker")
-	return dockerinstall.Install(writer, writer)
+	return dockerinstall.Install(ctx, writer, writer)
 }
 
 func configureDockerDaemon(ctx context.Context) error {

hack/automations/main_test.go

@@ -1,21 +1,82 @@
 package main
 
 import (
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
 )
 
-func runGenerator(t *testing.T, args ...string) string {
+// newSandbox copies the generator's real inputs (agents.yaml and the existing
+// .agents/agents tree) into an isolated temp directory and returns its path.
+// The generator writes agent.md files relative to its working directory, and
+// goreleaser's build hooks run this test suite once per target arch in
+// parallel; without isolation, concurrent runs race on the same real
+// .agents/agents/*/agent.md files in the checkout.
+func newSandbox(t *testing.T) string {
+	t.Helper()
+	root := repoRoot(t)
+	dir := t.TempDir()
+
+	copyFile(t,
+		filepath.Join(root, "hack", "automations", "agents.yaml"),
+		filepath.Join(dir, "hack", "automations", "agents.yaml"))
+	copyTree(t,
+		filepath.Join(root, ".agents", "agents"),
+		filepath.Join(dir, ".agents", "agents"))
+
+	return dir
+}
+
+func copyFile(t *testing.T, src, dst string) {
+	t.Helper()
+	b, err := os.ReadFile(src)
+	if err != nil {
+		t.Fatalf("read %s: %v", src, err)
+	}
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err)
+	}
+	if err := os.WriteFile(dst, b, 0o644); err != nil {
+		t.Fatalf("write %s: %v", dst, err)
+	}
+}
+
+func copyTree(t *testing.T, src, dst string) {
+	t.Helper()
+	err := filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(dst, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o755)
+		}
+		b, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		return os.WriteFile(target, b, 0o644)
+	})
+	if err != nil {
+		t.Fatalf("copy tree %s: %v", src, err)
+	}
+}
+
+func runGenerator(t *testing.T, dir string, args ...string) string {
 	t.Helper()
 	bin := filepath.Join(t.TempDir(), "automations")
 	if out, err := exec.Command("go", "build", "-o", bin, ".").CombinedOutput(); err != nil {
 		t.Fatalf("go build: %v\n%s", err, out)
 	}
 	cmd := exec.Command(bin, args...)
-	cmd.Dir = repoRoot(t)
+	cmd.Dir = dir
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		t.Logf("generator output:\n%s", out)
@@ -32,13 +93,13 @@ func repoRoot(t *testing.T) string {
 	return filepath.Dir(filepath.Dir(wd))
 }
 
-func agentFilePath(t *testing.T, id string) string {
-	t.Helper()
-	return filepath.Join(repoRoot(t), ".agents", "agents", id, "agent.md")
+func agentFilePath(dir, id string) string {
+	return filepath.Join(dir, ".agents", "agents", id, "agent.md")
 }
 
 func TestGenerateWritesAllAgents(t *testing.T) {
-	out := runGenerator(t)
+	dir := newSandbox(t)
+	out := runGenerator(t, dir)
 	if !strings.Contains(out, "wrote") {
 		t.Fatalf("expected write output, got:\n%s", out)
 	}
@@ -51,7 +112,7 @@ func TestGenerateWritesAllAgents(t *testing.T) {
 
 	pythonAgents := map[string]bool{"agent-analytics": true}
 	for _, id := range ids {
-		p := agentFilePath(t, id)
+		p := agentFilePath(dir, id)
 		b, err := os.ReadFile(p)
 		if err != nil {
 			t.Errorf("read %s: %v", p, err)
@@ -87,16 +148,18 @@ func TestGenerateWritesAllAgents(t *testing.T) {
 }
 
 func TestCheckIsIdempotent(t *testing.T) {
-	runGenerator(t)
-	out := runGenerator(t, "-check")
+	dir := newSandbox(t)
+	runGenerator(t, dir)
+	out := runGenerator(t, dir, "-check")
 	if strings.Contains(out, "DRIFT") {
 		t.Fatalf("freshly generated files report drift:\n%s", out)
 	}
 }
 
 func TestToolchainPins(t *testing.T) {
-	runGenerator(t)
-	goAgent := agentFilePath(t, "pkg-container")
+	dir := newSandbox(t)
+	runGenerator(t, dir)
+	goAgent := agentFilePath(dir, "pkg-container")
 	b, err := os.ReadFile(goAgent)
 	if err != nil {
 		t.Fatal(err)
@@ -107,7 +170,7 @@ func TestToolchainPins(t *testing.T) {
 			t.Errorf("pkg-container: missing pinned %q", want)
 		}
 	}
-	ciAgent := agentFilePath(t, "ci-optimizer")
+	ciAgent := agentFilePath(dir, "ci-optimizer")
 	cb, err := os.ReadFile(ciAgent)
 	if err != nil {
 		t.Fatal(err)
@@ -118,9 +181,10 @@ func TestToolchainPins(t *testing.T) {
 }
 
 func TestToolchainDownloadsAreBounded(t *testing.T) {
-	runGenerator(t)
+	dir := newSandbox(t)
+	runGenerator(t, dir)
 	for _, id := range []string{"ui-polish", "pkg-container", "agent-analytics"} {
-		p := agentFilePath(t, id)
+		p := agentFilePath(dir, id)
 		b, err := os.ReadFile(p)
 		if err != nil {
 			continue

pkg/dockerinstall/constants.go

@@ -13,6 +13,10 @@ const (
 	DeprecationDelay    = 10 * time.Second
 	RetryDelay          = 10 * time.Second
 
+	// Apt options.
+	AptTimeoutSeconds = 30
+	AptRetries        = 3
+
 	// Paths.
 	DefaultDownloadURL = "https://download.docker.com"
 	DefaultRepoFile    = "docker-ce.repo"

pkg/dockerinstall/debian.go

@@ -1,11 +1,19 @@
 package dockerinstall
 
 import (
+	"context"
 	"fmt"
 	"os/exec"
 	"strings"
 )
 
+// aptFlags configures apt-get to fail fast on transient network issues
+// instead of hanging indefinitely, retrying a bounded number of times first.
+var aptFlags = fmt.Sprintf(
+	`-o Acquire::http::Timeout="%d" -o Acquire::https::Timeout="%d" -o Acquire::Retries="%d"`,
+	AptTimeoutSeconds, AptTimeoutSeconds, AptRetries,
+)
+
 type DebianInstaller struct {
 	distro   *Distro
 	opts     *InstallOptions
@@ -20,8 +28,8 @@ func NewDebianInstaller(distro *Distro, opts *InstallOptions) *DebianInstaller {
 	}
 }
 
-func (i *DebianInstaller) Install(shC string) error {
-	if err := i.setupRepo(shC); err != nil {
+func (i *DebianInstaller) Install(ctx context.Context, shC string) error {
+	if err := i.setupRepo(ctx, shC); err != nil {
 		return err
 	}
 
@@ -32,17 +40,18 @@ func (i *DebianInstaller) Install(shC string) error {
 	pkgs := BuildPackageList(i.opts.version, pkgVersion, cliPkgVersion)
 
 	installCmd := fmt.Sprintf(
-		"DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends %s >/dev/null",
+		"DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq --no-install-recommends %s >/dev/null",
+		aptFlags,
 		pkgs,
 	)
-	if err := i.executor.RunWithRetry(shC, installCmd, DefaultTimeout); err != nil {
+	if err := i.executor.RunWithRetry(ctx, shC, installCmd, DefaultTimeout); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (i *DebianInstaller) setupRepo(shC string) error {
+func (i *DebianInstaller) setupRepo(ctx context.Context, shC string) error {
 	preReqs := "apt-transport-https ca-certificates curl"
 	if !commandExists("gpg") {
 		preReqs += " gnupg"
@@ -64,20 +73,31 @@ func (i *DebianInstaller) setupRepo(shC string) error {
 		i.opts.channel,
 	)
 
+	cmdAptUpdate := fmt.Sprintf("apt-get %s update -qq >/dev/null", aptFlags)
+
+	cmdAptInstall := fmt.Sprintf(
+		"DEBIAN_FRONTEND=noninteractive apt-get %s install -y -qq %s >/dev/null",
+		aptFlags, preReqs,
+	)
+
+	cmdKeyringDir := "mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings"
+
+	cmdDockerGPG := fmt.Sprintf(
+		"curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg",
+		i.opts.downloadURL, i.distro.ID,
+	)
+
 	cmds := []string{
-		"apt-get update -qq >/dev/null",
-		fmt.Sprintf("DEBIAN_FRONTEND=noninteractive apt-get install -y -qq %s >/dev/null", preReqs),
-		"mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings",
-		fmt.Sprintf(
-			"curl -fsSL \"%s/linux/%s/gpg\" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg",
-			i.opts.downloadURL, i.distro.ID,
-		),
+		cmdAptUpdate,
+		cmdAptInstall,
+		cmdKeyringDir,
+		cmdDockerGPG,
 		"chmod a+r /etc/apt/keyrings/docker.gpg",
 		fmt.Sprintf("echo %q > /etc/apt/sources.list.d/docker.list", aptRepo),
-		"apt-get update -qq >/dev/null",
+		cmdAptUpdate,
 	}
 
-	return i.executor.RunCommandsWithRetry(shC, cmds, DefaultTimeout)
+	return i.executor.RunCommandsWithRetry(ctx, shC, cmds, DefaultTimeout)
 }
 
 func (i *DebianInstaller) findVersions() (string, string, error) {