ci: pre-pull and cache heavy e2e images via GitHub Actions cache

.github/e2e-images.txt

@@ -0,0 +1,21 @@
+# Images pre-pulled and cached for e2e tests.
+#
+# ALLOWED: passive images that are never the subject of pull-policy or
+# pull-skip assertions in the test suite.
+#
+# EXCLUDED (must NOT appear here):
+#   - alpine:* tags  (used by TestComposePull, TestPullRefreshWindow…, etc.)
+#   - does_not_exists / doesn_t_exists_either (intentional pull-failure tags)
+#   - Any image referenced by fixtures/compose-pull/ or fixtures/image-identity/
+#
+# When modifying this list, verify it does not overlap with the fixtures under
+# pkg/e2e/fixtures/compose-pull/ and pkg/e2e/fixtures/image-identity/.
+docker:dind
+fluent/fluent-bit:3.1.7-debug
+golang:alpine
+gtardif/sentences-api
+gtardif/sentences-db
+gtardif/sentences-web
+mariadb
+nginx:alpine
+registry:3

.github/workflows/ci.yml

@@ -264,6 +264,30 @@ jobs:
         env:
           BUILD_TAGS: e2e
 
+      - name: Restore e2e image cache
+        id: image-cache
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: /tmp/e2e-images.tar
+          # Key encodes the engine version + the image list so any change
+          # to .github/e2e-images.txt invalidates the cache automatically.
+          key: e2e-images-v1-docker${{ matrix.engine }}-${{ hashFiles('.github/e2e-images.txt') }}
+
+      - name: Load cached e2e images
+        if: steps.image-cache.outputs.cache-hit == 'true'
+        run: docker load --input /tmp/e2e-images.tar
+
+      - name: Pull and cache e2e images
+        if: steps.image-cache.outputs.cache-hit != 'true'
+        run: |
+          # Pull all passive images in parallel (4 at a time).
+          # alpine:* and pull-policy test images are intentionally absent from
+          # .github/e2e-images.txt so they are never pre-pulled.
+          grep -v '^#' .github/e2e-images.txt | xargs -P 4 -n 1 docker pull
+          # Pack them into a single archive for the GHA cache.
+          docker save $(cat .github/e2e-images.txt | grep -v '^#' | tr '\n' ' ') \
+            -o /tmp/e2e-images.tar
+
       - name: Setup tmate session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled }}
         uses: mxschmitt/action-tmate@35b54afac29c97fb54faba5b513f8fbd1882f113 # v3.24

Makefile

@@ -32,6 +32,8 @@ endif
 BUILD_FLAGS?=
 TEST_FLAGS?=
 E2E_TEST?=
+E2E_PARALLEL_PLUGIN?=4
+E2E_STANDALONE_PARALLEL?=4
 ifneq ($(E2E_TEST),)
 	TEST_FLAGS:=$(TEST_FLAGS) -run '$(E2E_TEST)'
 endif
@@ -75,11 +77,11 @@ install: binary
 
 .PHONY: e2e-compose
 e2e-compose: example-provider ## Run end to end local tests in plugin mode. Set E2E_TEST=TestName to run a single test
-	go run gotest.tools/gotestsum@latest --format testname --junitfile "/tmp/report/report.xml" -- -v $(TEST_FLAGS) -count=1 -timeout 20m ./pkg/e2e
+	go run gotest.tools/gotestsum@latest --format testname --junitfile "/tmp/report/report.xml" -- -v $(TEST_FLAGS) -count=1 -parallel=$(E2E_PARALLEL_PLUGIN) -timeout 20m ./pkg/e2e
 
 .PHONY: e2e-compose-standalone
 e2e-compose-standalone: ## Run End to end local tests in standalone mode. Set E2E_TEST=TestName to run a single test
-	go run gotest.tools/gotestsum@latest --format testname --junitfile "/tmp/report/report.xml" -- $(TEST_FLAGS) -v -count=1 -parallel=1 -timeout 20m --tags=standalone ./pkg/e2e
+	go run gotest.tools/gotestsum@latest --format testname --junitfile "/tmp/report/report.xml" -- $(TEST_FLAGS) -v -count=1 -parallel=$(E2E_STANDALONE_PARALLEL) -timeout 20m --tags=standalone ./pkg/e2e
 
 .PHONY: build-and-e2e-compose
 build-and-e2e-compose: build e2e-compose ## Compile the compose cli-plugin and run end to end local tests in plugin mode. Set E2E_TEST=TestName to run a single test

pkg/e2e/build_test.go

@@ -105,7 +105,7 @@ func TestLocalComposeBuild(t *testing.T) {
 			res.Assert(t, icmd.Expected{Out: "COPY static /usr/share/nginx/html"})
 			res.Assert(t, icmd.Expected{Out: "COPY static2 /usr/share/nginx/html"})
 
-			output := HTTPGetWithRetry(t, "http://localhost:8070", http.StatusOK, 2*time.Second, 20*time.Second)
+			output := HTTPGetWithRetry(t, fmt.Sprintf("http://localhost:%d", c.ServicePublishedPort(t, "build-test", "nginx", 80)), http.StatusOK, 2*time.Second, 20*time.Second)
 			assert.Assert(t, strings.Contains(output, "Hello from Nginx container"))
 
 			c.RunDockerCmd(t, "image", "inspect", "build-test-nginx")
@@ -301,13 +301,15 @@ func TestBuildPlatformsWithCorrectBuildxConfig(t *testing.T) {
 	}
 	c := NewParallelCLI(t)
 
-	// declare builder
-	result := c.RunDockerCmd(t, "buildx", "create", "--name", "build-platform", "--use", "--bootstrap")
+	// declare a per-test-unique builder to avoid container-name collisions
+	// when tests run in parallel on the same Docker daemon.
+	builderName := BuilderName(t, "build-platform")
+	result := c.RunDockerCmd(t, "buildx", "create", "--name", builderName, "--use", "--bootstrap")
 	assert.NilError(t, result.Error)
 
 	t.Cleanup(func() {
 		c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/platforms", "down")
-		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", "build-platform")
+		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", builderName)
 	})
 
 	t.Run("platform not supported by builder", func(t *testing.T) {
@@ -365,14 +367,16 @@ func TestBuildPlatformsWithCorrectBuildxConfig(t *testing.T) {
 func TestBuildPrivileged(t *testing.T) {
 	c := NewParallelCLI(t)
 
-	// declare builder
-	result := c.RunDockerCmd(t, "buildx", "create", "--name", "build-privileged", "--use", "--bootstrap", "--buildkitd-flags",
+	// declare a per-test-unique builder to avoid container-name collisions
+	// when tests run in parallel on the same Docker daemon.
+	builderName := BuilderName(t, "build-privileged")
+	result := c.RunDockerCmd(t, "buildx", "create", "--name", builderName, "--use", "--bootstrap", "--buildkitd-flags",
 		`'--allow-insecure-entitlement=security.insecure'`)
 	assert.NilError(t, result.Error)
 
 	t.Cleanup(func() {
 		c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/privileged", "down")
-		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", "build-privileged")
+		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", builderName)
 	})
 
 	t.Run("use build privileged mode to run insecure build command", func(t *testing.T) {
@@ -480,8 +484,9 @@ func TestBuildPlatformsStandardErrors(t *testing.T) {
 
 func TestBuildBuilder(t *testing.T) {
 	c := NewParallelCLI(t)
-	builderName := "build-with-builder"
-	// declare builder
+	// declare a per-test-unique builder to avoid container-name collisions
+	// when tests run in parallel on the same Docker daemon.
+	builderName := BuilderName(t, "build-with-builder")
 	result := c.RunDockerCmd(t, "buildx", "create", "--name", builderName, "--use", "--bootstrap")
 	assert.NilError(t, result.Error)
 
@@ -507,14 +512,16 @@ func TestBuildBuilder(t *testing.T) {
 func TestBuildEntitlements(t *testing.T) {
 	c := NewParallelCLI(t)
 
-	// declare builder
-	result := c.RunDockerCmd(t, "buildx", "create", "--name", "build-insecure", "--use", "--bootstrap", "--buildkitd-flags",
+	// declare a per-test-unique builder to avoid container-name collisions
+	// when tests run in parallel on the same Docker daemon.
+	builderName := BuilderName(t, "build-insecure")
+	result := c.RunDockerCmd(t, "buildx", "create", "--name", builderName, "--use", "--bootstrap", "--buildkitd-flags",
 		`'--allow-insecure-entitlement=security.insecure'`)
 	assert.NilError(t, result.Error)
 
 	t.Cleanup(func() {
 		c.RunDockerComposeCmd(t, "--project-directory", "fixtures/build-test/entitlements", "down")
-		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", "build-insecure")
+		_ = c.RunDockerCmd(t, "buildx", "rm", "-f", builderName)
 	})
 
 	t.Run("use build privileged mode to run insecure build command", func(t *testing.T) {
@@ -624,7 +631,9 @@ func TestBuildTLS(t *testing.T) {
 	t.Helper()
 
 	c := NewParallelCLI(t)
-	const dindBuilder = "e2e-dind-builder"
+	// Use a per-test-unique name to avoid container/context collisions when
+	// tests run in parallel on the same Docker daemon.
+	dindBuilder := BuilderName(t, "e2e-dind-builder")
 	tmp := t.TempDir()
 
 	t.Cleanup(func() {

pkg/e2e/cascade_test.go

@@ -57,7 +57,7 @@ func TestCascadeIgnoresOneOffContainer(t *testing.T) {
 	})
 
 	poll.WaitOn(t, expectOutput(res, "Attaching to running-1"),
-		poll.WithDelay(100*time.Millisecond), poll.WithTimeout(30*time.Second))
+		poll.WithDelay(500*time.Millisecond), poll.WithTimeout(30*time.Second))
 
 	c.RunDockerComposeCmd(t, "-f", "./fixtures/cascade/compose.yaml",
 		"run", "--rm", "--no-deps", "running", "/bin/true")

pkg/e2e/compose_test.go

@@ -44,7 +44,8 @@ func TestLocalComposeUp(t *testing.T) {
 		res := c.RunDockerComposeCmd(t, "-p", projectName, "ps")
 		res.Assert(t, icmd.Expected{Out: `web`})
 
-		endpoint := "http://localhost:90"
+		webPort := c.ServicePublishedPort(t, projectName, "web", 80)
+		endpoint := fmt.Sprintf("http://localhost:%d", webPort)
 		output := HTTPGetWithRetry(t, endpoint+"/words/noun", http.StatusOK, 2*time.Second, 20*time.Second)
 		assert.Assert(t, strings.Contains(output, `"word":`))
 

pkg/e2e/fixtures/build-test/compose.yaml

@@ -2,7 +2,7 @@ services:
   nginx:
     build: nginx-build
     ports:
-      - 8070:80
+      - 80
 
   nginx2:
     build: nginx-build2

pkg/e2e/fixtures/ipc-test/compose.yaml

@@ -6,7 +6,7 @@ services:
   container:
     image: alpine
     command: top
-    ipc: "container:ipc_mode_container"
+    ipc: "container:ipc_e2e-src"
   shareable:
     image: alpine
     command: top

pkg/e2e/fixtures/logging-driver/compose.yaml

@@ -2,6 +2,11 @@ services:
   fluentbit:
     image: fluent/fluent-bit:3.1.7-debug
     ports:
+      # Port 24224 intentionally fixed: the Docker fluentd log driver sets
+      # fluentd-address at service-start time (see compose.yaml and
+      # TestLoggingDriver in compose_up_test.go). Dynamic resolution would
+      # require a two-phase compose up which is unnecessarily invasive.
+      # TestLoggingDriver is serial (no t.Parallel()), so collision risk is nil.
       - "24224:24224"
       - "24224:24224/udp"
     environment:

pkg/e2e/fixtures/network-test/compose.yaml

@@ -15,15 +15,15 @@ services:
     image: gtardif/sentences-api
     init: true
     ports:
-      - "8080:8080"
+      - "8080"
     networks:
       - dbnet
       - servicenet
   web:
     image: gtardif/sentences-web
     init: true
     ports:
-      - "80:80"
+      - "80"
     labels:
       - "my-label=test"
     networks:

pkg/e2e/fixtures/sentences/compose.yaml

@@ -6,12 +6,12 @@ services:
     image: gtardif/sentences-api
     init: true
     ports:
-      - "95:8080"
+      - "8080"
   web:
     image: gtardif/sentences-web
     init: true
     ports:
-      - "90:80"
+      - "80"
     labels:
       - "my-label=test"
     healthcheck:

pkg/e2e/fixtures/volume-test/compose.yaml

@@ -4,7 +4,7 @@ services:
     volumes:
       - ./static:/usr/share/nginx/html
     ports:
-      - 8090:80
+      - 80
 
   nginx2:
     build: nginx-build
@@ -13,7 +13,7 @@ services:
       - /usr/src/app/node_modules
       - otherVol:/usr/share/nginx/test
     ports:
-      - 9090:80
+      - 80
     configs:
       - myconfig
     secrets:

pkg/e2e/framework.go

@@ -22,10 +22,12 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -517,3 +519,42 @@ func (c *CLI) cleanupWithDown(t testing.TB, project string, args ...string) {
 	t.Helper()
 	c.RunDockerComposeCmd(t, append([]string{"-p", project, "down", "-v", "--remove-orphans"}, args...)...)
 }
+
+// ServicePublishedPort returns the ephemeral host port mapped to targetPort
+// on the named service in the given compose project. It requires the service
+// to be already running. The test fails immediately if the mapping cannot be
+// resolved.
+func (c *CLI) ServicePublishedPort(t testing.TB, project, service string, targetPort int) int {
+	t.Helper()
+	res := c.RunDockerComposeCmd(t, "-p", project, "port", service, strconv.Itoa(targetPort))
+	addr := strings.SplitN(res.Stdout(), "\n", 2)[0]
+	_, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("ServicePublishedPort: cannot parse compose port output %q: %v", addr, err)
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		t.Fatalf("ServicePublishedPort: invalid port number %q: %v", portStr, err)
+	}
+	return port
+}
+
+// BuilderName returns a buildx builder name that is unique to the test,
+// preventing container-name collisions when tests run in parallel on the
+// same Docker daemon.
+func BuilderName(t testing.TB, base string) string {
+	t.Helper()
+	safe := strings.Map(func(r rune) rune {
+		switch {
+		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-':
+			return r
+		default:
+			return '-'
+		}
+	}, t.Name())
+	name := base + "-" + safe
+	if len(name) > 60 {
+		name = strings.TrimRight(name[:60], "-")
+	}
+	return name
+}

pkg/e2e/ipc_test.go

@@ -30,7 +30,7 @@ func TestIPC(t *testing.T) {
 	const projectName = "ipc_e2e"
 	var cid string
 	t.Run("create ipc mode container", func(t *testing.T) {
-		res := c.RunDockerCmd(t, "run", "-d", "--rm", "--ipc=shareable", "--name", "ipc_mode_container", "alpine",
+		res := c.RunDockerCmd(t, "run", "-d", "--rm", "--ipc=shareable", "--name", projectName+"-src", "alpine",
 			"top")
 		cid = strings.Trim(res.Stdout(), "\n")
 	})
@@ -59,6 +59,6 @@ func TestIPC(t *testing.T) {
 		_ = c.RunDockerComposeCmd(t, "--project-name", projectName, "down")
 	})
 	t.Run("remove ipc mode container", func(t *testing.T) {
-		_ = c.RunDockerCmd(t, "rm", "-f", "ipc_mode_container")
+		_ = c.RunDockerCmd(t, "rm", "-f", projectName+"-src")
 	})
 }

pkg/e2e/networks_test.go

@@ -43,7 +43,8 @@ func TestNetworks(t *testing.T) {
 	res := c.RunDockerComposeCmd(t, "ps")
 	res.Assert(t, icmd.Expected{Out: `web`})
 
-	endpoint := "http://localhost:80"
+	webPort := c.ServicePublishedPort(t, projectName, "web", 80)
+	endpoint := fmt.Sprintf("http://localhost:%d", webPort)
 	output := HTTPGetWithRetry(t, endpoint+"/words/noun", http.StatusOK, 2*time.Second, 20*time.Second)
 	assert.Assert(t, strings.Contains(output, `"word":`))
 
@@ -52,7 +53,7 @@ func TestNetworks(t *testing.T) {
 	res.Assert(t, icmd.Expected{Out: "microservices"})
 
 	res = c.RunDockerComposeCmd(t, "port", "words", "8080")
-	res.Assert(t, icmd.Expected{Out: `0.0.0.0:8080`})
+	res.Assert(t, icmd.Expected{Out: `0.0.0.0:`})
 
 	c.RunDockerComposeCmd(t, "down", "-t0", "-v")
 	res = c.RunDockerCmd(t, "network", "ls")

pkg/e2e/publish_test.go

@@ -178,7 +178,7 @@ func TestPublish(t *testing.T) {
 			return poll.Success()
 		}
 		return poll.Continue("registry not ready, status %d", resp.StatusCode)
-	}, poll.WithTimeout(10*time.Second), poll.WithDelay(100*time.Millisecond))
+	}, poll.WithTimeout(10*time.Second), poll.WithDelay(500*time.Millisecond))
 
 	res := c.RunDockerComposeCmd(t, "-f", "./fixtures/publish/oci/compose.yaml", "-f", "./fixtures/publish/oci/compose-override.yaml",
 		"-p", projectName, "publish", "--with-env", "--yes", "--insecure-registry", registry+"/test:test")

pkg/e2e/volumes_test.go

@@ -44,7 +44,8 @@ func TestLocalComposeVolume(t *testing.T) {
 	})
 
 	t.Run("access bind mount data", func(t *testing.T) {
-		output := HTTPGetWithRetry(t, "http://localhost:8090", http.StatusOK, 2*time.Second, 20*time.Second)
+		port := c.ServicePublishedPort(t, projectName, "nginx", 80)
+		output := HTTPGetWithRetry(t, fmt.Sprintf("http://localhost:%d", port), http.StatusOK, 2*time.Second, 20*time.Second)
 		assert.Assert(t, strings.Contains(output, "Hello from Nginx container"))
 	})