CNTRLPLANE-3329: Use GOCACHEPROG for zero-copy EFS build cache

.github/actions/warm-go-cache/action.yaml

@@ -1,22 +1,17 @@
 name: 'Warm Go build cache'
-description: 'Set GOCACHE via fuse-overlayfs over the EFS-backed PV or a writable fallback'
+description: 'Set GOCACHEPROG to use EFS-backed read-only cache with local writable overlay'
 runs:
   using: composite
   steps:
     - shell: bash
       run: |
-        mkdir -p /tmp/go-build-cache
-        mounted=false
-        if [ -d /cache/go-build ] && command -v fuse-overlayfs >/dev/null 2>&1 && [ -e /dev/fuse ]; then
-          mkdir -p /tmp/go-cache-upper /tmp/go-cache-work
-          if fuse-overlayfs -o lowerdir=/cache/go-build,upperdir=/tmp/go-cache-upper,workdir=/tmp/go-cache-work /tmp/go-build-cache; then
-            mounted=true
-          else
-            echo "::warning::fuse-overlayfs mount failed, falling back to copy"
+        if command -v gocacheprog >/dev/null 2>&1; then
+          mkdir -p /tmp/go-build-cache
+          args="--rw /tmp/go-build-cache"
+          if [ -d /cache/go-build ]; then
+            args="--ro /cache/go-build $args"
           fi
+          echo "GOCACHEPROG=gocacheprog $args" >> "$GITHUB_ENV"
+        else
+          echo "::warning::gocacheprog not found, using default cache"
         fi
-        if [ "$mounted" = "false" ] && [ -d /cache/go-build ]; then
-          timeout 120 cp -a /cache/go-build/. /tmp/go-build-cache/ || \
-            echo "::warning::Failed to copy EFS cache, proceeding without cache"
-        fi
-        echo "GOCACHE=/tmp/go-build-cache" >> "$GITHUB_ENV"

Dockerfile.github-actions-runner

@@ -11,7 +11,6 @@ RUN apt-get update && \
         curl \
         ca-certificates \
         python3-pip \
-        fuse-overlayfs \
     && rm -rf /var/lib/apt/lists/*
 
 ARG TARGETARCH
@@ -35,4 +34,7 @@ RUN cd /tmp/tools && \
         -o /opt/lint-tools/kube-api-linter.so sigs.k8s.io/kube-api-linter/pkg/plugin && \
     rm -rf /tmp/tools
 
+COPY contrib/ci/gocacheprog/ /tmp/gocacheprog/
+RUN cd /tmp/gocacheprog && go build -o /usr/local/bin/gocacheprog . && rm -rf /tmp/gocacheprog
+
 USER runner

contrib/ci/gocacheprog/go.mod

@@ -0,0 +1,3 @@
+module github.com/openshift/hypershift/contrib/ci/gocacheprog
+
+go 1.25

contrib/ci/gocacheprog/main.go

@@ -0,0 +1,196 @@
+// gocacheprog implements the GOCACHEPROG protocol (Go 1.24+) to serve
+// a read-only Go build cache with a writable overlay. GET requests are
+// served from a writable local directory first, then from a read-only
+// shared directory (e.g. an EFS-backed PVC). PUT requests always write
+// to the local directory. This eliminates the need to copy the shared
+// cache at job start.
+package main
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type request struct {
+	ID       int64  `json:"ID"`
+	Command  string `json:"Command"`
+	ActionID []byte `json:"ActionID,omitempty"`
+	OutputID []byte `json:"OutputID,omitempty"`
+	Body     []byte `json:"-"`
+	BodySize int64  `json:"BodySize,omitempty"`
+}
+
+type response struct {
+	ID            int64      `json:"ID"`
+	Err           string     `json:"Err,omitempty"`
+	KnownCommands []string   `json:"KnownCommands,omitempty"`
+	Miss          bool       `json:"Miss,omitempty"`
+	OutputID      []byte     `json:"OutputID,omitempty"`
+	Size          int64      `json:"Size,omitempty"`
+	Time          *time.Time `json:"Time,omitempty"`
+	DiskPath      string     `json:"DiskPath,omitempty"`
+}
+
+func main() {
+	roDir := flag.String("ro", "", "read-only cache directory (e.g. EFS mount)")
+	rwDir := flag.String("rw", "", "writable cache directory (e.g. /tmp/go-build-cache)")
+	flag.Parse()
+
+	if *rwDir == "" {
+		fmt.Fprintln(os.Stderr, "gocacheprog: --rw is required")
+		os.Exit(1)
+	}
+
+	jd := json.NewDecoder(os.Stdin)
+	je := json.NewEncoder(os.Stdout)
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+
+	je.Encode(response{KnownCommands: []string{"get", "put", "close"}})
+
+	for {
+		var req request
+		if err := jd.Decode(&req); err != nil {
+			if err == io.EOF {
+				break
+			}
+			log.Fatalf("gocacheprog: decode request: %v", err)
+		}
+
+		if req.Command == "put" && req.BodySize > 0 {
+			if err := jd.Decode(&req.Body); err != nil {
+				log.Fatalf("gocacheprog: decode body: %v", err)
+			}
+		}
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			res := handleRequest(&req, *roDir, *rwDir)
+			mu.Lock()
+			je.Encode(res)
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+}
+
+func handleRequest(req *request, roDir, rwDir string) response {
+	switch req.Command {
+	case "get":
+		return handleGet(req, roDir, rwDir)
+	case "put":
+		return handlePut(req, rwDir)
+	case "close":
+		return response{ID: req.ID}
+	default:
+		return response{ID: req.ID, Err: "unknown command"}
+	}
+}
+
+// actionFile returns the path to a Go cache action entry.
+// Format: <dir>/<first-byte-hex>/<full-hex-actionID>-a
+func actionFile(dir string, id []byte) string {
+	if len(id) == 0 {
+		return ""
+	}
+	h := hex.EncodeToString(id)
+	return filepath.Join(dir, h[:2], h+"-a")
+}
+
+// outputFile returns the path to a Go cache data file.
+// Format: <dir>/<first-byte-hex>/<full-hex-outputID>-d
+func outputFile(dir string, id []byte) string {
+	if len(id) == 0 {
+		return ""
+	}
+	h := hex.EncodeToString(id)
+	return filepath.Join(dir, h[:2], h+"-d")
+}
+
+// lookup reads a Go cache action entry and verifies the data file exists.
+// The action entry format is: v1 <hexActionID> <hexOutputID> <size> <unixnanos>
+func lookup(dir string, actionID []byte) (resp response, ok bool) {
+	data, err := os.ReadFile(actionFile(dir, actionID))
+	if err != nil {
+		return
+	}
+	fields := strings.Fields(strings.TrimSpace(string(data)))
+	if len(fields) != 5 || fields[0] != "v1" {
+		return
+	}
+	if fields[1] != hex.EncodeToString(actionID) {
+		return
+	}
+	outputID, err := hex.DecodeString(fields[2])
+	if err != nil {
+		return
+	}
+	nanos, err := strconv.ParseInt(fields[4], 10, 64)
+	if err != nil {
+		return
+	}
+	dPath := outputFile(dir, outputID)
+	fi, err := os.Stat(dPath)
+	if err != nil {
+		return
+	}
+	t := time.Unix(0, nanos)
+	return response{
+		OutputID: outputID,
+		Size:     fi.Size(),
+		Time:     &t,
+		DiskPath: dPath,
+	}, true
+}
+
+func handleGet(req *request, roDir, rwDir string) response {
+	if resp, ok := lookup(rwDir, req.ActionID); ok {
+		resp.ID = req.ID
+		return resp
+	}
+	if roDir != "" {
+		if resp, ok := lookup(roDir, req.ActionID); ok {
+			resp.ID = req.ID
+			return resp
+		}
+	}
+	return response{ID: req.ID, Miss: true}
+}
+
+func handlePut(req *request, rwDir string) response {
+	dPath := outputFile(rwDir, req.OutputID)
+	if err := os.MkdirAll(filepath.Dir(dPath), 0o777); err != nil {
+		return response{ID: req.ID, Err: err.Error()}
+	}
+	if err := os.WriteFile(dPath, req.Body, 0o666); err != nil {
+		return response{ID: req.ID, Err: err.Error()}
+	}
+
+	aPath := actionFile(rwDir, req.ActionID)
+	if err := os.MkdirAll(filepath.Dir(aPath), 0o777); err != nil {
+		return response{ID: req.ID, Err: err.Error()}
+	}
+	entry := fmt.Sprintf("v1 %s %s %d %d\n",
+		hex.EncodeToString(req.ActionID),
+		hex.EncodeToString(req.OutputID),
+		req.BodySize,
+		time.Now().UnixNano(),
+	)
+	if err := os.WriteFile(aPath, []byte(entry), 0o666); err != nil {
+		return response{ID: req.ID, Err: err.Error()}
+	}
+
+	return response{ID: req.ID, DiskPath: dPath}
+}
+