diff --git a/.dockerignore b/.dockerignore index a78f0089c51c..0b19c9760285 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ bin/ hack/tools/bin/ contrib/ +!contrib/ci/gocacheprog/ .github/ .tekton/ .ci-operator.yaml diff --git a/.github/actions/warm-go-cache/action.yaml b/.github/actions/warm-go-cache/action.yaml index 993dca6e84fc..5a2cb19c7d7f 100644 --- a/.github/actions/warm-go-cache/action.yaml +++ b/.github/actions/warm-go-cache/action.yaml @@ -1,22 +1,17 @@ name: 'Warm Go build cache' -description: 'Set GOCACHE via fuse-overlayfs over the EFS-backed PV or a writable fallback' +description: 'Set GOCACHEPROG to use EFS-backed read-only cache with local writable overlay' runs: using: composite steps: - shell: bash run: | - mkdir -p /tmp/go-build-cache - mounted=false - if [ -d /cache/go-build ] && command -v fuse-overlayfs >/dev/null 2>&1 && [ -e /dev/fuse ]; then - mkdir -p /tmp/go-cache-upper /tmp/go-cache-work - if fuse-overlayfs -o lowerdir=/cache/go-build,upperdir=/tmp/go-cache-upper,workdir=/tmp/go-cache-work /tmp/go-build-cache; then - mounted=true - else - echo "::warning::fuse-overlayfs mount failed, falling back to copy" + if command -v gocacheprog >/dev/null 2>&1; then + mkdir -p /tmp/go-build-cache + args="--rw /tmp/go-build-cache" + if [ -d /cache/go-build ]; then + args="--ro /cache/go-build $args" fi + echo "GOCACHEPROG=gocacheprog $args" >> "$GITHUB_ENV" + else + echo "::warning::gocacheprog not found, using default cache" fi - if [ "$mounted" = "false" ] && [ -d /cache/go-build ]; then - timeout 120 cp -a /cache/go-build/. /tmp/go-build-cache/ || \ - echo "::warning::Failed to copy EFS cache, proceeding without cache" - fi - echo "GOCACHE=/tmp/go-build-cache" >> "$GITHUB_ENV" diff --git a/Dockerfile.github-actions-runner b/Dockerfile.github-actions-runner index edbf622f822f..4f98f409b168 100644 --- a/Dockerfile.github-actions-runner +++ b/Dockerfile.github-actions-runner @@ -11,7 +11,6 @@ RUN apt-get update && \ curl \ ca-certificates \ python3-pip \ - fuse-overlayfs \ && rm -rf /var/lib/apt/lists/* ARG TARGETARCH @@ -35,4 +34,7 @@ RUN cd /tmp/tools && \ -o /opt/lint-tools/kube-api-linter.so sigs.k8s.io/kube-api-linter/pkg/plugin && \ rm -rf /tmp/tools +COPY contrib/ci/gocacheprog/ /tmp/gocacheprog/ +RUN cd /tmp/gocacheprog && go build -o /usr/local/bin/gocacheprog . && rm -rf /tmp/gocacheprog + USER runner diff --git a/contrib/ci/gocacheprog/go.mod b/contrib/ci/gocacheprog/go.mod new file mode 100644 index 000000000000..d24d8951481a --- /dev/null +++ b/contrib/ci/gocacheprog/go.mod @@ -0,0 +1,3 @@ +module github.com/openshift/hypershift/contrib/ci/gocacheprog + +go 1.25 diff --git a/contrib/ci/gocacheprog/main.go b/contrib/ci/gocacheprog/main.go new file mode 100644 index 000000000000..d2ad024be99e --- /dev/null +++ b/contrib/ci/gocacheprog/main.go @@ -0,0 +1,196 @@ +// gocacheprog implements the GOCACHEPROG protocol (Go 1.24+) to serve +// a read-only Go build cache with a writable overlay. GET requests are +// served from a writable local directory first, then from a read-only +// shared directory (e.g. an EFS-backed PVC). PUT requests always write +// to the local directory. This eliminates the need to copy the shared +// cache at job start. +package main + +import ( + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type request struct { + ID int64 `json:"ID"` + Command string `json:"Command"` + ActionID []byte `json:"ActionID,omitempty"` + OutputID []byte `json:"OutputID,omitempty"` + Body []byte `json:"-"` + BodySize int64 `json:"BodySize,omitempty"` +} + +type response struct { + ID int64 `json:"ID"` + Err string `json:"Err,omitempty"` + KnownCommands []string `json:"KnownCommands,omitempty"` + Miss bool `json:"Miss,omitempty"` + OutputID []byte `json:"OutputID,omitempty"` + Size int64 `json:"Size,omitempty"` + Time *time.Time `json:"Time,omitempty"` + DiskPath string `json:"DiskPath,omitempty"` +} + +func main() { + roDir := flag.String("ro", "", "read-only cache directory (e.g. EFS mount)") + rwDir := flag.String("rw", "", "writable cache directory (e.g. /tmp/go-build-cache)") + flag.Parse() + + if *rwDir == "" { + fmt.Fprintln(os.Stderr, "gocacheprog: --rw is required") + os.Exit(1) + } + + jd := json.NewDecoder(os.Stdin) + je := json.NewEncoder(os.Stdout) + var mu sync.Mutex + var wg sync.WaitGroup + + je.Encode(response{KnownCommands: []string{"get", "put", "close"}}) + + for { + var req request + if err := jd.Decode(&req); err != nil { + if err == io.EOF { + break + } + log.Fatalf("gocacheprog: decode request: %v", err) + } + + if req.Command == "put" && req.BodySize > 0 { + if err := jd.Decode(&req.Body); err != nil { + log.Fatalf("gocacheprog: decode body: %v", err) + } + } + + wg.Add(1) + go func() { + defer wg.Done() + res := handleRequest(&req, *roDir, *rwDir) + mu.Lock() + je.Encode(res) + mu.Unlock() + }() + } + wg.Wait() +} + +func handleRequest(req *request, roDir, rwDir string) response { + switch req.Command { + case "get": + return handleGet(req, roDir, rwDir) + case "put": + return handlePut(req, rwDir) + case "close": + return response{ID: req.ID} + default: + return response{ID: req.ID, Err: "unknown command"} + } +} + +// actionFile returns the path to a Go cache action entry. +// Format: //-a +func actionFile(dir string, id []byte) string { + if len(id) == 0 { + return "" + } + h := hex.EncodeToString(id) + return filepath.Join(dir, h[:2], h+"-a") +} + +// outputFile returns the path to a Go cache data file. +// Format: //-d +func outputFile(dir string, id []byte) string { + if len(id) == 0 { + return "" + } + h := hex.EncodeToString(id) + return filepath.Join(dir, h[:2], h+"-d") +} + +// lookup reads a Go cache action entry and verifies the data file exists. +// The action entry format is: v1 +func lookup(dir string, actionID []byte) (resp response, ok bool) { + data, err := os.ReadFile(actionFile(dir, actionID)) + if err != nil { + return + } + fields := strings.Fields(strings.TrimSpace(string(data))) + if len(fields) != 5 || fields[0] != "v1" { + return + } + if fields[1] != hex.EncodeToString(actionID) { + return + } + outputID, err := hex.DecodeString(fields[2]) + if err != nil { + return + } + nanos, err := strconv.ParseInt(fields[4], 10, 64) + if err != nil { + return + } + dPath := outputFile(dir, outputID) + fi, err := os.Stat(dPath) + if err != nil { + return + } + t := time.Unix(0, nanos) + return response{ + OutputID: outputID, + Size: fi.Size(), + Time: &t, + DiskPath: dPath, + }, true +} + +func handleGet(req *request, roDir, rwDir string) response { + if resp, ok := lookup(rwDir, req.ActionID); ok { + resp.ID = req.ID + return resp + } + if roDir != "" { + if resp, ok := lookup(roDir, req.ActionID); ok { + resp.ID = req.ID + return resp + } + } + return response{ID: req.ID, Miss: true} +} + +func handlePut(req *request, rwDir string) response { + dPath := outputFile(rwDir, req.OutputID) + if err := os.MkdirAll(filepath.Dir(dPath), 0o777); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + if err := os.WriteFile(dPath, req.Body, 0o666); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + + aPath := actionFile(rwDir, req.ActionID) + if err := os.MkdirAll(filepath.Dir(aPath), 0o777); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + entry := fmt.Sprintf("v1 %s %s %d %d\n", + hex.EncodeToString(req.ActionID), + hex.EncodeToString(req.OutputID), + req.BodySize, + time.Now().UnixNano(), + ) + if err := os.WriteFile(aPath, []byte(entry), 0o666); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + + return response{ID: req.ID, DiskPath: dPath} +} + diff --git a/contrib/ci/gocacheprog/main_test.go b/contrib/ci/gocacheprog/main_test.go new file mode 100644 index 000000000000..0e005cc17619 --- /dev/null +++ b/contrib/ci/gocacheprog/main_test.go @@ -0,0 +1,421 @@ +package main + +import ( + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +func mustDecodeHex(t *testing.T, s string) []byte { + t.Helper() + b, err := hex.DecodeString(s) + if err != nil { + t.Fatalf("bad hex %q: %v", s, err) + } + return b +} + +func TestActionFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + id []byte + want string + }{ + { + name: "When id is nil it should return empty", + id: nil, + want: "", + }, + { + name: "When id is empty it should return empty", + id: []byte{}, + want: "", + }, + { + name: "When id is valid it should return the correct path", + id: mustDecodeHex(t, "abcdef0123456789"), + want: filepath.Join("/cache", "ab", "abcdef0123456789-a"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := actionFile("/cache", tt.id) + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestOutputFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + id []byte + want string + }{ + { + name: "When id is nil it should return empty", + id: nil, + want: "", + }, + { + name: "When id is valid it should return the correct path", + id: mustDecodeHex(t, "abcdef0123456789"), + want: filepath.Join("/cache", "ab", "abcdef0123456789-d"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := outputFile("/cache", tt.id) + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +func writeCacheEntry(t *testing.T, dir string, actionID, outputID []byte, body []byte) { + t.Helper() + aPath := actionFile(dir, actionID) + dPath := outputFile(dir, outputID) + if err := os.MkdirAll(filepath.Dir(aPath), 0o777); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(dPath), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dPath, body, 0o666); err != nil { + t.Fatal(err) + } + entry := fmt.Sprintf("v1 %s %s %d %d\n", + hex.EncodeToString(actionID), + hex.EncodeToString(outputID), + len(body), + time.Now().UnixNano(), + ) + if err := os.WriteFile(aPath, []byte(entry), 0o666); err != nil { + t.Fatal(err) + } +} + +func TestLookup(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "aaaaaaaaaaaaaaaa") + outputID := mustDecodeHex(t, "bbbbbbbbbbbbbbbb") + body := []byte("hello world") + + t.Run("When entry exists it should return hit", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeCacheEntry(t, dir, actionID, outputID, body) + + resp, ok := lookup(dir, actionID) + if !ok { + t.Fatal("expected hit, got miss") + } + if hex.EncodeToString(resp.OutputID) != hex.EncodeToString(outputID) { + t.Errorf("outputID = %x, want %x", resp.OutputID, outputID) + } + if resp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", resp.Size, len(body)) + } + if resp.Time == nil { + t.Error("expected non-nil Time") + } + if resp.DiskPath == "" { + t.Error("expected non-empty DiskPath") + } + }) + + t.Run("When entry does not exist it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss, got hit") + } + }) + + t.Run("When action file has wrong format it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + aPath := actionFile(dir, actionID) + os.MkdirAll(filepath.Dir(aPath), 0o777) + os.WriteFile(aPath, []byte("garbage"), 0o666) + + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss for malformed entry") + } + }) + + t.Run("When data file is missing it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + aPath := actionFile(dir, actionID) + os.MkdirAll(filepath.Dir(aPath), 0o777) + entry := fmt.Sprintf("v1 %s %s 11 %d\n", + hex.EncodeToString(actionID), + hex.EncodeToString(outputID), + time.Now().UnixNano(), + ) + os.WriteFile(aPath, []byte(entry), 0o666) + + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss when data file is absent") + } + }) +} + +func TestHandleGet(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "1111111111111111") + outputID := mustDecodeHex(t, "2222222222222222") + body := []byte("cached output") + + t.Run("When entry is in rw dir it should return hit from rw", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + writeCacheEntry(t, rwDir, actionID, outputID, body) + + req := &request{ID: 42, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit") + } + if resp.ID != 42 { + t.Errorf("ID = %d, want 42", resp.ID) + } + if resp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", resp.Size, len(body)) + } + }) + + t.Run("When entry is only in ro dir it should return hit from ro", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + writeCacheEntry(t, roDir, actionID, outputID, body) + + req := &request{ID: 7, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit from ro") + } + if resp.ID != 7 { + t.Errorf("ID = %d, want 7", resp.ID) + } + }) + + t.Run("When entry is in neither dir it should return miss", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + + req := &request{ID: 3, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if !resp.Miss { + t.Fatal("expected miss") + } + }) + + t.Run("When roDir is empty string it should skip ro lookup", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + + req := &request{ID: 5, Command: "get", ActionID: actionID} + resp := handleGet(req, "", rwDir) + if !resp.Miss { + t.Fatal("expected miss") + } + }) + + t.Run("When rw shadows ro it should prefer rw", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + rwBody := []byte("rw content") + roBody := []byte("ro content, different size") + writeCacheEntry(t, rwDir, actionID, outputID, rwBody) + writeCacheEntry(t, roDir, actionID, outputID, roBody) + + req := &request{ID: 1, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit") + } + if resp.Size != int64(len(rwBody)) { + t.Errorf("size = %d, want %d (rw should shadow ro)", resp.Size, len(rwBody)) + } + }) +} + +func TestHandlePut(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "3333333333333333") + outputID := mustDecodeHex(t, "4444444444444444") + body := []byte("new output data") + + t.Run("When writing succeeds it should create action and data files", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + req := &request{ + ID: 10, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(req, rwDir) + if resp.Err != "" { + t.Fatalf("unexpected error: %s", resp.Err) + } + if resp.ID != 10 { + t.Errorf("ID = %d, want 10", resp.ID) + } + if resp.DiskPath == "" { + t.Error("expected non-empty DiskPath") + } + + data, err := os.ReadFile(resp.DiskPath) + if err != nil { + t.Fatalf("reading data file: %v", err) + } + if string(data) != string(body) { + t.Errorf("data = %q, want %q", data, body) + } + + lookupResp, ok := lookup(rwDir, actionID) + if !ok { + t.Fatal("expected lookup to succeed after put") + } + if lookupResp.Size != int64(len(body)) { + t.Errorf("lookup size = %d, want %d", lookupResp.Size, len(body)) + } + }) + + t.Run("When rwDir is unwritable it should return error", func(t *testing.T) { + t.Parallel() + req := &request{ + ID: 11, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(req, "/proc/nonexistent-gocacheprog-test") + if resp.Err == "" { + t.Fatal("expected error for unwritable dir") + } + if resp.ID != 11 { + t.Errorf("ID = %d, want 11", resp.ID) + } + }) +} + +func TestHandleRequest(t *testing.T) { + t.Parallel() + + t.Run("When command is close it should return empty response", func(t *testing.T) { + t.Parallel() + req := &request{ID: 99, Command: "close"} + resp := handleRequest(req, "", t.TempDir()) + if resp.ID != 99 { + t.Errorf("ID = %d, want 99", resp.ID) + } + if resp.Err != "" { + t.Errorf("unexpected error: %s", resp.Err) + } + }) + + t.Run("When command is unknown it should return error", func(t *testing.T) { + t.Parallel() + req := &request{ID: 100, Command: "delete"} + resp := handleRequest(req, "", t.TempDir()) + if resp.Err != "unknown command" { + t.Errorf("Err = %q, want %q", resp.Err, "unknown command") + } + }) +} + +func TestPutThenGet(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + actionID := mustDecodeHex(t, "5555555555555555") + outputID := mustDecodeHex(t, "6666666666666666") + body := []byte("round trip data") + + putReq := &request{ + ID: 1, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + putResp := handlePut(putReq, rwDir) + if putResp.Err != "" { + t.Fatalf("put error: %s", putResp.Err) + } + + getReq := &request{ID: 2, Command: "get", ActionID: actionID} + getResp := handleGet(getReq, "", rwDir) + if getResp.Miss { + t.Fatal("expected hit after put") + } + if getResp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", getResp.Size, len(body)) + } + if hex.EncodeToString(getResp.OutputID) != hex.EncodeToString(outputID) { + t.Errorf("outputID = %x, want %x", getResp.OutputID, outputID) + } +} + +func TestConcurrentPutAndGet(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + var wg sync.WaitGroup + + for i := range 50 { + wg.Add(1) + go func() { + defer wg.Done() + actionID := mustDecodeHex(t, fmt.Sprintf("%016x", i)) + outputID := mustDecodeHex(t, fmt.Sprintf("%016x", i+1000)) + body := []byte(fmt.Sprintf("body-%d", i)) + + putReq := &request{ + ID: int64(i), + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(putReq, rwDir) + if resp.Err != "" { + t.Errorf("put %d error: %s", i, resp.Err) + return + } + + getReq := &request{ID: int64(i + 1000), Command: "get", ActionID: actionID} + getResp := handleGet(getReq, "", rwDir) + if getResp.Miss { + t.Errorf("get %d: expected hit after put", i) + } + }() + } + wg.Wait() +}