fix(#705): refresh OIDC token during agent runs to prevent expiry

internal/cli/run.go

@@ -5,6 +5,7 @@ import (
 	"bufio"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/sha256"
 	"debug/elf"
 	"encoding/hex"
@@ -18,6 +19,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -448,6 +450,26 @@ func runAgent(agentName, fullsendDir, outputBase, targetRepo, fullsendBinary str
 		return fmt.Errorf("creating run directory: %w", err)
 	}
 
+	oidcCtx, oidcCancel := context.WithCancel(context.Background())
+	var oidcWg sync.WaitGroup
+	if oidcURL := os.Getenv("FULLSEND_GCP_OIDC_URL"); oidcURL != "" {
+		oidcAuth, err := readOIDCAuthFile(os.Getenv("FULLSEND_GCP_OIDC_AUTH_FILE"))
+		if err != nil {
+			printer.StepWarn("OIDC token refresh disabled: " + err.Error())
+		} else {
+			printer.StepDone("OIDC token refresh enabled (WIF mode)")
+			oidcWg.Add(1)
+			go func() {
+				defer oidcWg.Done()
+				runOIDCRefresh(oidcCtx, sshConfigPath, sandboxName, oidcURL, oidcAuth, printer)
+			}()
+		}
+	}
+	defer func() {
+		oidcCancel()
+		oidcWg.Wait()
+	}()
+
 	var lastExitCode int
 	var runCount int
 
@@ -922,6 +944,92 @@ func runHeartbeat(printer *ui.Printer, start time.Time, timeout time.Duration, d
 	}
 }
 
+func readOIDCAuthFile(path string) (string, error) {
+	if path == "" {
+		return "", fmt.Errorf("FULLSEND_GCP_OIDC_AUTH_FILE not set")
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", fmt.Errorf("reading OIDC auth file: %w", err)
+	}
+	val := strings.TrimSpace(string(data))
+	if val == "" {
+		return "", fmt.Errorf("OIDC auth file is empty")
+	}
+	return val, nil
+}
+
+var oidcRefreshInterval = 4 * time.Minute
+
+func runOIDCRefresh(ctx context.Context, sshConfigPath, sandboxName, oidcURL, oidcAuth string, printer *ui.Printer) {
+	ticker := time.NewTicker(oidcRefreshInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := refreshOIDCToken(ctx, sshConfigPath, sandboxName, oidcURL, oidcAuth); err != nil {
+				if ctx.Err() != nil {
+					return
+				}
+				printer.StepWarn("OIDC token refresh failed: " + err.Error())
+			} else {
+				printer.StepDone("OIDC token refreshed")
+			}
+		}
+	}
+}
+
+func refreshOIDCToken(ctx context.Context, sshConfigPath, sandboxName, oidcURL, oidcAuth string) error {
+	req, err := http.NewRequestWithContext(ctx, "GET", oidcURL, nil)
+	if err != nil {
+		return fmt.Errorf("creating request: %w", err)
+	}
+	req.Header.Set("Authorization", oidcAuth)
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("fetching OIDC token: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("OIDC endpoint returned HTTP %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
+	if err != nil {
+		return fmt.Errorf("reading OIDC token response: %w", err)
+	}
+	if len(body) == 0 {
+		return fmt.Errorf("OIDC endpoint returned empty token")
+	}
+	if !json.Valid(body) {
+		return fmt.Errorf("OIDC endpoint returned non-JSON response")
+	}
+
+	tmpFile, err := os.CreateTemp("", "fullsend-oidc-*.token")
+	if err != nil {
+		return fmt.Errorf("creating temp token file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+
+	if _, err := tmpFile.Write(body); err != nil {
+		tmpFile.Close()
+		return fmt.Errorf("writing temp token file: %w", err)
+	}
+	tmpFile.Close()
+
+	remotePath := sandbox.SandboxWorkspace + "/.gcp-oidc-token"
+	if err := sandbox.SCP(sshConfigPath, sandboxName, tmpFile.Name(), remotePath); err != nil {
+		return fmt.Errorf("copying token to sandbox: %w", err)
+	}
+
+	return nil
+}
+
 func buildClaudeCommand(agentName, model, repoDir string) string {
 	envFile := sandbox.SandboxWorkspace + "/.env"
 

internal/cli/run_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
@@ -13,7 +14,9 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -354,6 +357,122 @@ func TestResolveLinuxBinary_Download(t *testing.T) {
 	assert.NoError(t, validateLinuxBinary(binPath), "downloaded binary should be a valid Linux/amd64 ELF")
 }
 
+func TestReadOIDCAuthFile_Success(t *testing.T) {
+	f := filepath.Join(t.TempDir(), "auth")
+	require.NoError(t, os.WriteFile(f, []byte("bearer test-token"), 0o600))
+	val, err := readOIDCAuthFile(f)
+	require.NoError(t, err)
+	assert.Equal(t, "bearer test-token", val)
+}
+
+func TestReadOIDCAuthFile_EmptyPath(t *testing.T) {
+	_, err := readOIDCAuthFile("")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "not set")
+}
+
+func TestReadOIDCAuthFile_EmptyFile(t *testing.T) {
+	f := filepath.Join(t.TempDir(), "auth")
+	require.NoError(t, os.WriteFile(f, []byte(""), 0o600))
+	_, err := readOIDCAuthFile(f)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "empty")
+}
+
+func TestRefreshOIDCToken_FetchSucceedsSCPFails(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "bearer test-auth", r.Header.Get("Authorization"))
+		fmt.Fprint(w, `{"value":"fresh-oidc-token-content"}`)
+	}))
+	defer srv.Close()
+
+	err := refreshOIDCToken(context.Background(), "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "copying token to sandbox")
+}
+
+func TestRefreshOIDCToken_HTTPError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+	}))
+	defer srv.Close()
+
+	err := refreshOIDCToken(context.Background(), "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "403")
+}
+
+func TestRefreshOIDCToken_EmptyResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	err := refreshOIDCToken(context.Background(), "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "empty token")
+}
+
+func TestRefreshOIDCToken_NonJSONResponse(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, "<html>Service Unavailable</html>")
+	}))
+	defer srv.Close()
+
+	err := refreshOIDCToken(context.Background(), "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "non-JSON response")
+}
+
+func TestRefreshOIDCToken_CancelledContext(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `{"value":"fresh-oidc-token-content"}`)
+	}))
+	defer srv.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err := refreshOIDCToken(ctx, "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "fetching OIDC token")
+}
+
+func TestRunOIDCRefresh_TicksAndStops(t *testing.T) {
+	var calls atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		fmt.Fprint(w, `{"value":"fresh-oidc-token-content"}`)
+	}))
+	defer srv.Close()
+
+	origInterval := oidcRefreshInterval
+	oidcRefreshInterval = 50 * time.Millisecond
+	defer func() { oidcRefreshInterval = origInterval }()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	printer := ui.New(io.Discard)
+
+	finished := make(chan struct{})
+	go func() {
+		runOIDCRefresh(ctx, "nonexistent-ssh-config", "nonexistent-sandbox", srv.URL, "bearer test-auth", printer)
+		close(finished)
+	}()
+
+	require.Eventually(t, func() bool { return calls.Load() >= 2 }, 2*time.Second, 10*time.Millisecond,
+		"expected at least 2 refresh calls")
+
+	cancel()
+
+	select {
+	case <-finished:
+	case <-time.After(2 * time.Second):
+		t.Fatal("runOIDCRefresh did not exit after context was cancelled")
+	}
+
+	assert.GreaterOrEqual(t, calls.Load(), int32(2))
+}
+
 func TestDownloadChecksumForAsset_ParsesLine(t *testing.T) {
 	body := "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014  fullsend_1.0.0_linux_arm64.tar.gz\n" +
 		"60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752  fullsend_1.0.0_linux_amd64.tar.gz\n"

internal/scaffold/fullsend-repo/scripts/prepare-sandbox-credentials.sh

@@ -9,9 +9,9 @@ set -euo pipefail
 # endpoint, so this script pre-fetches the OIDC token and rewrites the config
 # to use a file-based credential source instead.
 #
-# Note: the OIDC token expires after ~10 min but the GCP access token obtained
-# via STS lasts 1 hour. Runs exceeding 1 hour will fail on access token refresh
-# since the static OIDC token will have expired.
+# Note: the OIDC token expires after ~10 min. The fullsend CLI refreshes it
+# automatically using FULLSEND_GCP_OIDC_URL and FULLSEND_GCP_OIDC_AUTH_FILE
+# exported below.
 #
 # In SA-key mode (type != external_account), this script is a no-op.
 
@@ -43,6 +43,12 @@ if [[ "$CRED_TYPE" == "external_account" ]]; then
     }
   }' "$CRED_CONFIG" > "$SANDBOX_CREDS"
 
+  OIDC_AUTH_FILE="$RUNNER_TEMP/gcp-oidc-auth"
+  printf '%s' "$OIDC_AUTH" > "$OIDC_AUTH_FILE"
+  chmod 600 "$OIDC_AUTH_FILE"
+
   echo "GOOGLE_APPLICATION_CREDENTIALS=$SANDBOX_CREDS" >> "$GITHUB_ENV"
   echo "GCP_OIDC_TOKEN_FILE=$OIDC_DEST" >> "$GITHUB_ENV"
+  echo "FULLSEND_GCP_OIDC_URL=$OIDC_URL" >> "$GITHUB_ENV"
+  echo "FULLSEND_GCP_OIDC_AUTH_FILE=$OIDC_AUTH_FILE" >> "$GITHUB_ENV"
 fi