diff --git a/internal/dispatch/gcf/gcp.go b/internal/dispatch/gcf/gcp.go index fb3394e75a..4cdca1d7f8 100644 --- a/internal/dispatch/gcf/gcp.go +++ b/internal/dispatch/gcf/gcp.go @@ -204,7 +204,8 @@ func (c *LiveGCFClient) getPollDelay() func(time.Duration) <-chan time.Time { return time.After } -// CreateServiceAccount creates a new service account. +// CreateServiceAccount creates a new service account. Retries on HTTP 429 +// (quota exhaustion) with exponential backoff via doWIFRequestWithRetry. func (c *LiveGCFClient) CreateServiceAccount(ctx context.Context, projectID, saName, displayName string) error { reqURL := fmt.Sprintf("https://iam.googleapis.com/v1/projects/%s/serviceAccounts", url.PathEscape(projectID)) @@ -222,7 +223,7 @@ func (c *LiveGCFClient) CreateServiceAccount(ctx context.Context, projectID, saN } payload := string(payloadBytes) - resp, err := c.Client.DoRequest(ctx, http.MethodPost, reqURL, payload) + resp, err := c.doWIFRequestWithRetry(ctx, http.MethodPost, reqURL, payload) if err != nil { return fmt.Errorf("creating service account: %w", err) } @@ -259,7 +260,8 @@ func (c *LiveGCFClient) DeleteServiceAccount(ctx context.Context, projectID, saE return nil } -// CreateWIFPool creates a new WIF pool. +// CreateWIFPool creates a new WIF pool. Retries on HTTP 429 (quota +// exhaustion) with exponential backoff via doWIFRequestWithRetry. func (c *LiveGCFClient) CreateWIFPool(ctx context.Context, projectNumber, poolID, displayName string) error { reqURL := fmt.Sprintf("https://iam.googleapis.com/v1/projects/%s/locations/global/workloadIdentityPools?workloadIdentityPoolId=%s", url.PathEscape(projectNumber), url.QueryEscape(poolID)) @@ -269,7 +271,7 @@ func (c *LiveGCFClient) CreateWIFPool(ctx context.Context, projectNumber, poolID } payload := string(payloadBytes) - resp, err := c.Client.DoRequest(ctx, http.MethodPost, reqURL, payload) + resp, err := c.doWIFRequestWithRetry(ctx, http.MethodPost, reqURL, payload) if err != nil { return fmt.Errorf("creating WIF pool: %w", err) } diff --git a/internal/dispatch/gcf/gcp_test.go b/internal/dispatch/gcf/gcp_test.go index acc927bd6d..7511dce9a8 100644 --- a/internal/dispatch/gcf/gcp_test.go +++ b/internal/dispatch/gcf/gcp_test.go @@ -2965,6 +2965,56 @@ func TestLiveGCFClient_DeleteWIFPool(t *testing.T) { }) } +// --- IAM quota retry (429) for CreateWIFPool and CreateServiceAccount --- + +func TestLiveGCFClient_CreateWIFPool_RetriesOn429(t *testing.T) { + origDelay := iamRetryDelay + iamRetryDelay = func(_ int) time.Duration { return time.Millisecond } + t.Cleanup(func() { iamRetryDelay = origDelay }) + + t.Run("succeeds after transient 429", func(t *testing.T) { + callCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + callCount++ + if callCount == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, `{"name":"operations/pool-op","done":true}`) + })) + defer srv.Close() + + err := newTestClient(srv).CreateWIFPool(context.Background(), "123", "pool", "Pool") + require.NoError(t, err) + assert.Equal(t, 2, callCount) + }) +} + +func TestLiveGCFClient_CreateServiceAccount_RetriesOn429(t *testing.T) { + origDelay := iamRetryDelay + iamRetryDelay = func(_ int) time.Duration { return time.Millisecond } + t.Cleanup(func() { iamRetryDelay = origDelay }) + + t.Run("succeeds after transient 429", func(t *testing.T) { + callCount := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + callCount++ + if callCount == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, `{"email":"sa@proj.iam.gserviceaccount.com"}`) + })) + defer srv.Close() + + err := newTestClient(srv).CreateServiceAccount(context.Background(), "proj", "sa", "SA") + require.NoError(t, err) + assert.Equal(t, 2, callCount) + }) +} + // --- encodeBase64 --- func TestEncodeBase64(t *testing.T) {