fix(#5716): fail fast on concluded harness workflow failures

pkg/behaviourtest/drivers/ci/githubactions/githubactions.go

@@ -27,6 +27,11 @@ const (
 
 	assertNoWorkflowChecks = 3
 	assertNoWorkflowDelay  = 10 * time.Second
+
+	// harnessWorkflowFile is the workflow filename used by the harness.
+	// Used to scope fail-fast and diagnostic queries to harness runs only,
+	// avoiding false positives from unrelated workflow failures.
+	harnessWorkflowFile = "fullsend.yaml"
 )
 
 // Driver implements ci.Driver against GitHub Actions.
@@ -431,7 +436,57 @@ func readLimited(r io.Reader, limit int64) ([]byte, error) {
 	return data, nil
 }
 
+// isTerminalFailure reports whether a workflow run conclusion represents a
+// real failure that should trigger fail-fast. Skipped and cancelled runs are
+// excluded because they are typically concurrency-group noise, not harness
+// failures.
+func isTerminalFailure(conclusion string) bool {
+	switch conclusion {
+	case "failure", "timed_out", "startup_failure":
+		return true
+	default:
+		return false
+	}
+}
+
+// listHarnessRunsAfter returns recent harness workflow runs created after the
+// given time. It queries only the harness workflow (fullsend.yaml) to avoid
+// false-positive fail-fast from unrelated workflow failures in the same repo.
+func (d *Driver) listHarnessRunsAfter(ctx context.Context, owner, repo string, after time.Time) []forge.WorkflowRun {
+	runs, err := d.Client.ListWorkflowRuns(ctx, owner, repo, harnessWorkflowFile)
+	if err != nil {
+		return nil
+	}
+	var matched []forge.WorkflowRun
+	for _, run := range runs {
+		runTime, parseErr := time.Parse(time.RFC3339, run.CreatedAt)
+		if parseErr != nil || runTime.Before(after) {
+			continue
+		}
+		matched = append(matched, run)
+	}
+	return matched
+}
+
+// formatRunDiagnostics builds a human-readable summary of recent workflow
+// runs for inclusion in error messages.
+func formatRunDiagnostics(runs []forge.WorkflowRun) string {
+	if len(runs) == 0 {
+		return "no recent workflow runs found after trigger time"
+	}
+	var b strings.Builder
+	fmt.Fprintf(&b, "recent workflow runs (%d):", len(runs))
+	for _, r := range runs {
+		fmt.Fprintf(&b, "\n  run %d: status=%s conclusion=%s url=%s",
+			r.ID, r.Status, r.Conclusion, r.HTMLURL)
+	}
+	return b.String()
+}
+
 // WaitForHarnessAgent waits for a successful harness-run workflow and artifact for agent.
+// It fails fast when a recent workflow run has a terminal failure conclusion
+// (failure, timed_out, startup_failure) and includes diagnostic information
+// about recent runs on timeout.
 func (d *Driver) WaitForHarnessAgent(ctx context.Context, owner, repo, agent string, after time.Time) (*forge.WorkflowRun, error) {
 	artifactName := "fullsend-" + agent
 	deadline := time.Now().Add(dispatchWait)
@@ -441,27 +496,38 @@ func (d *Driver) WaitForHarnessAgent(ctx context.Context, owner, repo, agent str
 			return nil, ctx.Err()
 		case <-time.After(dispatchPoll):
 		}
+
+		// Check for the success artifact.
 		arts, err := d.Client.ListRepositoryArtifacts(ctx, owner, repo, 100)
-		if err != nil {
-			continue
-		}
-		art := selectRepositoryArtifactAfter(arts, artifactName, after)
-		if art == nil {
-			continue
-		}
-		run, err := d.Client.GetWorkflowRun(ctx, owner, repo, art.WorkflowRunID)
-		if err != nil {
-			continue
-		}
-		if run.Status != "completed" {
-			continue
+		if err == nil {
+			if art := selectRepositoryArtifactAfter(arts, artifactName, after); art != nil {
+				run, err := d.Client.GetWorkflowRun(ctx, owner, repo, art.WorkflowRunID)
+				if err == nil {
+					if run.Status == "completed" {
+						if run.Conclusion == "success" {
+							return run, nil
+						}
+						return nil, fmt.Errorf("harness run for %q concluded with %q (run %d: %s)",
+							agent, run.Conclusion, run.ID, run.HTMLURL)
+					}
+				}
+			}
 		}
-		if run.Conclusion != "success" {
-			return nil, fmt.Errorf("harness run for %q concluded with %q", agent, run.Conclusion)
+
+		// Fail-fast: check recent runs for terminal failures.
+		recentRuns := d.listHarnessRunsAfter(ctx, owner, repo, after)
+		for _, r := range recentRuns {
+			if r.Status == "completed" && isTerminalFailure(r.Conclusion) {
+				return nil, fmt.Errorf("harness agent %q: workflow run %d concluded with %q before producing artifact (url=%s)",
+					agent, r.ID, r.Conclusion, r.HTMLURL)
+			}
 		}
-		return run, nil
 	}
-	return nil, fmt.Errorf("harness agent %q did not complete successfully", agent)
+
+	// Timeout: include diagnostics about recent workflow runs.
+	recentRuns := d.listHarnessRunsAfter(ctx, owner, repo, after)
+	return nil, fmt.Errorf("harness agent %q did not complete successfully; %s",
+		agent, formatRunDiagnostics(recentRuns))
 }
 
 // CountHarnessDispatches returns the number of fullsend-{agent} artifacts

pkg/behaviourtest/drivers/ci/githubactions/githubactions_test.go

@@ -271,3 +271,246 @@ func TestWaitForHarnessAgent_FromRepositoryArtifact(t *testing.T) {
 	require.NotNil(t, run)
 	assert.Equal(t, 99, run.ID)
 }
+
+func TestWaitForHarnessAgent_FailFastOnFailure(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	// No success artifact — the harness failed before uploading one.
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID:         42,
+			Status:     "completed",
+			Conclusion: "failure",
+			CreatedAt:  "2026-01-02T00:00:00Z",
+			HTMLURL:    "https://github.com/org/repo/actions/runs/42",
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.Error(t, err)
+	assert.Nil(t, run)
+	assert.Contains(t, err.Error(), "workflow run 42")
+	assert.Contains(t, err.Error(), `"failure"`)
+	assert.Contains(t, err.Error(), "https://github.com/org/repo/actions/runs/42")
+}
+
+func TestWaitForHarnessAgent_FailFastOnTimedOut(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID:         50,
+			Status:     "completed",
+			Conclusion: "timed_out",
+			CreatedAt:  "2026-01-02T00:00:00Z",
+			HTMLURL:    "https://github.com/org/repo/actions/runs/50",
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.Error(t, err)
+	assert.Nil(t, run)
+	assert.Contains(t, err.Error(), `"timed_out"`)
+}
+
+func TestWaitForHarnessAgent_FailFastOnStartupFailure(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID:         60,
+			Status:     "completed",
+			Conclusion: "startup_failure",
+			CreatedAt:  "2026-01-02T00:00:00Z",
+			HTMLURL:    "https://github.com/org/repo/actions/runs/60",
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.Error(t, err)
+	assert.Nil(t, run)
+	assert.Contains(t, err.Error(), `"startup_failure"`)
+}
+
+func TestWaitForHarnessAgent_SkippedDoesNotTriggerFailFast(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	// A skipped harness run exists but should not trigger fail-fast.
+	// The success run is keyed separately so GetWorkflowRun finds it by ID.
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID: 70, Status: "completed", Conclusion: "skipped",
+			CreatedAt: "2026-01-02T00:00:00Z",
+			HTMLURL:   "https://github.com/org/repo/actions/runs/70",
+		},
+		"org/repo/success": {
+			ID: 99, Status: "completed", Conclusion: "success", CreatedAt: "2026-01-02T00:00:00Z",
+		},
+	}
+	// Provide a success artifact so the function can succeed.
+	client.RepositoryArtifacts = map[string][]forge.RepositoryArtifact{
+		"org/repo": {
+			{
+				ID:            10,
+				Name:          "fullsend-triage",
+				CreatedAt:     "2026-01-02T00:00:00Z",
+				WorkflowRunID: 99,
+			},
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.NoError(t, err)
+	require.NotNil(t, run)
+	assert.Equal(t, 99, run.ID)
+}
+
+func TestWaitForHarnessAgent_CancelledDoesNotTriggerFailFast(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	// A cancelled harness run should not trigger fail-fast.
+	// The success run is keyed separately so GetWorkflowRun finds it by ID.
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID: 80, Status: "completed", Conclusion: "cancelled",
+			CreatedAt: "2026-01-02T00:00:00Z",
+			HTMLURL:   "https://github.com/org/repo/actions/runs/80",
+		},
+		"org/repo/success": {
+			ID: 99, Status: "completed", Conclusion: "success", CreatedAt: "2026-01-02T00:00:00Z",
+		},
+	}
+	// Provide a success artifact so the function can succeed.
+	client.RepositoryArtifacts = map[string][]forge.RepositoryArtifact{
+		"org/repo": {
+			{
+				ID:            10,
+				Name:          "fullsend-triage",
+				CreatedAt:     "2026-01-02T00:00:00Z",
+				WorkflowRunID: 99,
+			},
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.NoError(t, err)
+	require.NotNil(t, run)
+	assert.Equal(t, 99, run.ID)
+}
+
+func TestWaitForHarnessAgent_IgnoresRunsBeforeTriggerTime(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	// Failed harness run is before the trigger time — should not trigger fail-fast.
+	// The success run is keyed separately so GetWorkflowRun finds it by ID.
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID: 90, Status: "completed", Conclusion: "failure",
+			CreatedAt: "2026-01-02T00:00:00Z",
+			HTMLURL:   "https://github.com/org/repo/actions/runs/90",
+		},
+		"org/repo/success": {
+			ID: 99, Status: "completed", Conclusion: "success", CreatedAt: "2026-07-01T00:00:00Z",
+		},
+	}
+	// Provide a success artifact so the function can succeed.
+	client.RepositoryArtifacts = map[string][]forge.RepositoryArtifact{
+		"org/repo": {
+			{
+				ID:            10,
+				Name:          "fullsend-triage",
+				CreatedAt:     "2026-07-01T00:00:00Z",
+				WorkflowRunID: 99,
+			},
+		},
+	}
+
+	d := &Driver{Client: client}
+	run, err := d.WaitForHarnessAgent(context.Background(), "org", "repo", "triage", after)
+	require.NoError(t, err)
+	require.NotNil(t, run)
+	assert.Equal(t, 99, run.ID)
+}
+
+func TestWaitForHarnessAgent_TimeoutIncludesDiagnostics(t *testing.T) {
+	t.Parallel()
+
+	after := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	client := forge.NewFakeClient()
+	// No artifacts, no terminal failures — will time out.
+	// Use in-progress harness run to avoid fail-fast.
+	client.WorkflowRuns = map[string]*forge.WorkflowRun{
+		"org/repo/fullsend.yaml": {
+			ID: 100, Status: "in_progress",
+			CreatedAt: "2026-01-02T00:00:00Z",
+			HTMLURL:   "https://github.com/org/repo/actions/runs/100",
+		},
+	}
+
+	d := &Driver{Client: client}
+	// Use a cancelled context to avoid waiting the full deadline.
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := d.WaitForHarnessAgent(ctx, "org", "repo", "triage", after)
+	require.Error(t, err)
+	// Should return context error, not a timeout with diagnostics,
+	// because the context was cancelled before the deadline.
+	assert.ErrorIs(t, err, context.Canceled)
+}
+
+func TestIsTerminalFailure(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		conclusion string
+		want       bool
+	}{
+		{"failure", true},
+		{"timed_out", true},
+		{"startup_failure", true},
+		{"skipped", false},
+		{"cancelled", false},
+		{"success", false},
+		{"", false},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.want, isTerminalFailure(tt.conclusion),
+			"isTerminalFailure(%q)", tt.conclusion)
+	}
+}
+
+func TestFormatRunDiagnostics_Empty(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, "no recent workflow runs found after trigger time",
+		formatRunDiagnostics(nil))
+}
+
+func TestFormatRunDiagnostics_WithRuns(t *testing.T) {
+	t.Parallel()
+
+	runs := []forge.WorkflowRun{
+		{ID: 1, Status: "completed", Conclusion: "failure", HTMLURL: "https://example.com/1"},
+		{ID: 2, Status: "in_progress", HTMLURL: "https://example.com/2"},
+	}
+	got := formatRunDiagnostics(runs)
+	assert.Contains(t, got, "recent workflow runs (2)")
+	assert.Contains(t, got, "run 1: status=completed conclusion=failure")
+	assert.Contains(t, got, "run 2: status=in_progress")
+}