feat(op-acceptance-tests): flake-shake; tweaks

.circleci/config.yml

@@ -79,7 +79,7 @@ parameters:
     default: false
   flake-shake-iterations:
     type: integer
-    default: 100
+    default: 200
   flake-shake-workers:
     type: integer
     default: 10
@@ -1759,8 +1759,8 @@ jobs:
               --days 3 \
               --gate flake-shake \
               --min-runs 300 \
-              --max-failure-rate 0.01 \
-              --min-age-days 3 \
+              --max-failure-rate 0.0 \
+              --min-age-days 2 \
               --dry-run=false \
               --require-clean-24h \
               --out ./final-promotion \

op-acceptance-tests/cmd/flake-shake-promoter/main.go

@@ -53,6 +53,7 @@ type jobList struct {
 type job struct {
 	Name      string `json:"name"`
 	JobNumber int    `json:"job_number"`
+	WebURL    string `json:"web_url"`
 }
 
 type artifactsList struct {
@@ -107,6 +108,7 @@ type testEntry struct {
 	Package  string                 `yaml:"package"`
 	Timeout  string                 `yaml:"timeout,omitempty"`
 	Metadata map[string]interface{} `yaml:"metadata,omitempty"`
+	Owner    string                 `yaml:"owner,omitempty"`
 }
 
 // Aggregated per test across days
@@ -129,13 +131,15 @@ type promoteCandidate struct {
 	PassRate     float64 `json:"pass_rate"`
 	Timeout      string  `json:"timeout"`
 	FirstSeenDay string  `json:"first_seen_day"`
+	Owner        string  `json:"owner,omitempty"`
 }
 
 // Map tests in flake-shake: key -> (timeout, name)
 type testInfo struct {
 	Timeout   string
 	Name      string
 	Meta      map[string]interface{}
+	Owner     string
 	GateIndex int
 	TestIndex int
 }
@@ -264,7 +268,14 @@ func main() {
 	title := "chore(op-acceptance-tests): flake-shake; test promotions"
 	var body bytes.Buffer
 	body.WriteString("## 🤖 Automated Flake-Shake Test Promotion\n\n")
+
+	// Attempt to resolve the CircleCI report job web URL for artifacts page
+	reportArtifactsURL := resolveReportArtifactsURL(opts, ctx)
+
 	body.WriteString(fmt.Sprintf("Promoting %d test(s) from gate `"+opts.gateID+"` based on stability criteria.\n\n", len(candidates)))
+	if reportArtifactsURL != "" {
+		body.WriteString(fmt.Sprintf("Artifacts: %s\n\n", reportArtifactsURL))
+	}
 	body.WriteString("### Tests Being Promoted\n\n")
 	body.WriteString("| Test | Package | Total Runs | Pass Rate |\n|---|---|---:|---:|\n")
 	for _, c := range candidates {
@@ -482,7 +493,14 @@ func buildFlakeTests(cfg *acceptanceYAML, gateID, yamlPath string) (map[string]t
 	flakeTests := map[string]testInfo{}
 	for ti, t := range flakeGate.Tests {
 		key := keyFor(t.Package, t.Name)
-		flakeTests[key] = testInfo{Timeout: t.Timeout, Name: t.Name, Meta: t.Metadata, GateIndex: indexOfGate(cfg, gateID), TestIndex: ti}
+		// Prefer explicit YAML field owner; fallback to metadata.owner
+		owner := t.Owner
+		if owner == "" && t.Metadata != nil {
+			if v, ok := t.Metadata["owner"]; ok {
+				owner = fmt.Sprintf("%v", v)
+			}
+		}
+		flakeTests[key] = testInfo{Timeout: t.Timeout, Name: t.Name, Meta: t.Metadata, Owner: owner, GateIndex: indexOfGate(cfg, gateID), TestIndex: ti}
 	}
 	return flakeTests, flakeGate, gateIndex
 }
@@ -560,13 +578,22 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t
 		if totalRuns > 0 {
 			passRate = float64(totalPasses) / float64(totalRuns)
 		}
+		owner := info.Owner
+		if owner == "" {
+			if info.Meta != nil {
+				if v, ok := info.Meta["owner"]; ok {
+					owner = fmt.Sprintf("%v", v)
+				}
+			}
+		}
 		candidates = append(candidates, promoteCandidate{
 			Package:      pkg,
 			TestName:     "",
 			TotalRuns:    totalRuns,
 			PassRate:     passRate * 100.0,
 			Timeout:      info.Timeout,
 			FirstSeenDay: earliest,
+			Owner:        owner,
 		})
 	}
 	for key, s := range agg {
@@ -616,13 +643,22 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t
 		if s.TotalRuns > 0 {
 			passRate = float64(s.Passes) / float64(s.TotalRuns)
 		}
+		owner := info.Owner
+		if owner == "" {
+			if info.Meta != nil {
+				if v, ok := info.Meta["owner"]; ok {
+					owner = fmt.Sprintf("%v", v)
+				}
+			}
+		}
 		candidates = append(candidates, promoteCandidate{
 			Package:      s.Package,
 			TestName:     s.TestName,
 			TotalRuns:    s.TotalRuns,
 			PassRate:     passRate * 100.0,
 			Timeout:      info.Timeout,
 			FirstSeenDay: s.FirstSeenDay,
+			Owner:        owner,
 		})
 	}
 	return candidates, reasons
@@ -828,6 +864,71 @@ func listJobs(ctx *apiCtx, workflowID string) (jobList, error) {
 	return jl, nil
 }
 
+// resolveReportArtifactsURL attempts to find the web URL to the report job's artifacts page
+// by scanning recent pipelines/workflows for the configured workflow/report job names.
+// Returns an empty string if not found.
+func resolveReportArtifactsURL(opts promoterOpts, ctx *apiCtx) string {
+	// Scan the latest pipelines on the given branch; reuse collectReports traversal but short-circuit on first match
+	basePipelines := fmt.Sprintf("https://circleci.com/api/v2/project/gh/%s/%s/pipeline?branch=%s", url.PathEscape(opts.org), url.PathEscape(opts.repo), url.QueryEscape(opts.branch))
+	pageURL := basePipelines
+	now := time.Now().UTC()
+	since := now.AddDate(0, 0, -opts.daysBack)
+	for {
+		pl, nextToken, err := getPipelinesPage(ctx, pageURL)
+		if err != nil {
+			return ""
+		}
+		for _, p := range pl.Items {
+			if p.CreatedAt.Before(since) {
+				return ""
+			}
+			wfl, err := listWorkflows(ctx, p.ID)
+			if err != nil {
+				return ""
+			}
+			for _, w := range wfl.Items {
+				if w.Name != opts.workflowName {
+					continue
+				}
+				jl, err := listJobs(ctx, w.ID)
+				if err != nil {
+					return ""
+				}
+				for _, j := range jl.Items {
+					if j.Name != opts.reportJobName {
+						continue
+					}
+					if j.WebURL != "" {
+						url := j.WebURL
+						if !strings.Contains(url, "/artifacts") {
+							if strings.HasSuffix(url, "/") {
+								url = url + "artifacts"
+							} else {
+								url = url + "/artifacts"
+							}
+						}
+						return url
+					}
+					// Fallback: build URL from job number if web_url missing
+					if j.JobNumber != 0 {
+						url := fmt.Sprintf("https://app.circleci.com/pipelines/github/%s/%s?branch=%s", opts.org, opts.repo, url.QueryEscape(opts.branch))
+						_ = url // keep for future; better to use job-specific URL
+						// More specific URL pattern commonly used in UI includes workflow id; not available here.
+						// As a fallback, return the legacy build URL on circleci.com if org/repo/job present.
+						legacy := fmt.Sprintf("https://circleci.com/gh/%s/%s/%d", opts.org, opts.repo, j.JobNumber)
+						return legacy + "/artifacts"
+					}
+				}
+			}
+		}
+		if nextToken == "" {
+			break
+		}
+		pageURL = basePipelines + "&page-token=" + url.QueryEscape(nextToken)
+	}
+	return ""
+}
+
 func listArtifacts(ctx *apiCtx, org, repo string, jobNumber int, verbose bool) (artifactsList, error) {
 	artsURL := fmt.Sprintf("https://circleci.com/api/v2/project/gh/%s/%s/%d/artifacts", url.PathEscape(org), url.PathEscape(repo), jobNumber)
 	var al artifactsList

op-acceptance-tests/scripts/ci_flake_shake_prepare_slack.sh

@@ -30,15 +30,43 @@ PROMO_JSON=${1:-./final-promotion/promotion-ready.json}
 
 SLACK_BLOCKS="[]"
 if [ -f "$PROMO_JSON" ]; then
+  # Determine URL to the flake-shake report job (artifacts live there),
+  # falling back to the current job URL if not resolvable.
+  REPORT_JOB_URL="${CIRCLE_BUILD_URL:-}"
+  if [ -n "${CIRCLE_WORKFLOW_ID:-}" ] && [ -n "${CIRCLE_API_TOKEN:-}" ]; then
+    JOBS_JSON=$(curl -sfL -H "Circle-Token: ${CIRCLE_API_TOKEN}" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/jobs?limit=100" || true)
+    if [ -n "${JOBS_JSON:-}" ]; then
+      # Prefer web_url if available; otherwise construct URL from job_number
+      REPORT_WEB_URL=$(printf '%s' "$JOBS_JSON" | jq -r '.items[] | select(.name=="op-acceptance-tests-flake-shake-report") | .web_url // empty' | head -n1)
+      if [ -n "$REPORT_WEB_URL" ] && [ "$REPORT_WEB_URL" != "null" ]; then
+        REPORT_JOB_URL="$REPORT_WEB_URL"
+      else
+        REPORT_JOB_NUM=$(printf '%s' "$JOBS_JSON" | jq -r '.items[] | select(.name=="op-acceptance-tests-flake-shake-report") | .job_number // empty' | head -n1)
+        if [ -n "$REPORT_JOB_NUM" ] && [ "$REPORT_JOB_NUM" != "null" ] && [ -n "${CIRCLE_PROJECT_USERNAME:-}" ] && [ -n "${CIRCLE_PROJECT_REPONAME:-}" ]; then
+          REPORT_JOB_URL="https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${REPORT_JOB_NUM}"
+        fi
+      fi
+    fi
+  fi
+
+  # Ensure URL points to the artifacts page of the report job
+  REPORT_ARTIFACTS_URL="$REPORT_JOB_URL"
+  if [ -n "$REPORT_ARTIFACTS_URL" ]; then
+    if ! printf '%s' "$REPORT_ARTIFACTS_URL" | grep -q '/artifacts\($\|[?#]\)'; then
+      REPORT_ARTIFACTS_URL="${REPORT_ARTIFACTS_URL%/}/artifacts"
+    fi
+  fi
+
   # Build Block Kit blocks (header + link + divider + per-candidate sections)
   SLACK_BLOCKS=$(jq -c \
-    --arg url "${CIRCLE_BUILD_URL:-}" \
+    --arg url "${REPORT_ARTIFACTS_URL}" \
     --slurpfile meta "${PROMO_JSON%/*}/metadata.json" '
     def name_or_pkg(t): (if ((t.test_name|tostring)|length) == 0 then "(package)" else t.test_name end);
+    def owner_or_unknown(t): (if ((t.owner|tostring)|length) == 0 then "unknown" else t.owner end);
     def testblocks(t): [
       {"type":"section","fields":[
         {"type":"mrkdwn","text":"*Test:*\n\(name_or_pkg(t))"},
-        {"type":"mrkdwn","text":"*Package:*\n\(t.package)"}
+        {"type":"mrkdwn","text":"*Owner:*\n\(owner_or_unknown(t))"}
       ]},
       {"type":"section","fields":[
         {"type":"mrkdwn","text":"*Runs:*\n\(t.total_runs)"},