diff --git a/.circleci/config.yml b/.circleci/config.yml index 557385d223a..4a58863e537 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ parameters: default: false flake-shake-iterations: type: integer - default: 100 + default: 200 flake-shake-workers: type: integer default: 10 @@ -1759,8 +1759,8 @@ jobs: --days 3 \ --gate flake-shake \ --min-runs 300 \ - --max-failure-rate 0.01 \ - --min-age-days 3 \ + --max-failure-rate 0.0 \ + --min-age-days 2 \ --dry-run=false \ --require-clean-24h \ --out ./final-promotion \ diff --git a/op-acceptance-tests/cmd/flake-shake-promoter/main.go b/op-acceptance-tests/cmd/flake-shake-promoter/main.go index 77e57769a94..7d7d98315f8 100644 --- a/op-acceptance-tests/cmd/flake-shake-promoter/main.go +++ b/op-acceptance-tests/cmd/flake-shake-promoter/main.go @@ -53,6 +53,7 @@ type jobList struct { type job struct { Name string `json:"name"` JobNumber int `json:"job_number"` + WebURL string `json:"web_url"` } type artifactsList struct { @@ -107,6 +108,7 @@ type testEntry struct { Package string `yaml:"package"` Timeout string `yaml:"timeout,omitempty"` Metadata map[string]interface{} `yaml:"metadata,omitempty"` + Owner string `yaml:"owner,omitempty"` } // Aggregated per test across days @@ -129,6 +131,7 @@ type promoteCandidate struct { PassRate float64 `json:"pass_rate"` Timeout string `json:"timeout"` FirstSeenDay string `json:"first_seen_day"` + Owner string `json:"owner,omitempty"` } // Map tests in flake-shake: key -> (timeout, name) @@ -136,6 +139,7 @@ type testInfo struct { Timeout string Name string Meta map[string]interface{} + Owner string GateIndex int TestIndex int } @@ -264,7 +268,14 @@ func main() { title := "chore(op-acceptance-tests): flake-shake; test promotions" var body bytes.Buffer body.WriteString("## 🤖 Automated Flake-Shake Test Promotion\n\n") + + // Attempt to resolve the CircleCI report job web URL for artifacts page + reportArtifactsURL := resolveReportArtifactsURL(opts, ctx) + body.WriteString(fmt.Sprintf("Promoting %d test(s) from gate `"+opts.gateID+"` based on stability criteria.\n\n", len(candidates))) + if reportArtifactsURL != "" { + body.WriteString(fmt.Sprintf("Artifacts: %s\n\n", reportArtifactsURL)) + } body.WriteString("### Tests Being Promoted\n\n") body.WriteString("| Test | Package | Total Runs | Pass Rate |\n|---|---|---:|---:|\n") for _, c := range candidates { @@ -482,7 +493,14 @@ func buildFlakeTests(cfg *acceptanceYAML, gateID, yamlPath string) (map[string]t flakeTests := map[string]testInfo{} for ti, t := range flakeGate.Tests { key := keyFor(t.Package, t.Name) - flakeTests[key] = testInfo{Timeout: t.Timeout, Name: t.Name, Meta: t.Metadata, GateIndex: indexOfGate(cfg, gateID), TestIndex: ti} + // Prefer explicit YAML field owner; fallback to metadata.owner + owner := t.Owner + if owner == "" && t.Metadata != nil { + if v, ok := t.Metadata["owner"]; ok { + owner = fmt.Sprintf("%v", v) + } + } + flakeTests[key] = testInfo{Timeout: t.Timeout, Name: t.Name, Meta: t.Metadata, Owner: owner, GateIndex: indexOfGate(cfg, gateID), TestIndex: ti} } return flakeTests, flakeGate, gateIndex } @@ -560,6 +578,14 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t if totalRuns > 0 { passRate = float64(totalPasses) / float64(totalRuns) } + owner := info.Owner + if owner == "" { + if info.Meta != nil { + if v, ok := info.Meta["owner"]; ok { + owner = fmt.Sprintf("%v", v) + } + } + } candidates = append(candidates, promoteCandidate{ Package: pkg, TestName: "", @@ -567,6 +593,7 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t PassRate: passRate * 100.0, Timeout: info.Timeout, FirstSeenDay: earliest, + Owner: owner, }) } for key, s := range agg { @@ -616,6 +643,14 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t if s.TotalRuns > 0 { passRate = float64(s.Passes) / float64(s.TotalRuns) } + owner := info.Owner + if owner == "" { + if info.Meta != nil { + if v, ok := info.Meta["owner"]; ok { + owner = fmt.Sprintf("%v", v) + } + } + } candidates = append(candidates, promoteCandidate{ Package: s.Package, TestName: s.TestName, @@ -623,6 +658,7 @@ func selectPromotionCandidates(agg map[string]*aggStats, flakeTests map[string]t PassRate: passRate * 100.0, Timeout: info.Timeout, FirstSeenDay: s.FirstSeenDay, + Owner: owner, }) } return candidates, reasons @@ -828,6 +864,71 @@ func listJobs(ctx *apiCtx, workflowID string) (jobList, error) { return jl, nil } +// resolveReportArtifactsURL attempts to find the web URL to the report job's artifacts page +// by scanning recent pipelines/workflows for the configured workflow/report job names. +// Returns an empty string if not found. +func resolveReportArtifactsURL(opts promoterOpts, ctx *apiCtx) string { + // Scan the latest pipelines on the given branch; reuse collectReports traversal but short-circuit on first match + basePipelines := fmt.Sprintf("https://circleci.com/api/v2/project/gh/%s/%s/pipeline?branch=%s", url.PathEscape(opts.org), url.PathEscape(opts.repo), url.QueryEscape(opts.branch)) + pageURL := basePipelines + now := time.Now().UTC() + since := now.AddDate(0, 0, -opts.daysBack) + for { + pl, nextToken, err := getPipelinesPage(ctx, pageURL) + if err != nil { + return "" + } + for _, p := range pl.Items { + if p.CreatedAt.Before(since) { + return "" + } + wfl, err := listWorkflows(ctx, p.ID) + if err != nil { + return "" + } + for _, w := range wfl.Items { + if w.Name != opts.workflowName { + continue + } + jl, err := listJobs(ctx, w.ID) + if err != nil { + return "" + } + for _, j := range jl.Items { + if j.Name != opts.reportJobName { + continue + } + if j.WebURL != "" { + url := j.WebURL + if !strings.Contains(url, "/artifacts") { + if strings.HasSuffix(url, "/") { + url = url + "artifacts" + } else { + url = url + "/artifacts" + } + } + return url + } + // Fallback: build URL from job number if web_url missing + if j.JobNumber != 0 { + url := fmt.Sprintf("https://app.circleci.com/pipelines/github/%s/%s?branch=%s", opts.org, opts.repo, url.QueryEscape(opts.branch)) + _ = url // keep for future; better to use job-specific URL + // More specific URL pattern commonly used in UI includes workflow id; not available here. + // As a fallback, return the legacy build URL on circleci.com if org/repo/job present. + legacy := fmt.Sprintf("https://circleci.com/gh/%s/%s/%d", opts.org, opts.repo, j.JobNumber) + return legacy + "/artifacts" + } + } + } + } + if nextToken == "" { + break + } + pageURL = basePipelines + "&page-token=" + url.QueryEscape(nextToken) + } + return "" +} + func listArtifacts(ctx *apiCtx, org, repo string, jobNumber int, verbose bool) (artifactsList, error) { artsURL := fmt.Sprintf("https://circleci.com/api/v2/project/gh/%s/%s/%d/artifacts", url.PathEscape(org), url.PathEscape(repo), jobNumber) var al artifactsList diff --git a/op-acceptance-tests/scripts/ci_flake_shake_prepare_slack.sh b/op-acceptance-tests/scripts/ci_flake_shake_prepare_slack.sh index 33acc57194a..b05bcb1b6fd 100644 --- a/op-acceptance-tests/scripts/ci_flake_shake_prepare_slack.sh +++ b/op-acceptance-tests/scripts/ci_flake_shake_prepare_slack.sh @@ -30,15 +30,43 @@ PROMO_JSON=${1:-./final-promotion/promotion-ready.json} SLACK_BLOCKS="[]" if [ -f "$PROMO_JSON" ]; then + # Determine URL to the flake-shake report job (artifacts live there), + # falling back to the current job URL if not resolvable. + REPORT_JOB_URL="${CIRCLE_BUILD_URL:-}" + if [ -n "${CIRCLE_WORKFLOW_ID:-}" ] && [ -n "${CIRCLE_API_TOKEN:-}" ]; then + JOBS_JSON=$(curl -sfL -H "Circle-Token: ${CIRCLE_API_TOKEN}" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/jobs?limit=100" || true) + if [ -n "${JOBS_JSON:-}" ]; then + # Prefer web_url if available; otherwise construct URL from job_number + REPORT_WEB_URL=$(printf '%s' "$JOBS_JSON" | jq -r '.items[] | select(.name=="op-acceptance-tests-flake-shake-report") | .web_url // empty' | head -n1) + if [ -n "$REPORT_WEB_URL" ] && [ "$REPORT_WEB_URL" != "null" ]; then + REPORT_JOB_URL="$REPORT_WEB_URL" + else + REPORT_JOB_NUM=$(printf '%s' "$JOBS_JSON" | jq -r '.items[] | select(.name=="op-acceptance-tests-flake-shake-report") | .job_number // empty' | head -n1) + if [ -n "$REPORT_JOB_NUM" ] && [ "$REPORT_JOB_NUM" != "null" ] && [ -n "${CIRCLE_PROJECT_USERNAME:-}" ] && [ -n "${CIRCLE_PROJECT_REPONAME:-}" ]; then + REPORT_JOB_URL="https://circleci.com/gh/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${REPORT_JOB_NUM}" + fi + fi + fi + fi + + # Ensure URL points to the artifacts page of the report job + REPORT_ARTIFACTS_URL="$REPORT_JOB_URL" + if [ -n "$REPORT_ARTIFACTS_URL" ]; then + if ! printf '%s' "$REPORT_ARTIFACTS_URL" | grep -q '/artifacts\($\|[?#]\)'; then + REPORT_ARTIFACTS_URL="${REPORT_ARTIFACTS_URL%/}/artifacts" + fi + fi + # Build Block Kit blocks (header + link + divider + per-candidate sections) SLACK_BLOCKS=$(jq -c \ - --arg url "${CIRCLE_BUILD_URL:-}" \ + --arg url "${REPORT_ARTIFACTS_URL}" \ --slurpfile meta "${PROMO_JSON%/*}/metadata.json" ' def name_or_pkg(t): (if ((t.test_name|tostring)|length) == 0 then "(package)" else t.test_name end); + def owner_or_unknown(t): (if ((t.owner|tostring)|length) == 0 then "unknown" else t.owner end); def testblocks(t): [ {"type":"section","fields":[ {"type":"mrkdwn","text":"*Test:*\n\(name_or_pkg(t))"}, - {"type":"mrkdwn","text":"*Package:*\n\(t.package)"} + {"type":"mrkdwn","text":"*Owner:*\n\(owner_or_unknown(t))"} ]}, {"type":"section","fields":[ {"type":"mrkdwn","text":"*Runs:*\n\(t.total_runs)"},