-
Notifications
You must be signed in to change notification settings - Fork 10
[auto] #29 automation: issue claim lease + auto-release on failed/aborted runs #39
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -106,7 +106,14 @@ id = "ensure_docs_release" | |||||||||||||||||||||||||
| type = "ensure_running" | ||||||||||||||||||||||||||
| agent = "docs-release" | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| # 2) Intake and branch setup | ||||||||||||||||||||||||||
| # 2) Sweep stale claims before selecting a new issue | ||||||||||||||||||||||||||
| [[workflow.step]] | ||||||||||||||||||||||||||
| id = "sweep_stale_claims" | ||||||||||||||||||||||||||
| type = "command" | ||||||||||||||||||||||||||
| run = "scripts/automation/sweep_stale_claims.sh .tutti/state/claims" | ||||||||||||||||||||||||||
| fail_mode = "open" | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| # 3) Intake and branch setup | ||||||||||||||||||||||||||
| [[workflow.step]] | ||||||||||||||||||||||||||
| id = "select_issue" | ||||||||||||||||||||||||||
| type = "command" | ||||||||||||||||||||||||||
|
|
@@ -222,3 +229,10 @@ inject_files = [".tutti/state/auto/selected_issue.json", ".tutti/state/auto/bran | |||||||||||||||||||||||||
| wait_for_idle = true | ||||||||||||||||||||||||||
| wait_timeout_secs = 900 | ||||||||||||||||||||||||||
| text = "Summarize final readiness, residual risks, and merge recommendation." | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| # 10) Release claim after successful completion | ||||||||||||||||||||||||||
| [[workflow.step]] | ||||||||||||||||||||||||||
| id = "release_claim" | ||||||||||||||||||||||||||
| type = "command" | ||||||||||||||||||||||||||
| run = "scripts/automation/release_claim.sh .tutti/state/auto/selected_issue.json 'workflow completed successfully'" | ||||||||||||||||||||||||||
| fail_mode = "open" | ||||||||||||||||||||||||||
|
Comment on lines
+233
to
+238
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make successful-run claim release blocking. This is the only success-path cleanup in 💡 Minimal config fix [[workflow.step]]
id = "release_claim"
type = "command"
run = "scripts/automation/release_claim.sh .tutti/state/auto/selected_issue.json 'workflow completed successfully'"
-fail_mode = "open"
+fail_mode = "closed"📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents |
||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| # Release the automation-claimed label from the selected issue. | ||
| # Usage: release_claim.sh [selected_issue_json] [reason] | ||
|
|
||
| ISSUE_FILE="${1:-.tutti/state/auto/selected_issue.json}" | ||
| REASON="${2:-workflow completed}" | ||
|
|
||
| if [ ! -f "$ISSUE_FILE" ]; then | ||
| echo "No selected issue file at $ISSUE_FILE — nothing to release." >&2 | ||
| exit 0 | ||
| fi | ||
|
|
||
| REPO="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}" | ||
|
|
||
| ISSUE_NUM=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['issue_number'])" "$ISSUE_FILE") | ||
|
|
||
| if [ -z "$ISSUE_NUM" ]; then | ||
| echo "Could not read issue_number from $ISSUE_FILE" >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Remove the automation-claimed label (tolerate it already being absent). | ||
| gh issue edit "$ISSUE_NUM" --repo "$REPO" --remove-label "automation-claimed" 2>/dev/null || true | ||
|
|
||
| # Extract run_id from claim metadata if available. | ||
| RUN_ID=$(python3 -c " | ||
| import json,sys | ||
| d = json.load(open(sys.argv[1])) | ||
| print(d.get('claim', {}).get('run_id', 'unknown')) | ||
| " "$ISSUE_FILE" 2>/dev/null || echo "unknown") | ||
|
|
||
| # Post audit comment. | ||
| gh issue comment "$ISSUE_NUM" --repo "$REPO" \ | ||
| --body "🤖 **Claim released** — reason: ${REASON} (run \`${RUN_ID}\`)" \ | ||
| >/dev/null 2>&1 || true | ||
|
|
||
| # Remove claim state file. | ||
| CLAIM_FILE=".tutti/state/claims/${ISSUE_NUM}.json" | ||
| rm -f "$CLAIM_FILE" 2>/dev/null || true | ||
|
Comment on lines
+24
to
+41
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Don't drop the local lease unless the GitHub label was actually removed. This swallows every 💡 Suggested hardening-# Remove the automation-claimed label (tolerate it already being absent).
-gh issue edit "$ISSUE_NUM" --repo "$REPO" --remove-label "automation-claimed" 2>/dev/null || true
+# Remove the automation-claimed label, but only tolerate "already absent".
+REMOVE_ERR=""
+if ! REMOVE_ERR=$(gh issue edit "$ISSUE_NUM" --repo "$REPO" --remove-label "automation-claimed" 2>&1); then
+ case "$REMOVE_ERR" in
+ *"not found"*|*"does not have"*)
+ ;;
+ *)
+ echo "Failed to remove automation-claimed from issue #${ISSUE_NUM}: $REMOVE_ERR" >&2
+ exit 1
+ ;;
+ esac
+fi
@@
-# Remove claim state file.
-CLAIM_FILE=".tutti/state/claims/${ISSUE_NUM}.json"
-rm -f "$CLAIM_FILE" 2>/dev/null || true
+# Remove claim state file only after label removal succeeded.
+CLAIM_FILE=".tutti/state/claims/${ISSUE_NUM}.json"
+rm -f "$CLAIM_FILE"🤖 Prompt for AI Agents |
||
|
|
||
| echo "Released claim on issue #${ISSUE_NUM}: ${REASON}" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -41,18 +41,58 @@ PY | |
|
|
||
| gh issue edit "$ISSUE_NUM" --repo "$REPO" --add-label "automation-claimed" >/dev/null | ||
|
|
||
| # Generate a unique run ID for claim tracking. | ||
| RUN_ID="${GITHUB_RUN_ID:-local-$(date +%s)-$$}" | ||
| LEASE_TTL="${CLAIM_LEASE_TTL:-1800}" | ||
|
|
||
| ISSUE_DETAILS=$(gh issue view "$ISSUE_NUM" --repo "$REPO" --json body) | ||
| python3 - "$OUT_FILE" "$ISSUE_DETAILS" <<'PY' | ||
| python3 - "$OUT_FILE" "$ISSUE_DETAILS" "$RUN_ID" "$LEASE_TTL" "$REPO" <<'PY' | ||
| import json,sys | ||
| from datetime import datetime, timezone | ||
| out, details_raw = sys.argv[1], sys.argv[2] | ||
| run_id, lease_ttl, repo = sys.argv[3], int(sys.argv[4]), sys.argv[5] | ||
| with open(f"{out}.tmp", "r", encoding="utf-8") as f: | ||
| payload = json.load(f) | ||
| details = json.loads(details_raw or "{}") | ||
| payload["body"] = (details.get("body") or "").strip() | ||
| # Embed claim metadata into the selected issue state. | ||
| now = datetime.now(timezone.utc).isoformat() | ||
| payload["claim"] = { | ||
| "run_id": run_id, | ||
| "claimed_at": now, | ||
| "renewed_at": now, | ||
| "lease_ttl_secs": lease_ttl, | ||
| "repo": repo, | ||
| } | ||
| with open(f"{out}.tmp", "w", encoding="utf-8") as f: | ||
| json.dump(payload, f, indent=2) | ||
| PY | ||
|
|
||
| mv "${OUT_FILE}.tmp" "$OUT_FILE" | ||
|
|
||
| # Persist claim lease for the Rust-side sweeper / auto-release. | ||
| CLAIMS_DIR="$(dirname "$OUT_FILE")/../../state/claims" | ||
| mkdir -p "$CLAIMS_DIR" | ||
|
Comment on lines
+73
to
+75
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Write the lease to the canonical claims directory.
💡 Minimal fix-CLAIMS_DIR="$(dirname "$OUT_FILE")/../../state/claims"
+CLAIMS_DIR=".tutti/state/claims"🤖 Prompt for AI Agents |
||
| python3 - "$CLAIMS_DIR" "$ISSUE_NUM" "$RUN_ID" "$LEASE_TTL" "$REPO" <<'PY' | ||
| import json,sys,os | ||
| from datetime import datetime, timezone | ||
| claims_dir, issue_num = sys.argv[1], sys.argv[2] | ||
| run_id, lease_ttl, repo = sys.argv[3], int(sys.argv[4]), sys.argv[5] | ||
| now = datetime.now(timezone.utc).isoformat() | ||
| lease = { | ||
| "issue_number": int(issue_num), | ||
| "repo": repo, | ||
| "run_id": run_id, | ||
| "claimed_at": now, | ||
| "renewed_at": now, | ||
| "lease_ttl_secs": lease_ttl, | ||
| } | ||
| path = os.path.join(claims_dir, f"{issue_num}.json") | ||
| with open(path, "w", encoding="utf-8") as f: | ||
| json.dump(lease, f, indent=2) | ||
| PY | ||
|
|
||
| # Post audit comment on the issue. | ||
| gh issue comment "$ISSUE_NUM" --repo "$REPO" --body "🤖 **Claim acquired** by run \`$RUN_ID\` — lease ${LEASE_TTL}s" >/dev/null 2>&1 || true | ||
|
|
||
| echo "$OUT_FILE" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| # Sweep stale claim leases whose TTL has expired. | ||
| # Usage: sweep_stale_claims.sh [claims_dir] | ||
|
|
||
| CLAIMS_DIR="${1:-.tutti/state/claims}" | ||
|
|
||
| if [ ! -d "$CLAIMS_DIR" ]; then | ||
| echo "No claims directory at $CLAIMS_DIR — nothing to sweep." | ||
| exit 0 | ||
| fi | ||
|
|
||
| REPO="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}" | ||
|
|
||
| python3 - "$CLAIMS_DIR" "$REPO" <<'PY' | ||
| import json, sys, os, subprocess | ||
| from datetime import datetime, timezone | ||
|
|
||
| claims_dir, repo = sys.argv[1], sys.argv[2] | ||
| released = 0 | ||
|
|
||
| for fname in os.listdir(claims_dir): | ||
| if not fname.endswith(".json"): | ||
| continue | ||
| path = os.path.join(claims_dir, fname) | ||
| try: | ||
| with open(path) as f: | ||
| lease = json.load(f) | ||
| except (json.JSONDecodeError, OSError): | ||
| continue | ||
|
|
||
| renewed_at = datetime.fromisoformat(lease["renewed_at"].replace("Z", "+00:00")) | ||
| ttl = lease.get("lease_ttl_secs", 1800) | ||
| now = datetime.now(timezone.utc) | ||
| elapsed = (now - renewed_at).total_seconds() | ||
|
|
||
| if elapsed <= ttl: | ||
| remaining = int(ttl - elapsed) | ||
| print(f" active: issue #{lease['issue_number']} (run={lease['run_id']}, {remaining}s remaining)") | ||
| continue | ||
|
|
||
| issue_num = lease["issue_number"] | ||
| run_id = lease.get("run_id", "unknown") | ||
| expired_ago = int(elapsed - ttl) | ||
| print(f" stale: issue #{issue_num} (run={run_id}, expired {expired_ago}s ago) — releasing") | ||
|
|
||
| # Remove label. | ||
| subprocess.run( | ||
| ["gh", "issue", "edit", str(issue_num), "--repo", repo, "--remove-label", "automation-claimed"], | ||
| capture_output=True, | ||
| ) | ||
| # Post audit comment. | ||
| subprocess.run( | ||
| ["gh", "issue", "comment", str(issue_num), "--repo", repo, | ||
| "--body", f"🤖 **Claim released** — reason: lease expired (sweeper, run `{run_id}`)"], | ||
| capture_output=True, | ||
| ) | ||
| # Remove claim file. | ||
| os.remove(path) | ||
| released += 1 | ||
|
Comment on lines
+48
to
+61
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Only delete a stale lease after the GitHub release succeeds. Both 🤖 Prompt for AI Agents |
||
|
|
||
| if released == 0: | ||
| print("sweep: no stale claims found") | ||
| else: | ||
| print(f"sweep: released {released} stale claim(s)") | ||
| PY | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| use crate::claim; | ||
| use crate::config::{ | ||
| HookConfig, HookEvent, HookWorkflowSource, PermissionsConfig, ResilienceConfig, TuttiConfig, | ||
| WorkflowCommandCwd, WorkflowConfig, WorkflowFailMode, WorkflowStepConfig, | ||
|
|
@@ -3062,6 +3063,20 @@ pub fn execute_workflow_with_hooks( | |
| let result = executor.execute(resolved, options, agent_scope, Some(&run_id), resume)?; | ||
| reclaim_non_persistent_sessions(config, project_root, &running_before)?; | ||
|
|
||
| // Auto-release claim on workflow failure so issues don't stay permanently blocked. | ||
| if !result.success | ||
| && let Ok(Some(issue_num)) = claim::load_selected_issue_number(project_root) | ||
| && let Ok(Some(_lease)) = claim::load_claim(project_root, issue_num) | ||
| { | ||
| let reason = format!( | ||
| "workflow `{}` failed (steps {:?})", | ||
| result.workflow_name, result.failed_steps | ||
| ); | ||
| if let Err(e) = claim::release_claim(project_root, issue_num, &reason) { | ||
| eprintln!("claim: auto-release failed for issue #{}: {}", issue_num, e); | ||
| } | ||
| } | ||
|
Comment on lines
+3066
to
+3078
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Scope claim auto-release to the top-level owner run.
🤖 Prompt for AI Agents |
||
|
|
||
| // Recursion guard: don't emit workflow_complete from workflow_complete hooks. | ||
| if options.origin != ExecutionOrigin::HookWorkflowComplete { | ||
| let payload = WorkflowCompletePayload { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Smoke dispatches now have no claim cleanup path.
Line 62 still runs
sdlc-smoke, and that workflow selects an issue viaselect_issue.sh. Because this cleanup step is gated toinputs.mode == 'auto', smoke runs can applyautomation-claimedwithout any matching release on success or failure.🤖 Prompt for AI Agents