diff --git a/.github/workflows/sdlc-orchestrator.yml b/.github/workflows/sdlc-orchestrator.yml index 643f576..0f3b8fe 100644 --- a/.github/workflows/sdlc-orchestrator.yml +++ b/.github/workflows/sdlc-orchestrator.yml @@ -71,5 +71,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} ISSUE_LABEL: ${{ inputs.issue_label }} + GITHUB_RUN_ID: ${{ github.run_id }} run: | cargo run --quiet -- run sdlc-auto --strict + + - name: Release claim on failure/cancel + if: ${{ always() && inputs.mode == 'auto' && (failure() || cancelled()) }} + env: + GH_TOKEN: ${{ github.token }} + run: | + scripts/automation/release_claim.sh .tutti/state/auto/selected_issue.json "workflow ${{ job.status }}" diff --git a/docs/examples/tutti-codex-sdlc.toml b/docs/examples/tutti-codex-sdlc.toml index d396f4d..df6dd0a 100644 --- a/docs/examples/tutti-codex-sdlc.toml +++ b/docs/examples/tutti-codex-sdlc.toml @@ -106,7 +106,14 @@ id = "ensure_docs_release" type = "ensure_running" agent = "docs-release" -# 2) Intake and branch setup +# 2) Sweep stale claims before selecting a new issue +[[workflow.step]] +id = "sweep_stale_claims" +type = "command" +run = "scripts/automation/sweep_stale_claims.sh .tutti/state/claims" +fail_mode = "open" + +# 3) Intake and branch setup [[workflow.step]] id = "select_issue" type = "command" @@ -222,3 +229,10 @@ inject_files = [".tutti/state/auto/selected_issue.json", ".tutti/state/auto/bran wait_for_idle = true wait_timeout_secs = 900 text = "Summarize final readiness, residual risks, and merge recommendation." + +# 10) Release claim after successful completion +[[workflow.step]] +id = "release_claim" +type = "command" +run = "scripts/automation/release_claim.sh .tutti/state/auto/selected_issue.json 'workflow completed successfully'" +fail_mode = "open" diff --git a/scripts/automation/release_claim.sh b/scripts/automation/release_claim.sh new file mode 100755 index 0000000..24578b4 --- /dev/null +++ b/scripts/automation/release_claim.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Release the automation-claimed label from the selected issue. +# Usage: release_claim.sh [selected_issue_json] [reason] + +ISSUE_FILE="${1:-.tutti/state/auto/selected_issue.json}" +REASON="${2:-workflow completed}" + +if [ ! -f "$ISSUE_FILE" ]; then + echo "No selected issue file at $ISSUE_FILE — nothing to release." >&2 + exit 0 +fi + +REPO="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}" + +ISSUE_NUM=$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['issue_number'])" "$ISSUE_FILE") + +if [ -z "$ISSUE_NUM" ]; then + echo "Could not read issue_number from $ISSUE_FILE" >&2 + exit 1 +fi + +# Remove the automation-claimed label (tolerate it already being absent). +gh issue edit "$ISSUE_NUM" --repo "$REPO" --remove-label "automation-claimed" 2>/dev/null || true + +# Extract run_id from claim metadata if available. +RUN_ID=$(python3 -c " +import json,sys +d = json.load(open(sys.argv[1])) +print(d.get('claim', {}).get('run_id', 'unknown')) +" "$ISSUE_FILE" 2>/dev/null || echo "unknown") + +# Post audit comment. +gh issue comment "$ISSUE_NUM" --repo "$REPO" \ + --body "🤖 **Claim released** — reason: ${REASON} (run \`${RUN_ID}\`)" \ + >/dev/null 2>&1 || true + +# Remove claim state file. +CLAIM_FILE=".tutti/state/claims/${ISSUE_NUM}.json" +rm -f "$CLAIM_FILE" 2>/dev/null || true + +echo "Released claim on issue #${ISSUE_NUM}: ${REASON}" diff --git a/scripts/automation/select_issue.sh b/scripts/automation/select_issue.sh index d2c1914..c7a4e4f 100755 --- a/scripts/automation/select_issue.sh +++ b/scripts/automation/select_issue.sh @@ -41,18 +41,58 @@ PY gh issue edit "$ISSUE_NUM" --repo "$REPO" --add-label "automation-claimed" >/dev/null +# Generate a unique run ID for claim tracking. +RUN_ID="${GITHUB_RUN_ID:-local-$(date +%s)-$$}" +LEASE_TTL="${CLAIM_LEASE_TTL:-1800}" + ISSUE_DETAILS=$(gh issue view "$ISSUE_NUM" --repo "$REPO" --json body) -python3 - "$OUT_FILE" "$ISSUE_DETAILS" <<'PY' +python3 - "$OUT_FILE" "$ISSUE_DETAILS" "$RUN_ID" "$LEASE_TTL" "$REPO" <<'PY' import json,sys +from datetime import datetime, timezone out, details_raw = sys.argv[1], sys.argv[2] +run_id, lease_ttl, repo = sys.argv[3], int(sys.argv[4]), sys.argv[5] with open(f"{out}.tmp", "r", encoding="utf-8") as f: payload = json.load(f) details = json.loads(details_raw or "{}") payload["body"] = (details.get("body") or "").strip() +# Embed claim metadata into the selected issue state. +now = datetime.now(timezone.utc).isoformat() +payload["claim"] = { + "run_id": run_id, + "claimed_at": now, + "renewed_at": now, + "lease_ttl_secs": lease_ttl, + "repo": repo, +} with open(f"{out}.tmp", "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) PY mv "${OUT_FILE}.tmp" "$OUT_FILE" +# Persist claim lease for the Rust-side sweeper / auto-release. +CLAIMS_DIR="$(dirname "$OUT_FILE")/../../state/claims" +mkdir -p "$CLAIMS_DIR" +python3 - "$CLAIMS_DIR" "$ISSUE_NUM" "$RUN_ID" "$LEASE_TTL" "$REPO" <<'PY' +import json,sys,os +from datetime import datetime, timezone +claims_dir, issue_num = sys.argv[1], sys.argv[2] +run_id, lease_ttl, repo = sys.argv[3], int(sys.argv[4]), sys.argv[5] +now = datetime.now(timezone.utc).isoformat() +lease = { + "issue_number": int(issue_num), + "repo": repo, + "run_id": run_id, + "claimed_at": now, + "renewed_at": now, + "lease_ttl_secs": lease_ttl, +} +path = os.path.join(claims_dir, f"{issue_num}.json") +with open(path, "w", encoding="utf-8") as f: + json.dump(lease, f, indent=2) +PY + +# Post audit comment on the issue. +gh issue comment "$ISSUE_NUM" --repo "$REPO" --body "🤖 **Claim acquired** by run \`$RUN_ID\` — lease ${LEASE_TTL}s" >/dev/null 2>&1 || true + echo "$OUT_FILE" diff --git a/scripts/automation/sweep_stale_claims.sh b/scripts/automation/sweep_stale_claims.sh new file mode 100755 index 0000000..b94cf40 --- /dev/null +++ b/scripts/automation/sweep_stale_claims.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Sweep stale claim leases whose TTL has expired. +# Usage: sweep_stale_claims.sh [claims_dir] + +CLAIMS_DIR="${1:-.tutti/state/claims}" + +if [ ! -d "$CLAIMS_DIR" ]; then + echo "No claims directory at $CLAIMS_DIR — nothing to sweep." + exit 0 +fi + +REPO="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}" + +python3 - "$CLAIMS_DIR" "$REPO" <<'PY' +import json, sys, os, subprocess +from datetime import datetime, timezone + +claims_dir, repo = sys.argv[1], sys.argv[2] +released = 0 + +for fname in os.listdir(claims_dir): + if not fname.endswith(".json"): + continue + path = os.path.join(claims_dir, fname) + try: + with open(path) as f: + lease = json.load(f) + except (json.JSONDecodeError, OSError): + continue + + renewed_at = datetime.fromisoformat(lease["renewed_at"].replace("Z", "+00:00")) + ttl = lease.get("lease_ttl_secs", 1800) + now = datetime.now(timezone.utc) + elapsed = (now - renewed_at).total_seconds() + + if elapsed <= ttl: + remaining = int(ttl - elapsed) + print(f" active: issue #{lease['issue_number']} (run={lease['run_id']}, {remaining}s remaining)") + continue + + issue_num = lease["issue_number"] + run_id = lease.get("run_id", "unknown") + expired_ago = int(elapsed - ttl) + print(f" stale: issue #{issue_num} (run={run_id}, expired {expired_ago}s ago) — releasing") + + # Remove label. + subprocess.run( + ["gh", "issue", "edit", str(issue_num), "--repo", repo, "--remove-label", "automation-claimed"], + capture_output=True, + ) + # Post audit comment. + subprocess.run( + ["gh", "issue", "comment", str(issue_num), "--repo", repo, + "--body", f"🤖 **Claim released** — reason: lease expired (sweeper, run `{run_id}`)"], + capture_output=True, + ) + # Remove claim file. + os.remove(path) + released += 1 + +if released == 0: + print("sweep: no stale claims found") +else: + print(f"sweep: released {released} stale claim(s)") +PY diff --git a/src/automation/mod.rs b/src/automation/mod.rs index 3b3048b..de6f42b 100644 --- a/src/automation/mod.rs +++ b/src/automation/mod.rs @@ -1,3 +1,4 @@ +use crate::claim; use crate::config::{ HookConfig, HookEvent, HookWorkflowSource, PermissionsConfig, ResilienceConfig, TuttiConfig, WorkflowCommandCwd, WorkflowConfig, WorkflowFailMode, WorkflowStepConfig, @@ -3062,6 +3063,20 @@ pub fn execute_workflow_with_hooks( let result = executor.execute(resolved, options, agent_scope, Some(&run_id), resume)?; reclaim_non_persistent_sessions(config, project_root, &running_before)?; + // Auto-release claim on workflow failure so issues don't stay permanently blocked. + if !result.success + && let Ok(Some(issue_num)) = claim::load_selected_issue_number(project_root) + && let Ok(Some(_lease)) = claim::load_claim(project_root, issue_num) + { + let reason = format!( + "workflow `{}` failed (steps {:?})", + result.workflow_name, result.failed_steps + ); + if let Err(e) = claim::release_claim(project_root, issue_num, &reason) { + eprintln!("claim: auto-release failed for issue #{}: {}", issue_num, e); + } + } + // Recursion guard: don't emit workflow_complete from workflow_complete hooks. if options.origin != ExecutionOrigin::HookWorkflowComplete { let payload = WorkflowCompletePayload { diff --git a/src/claim/mod.rs b/src/claim/mod.rs new file mode 100644 index 0000000..43ad962 --- /dev/null +++ b/src/claim/mod.rs @@ -0,0 +1,416 @@ +use crate::error::{Result, TuttiError}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Default lease duration in seconds (30 minutes). +const DEFAULT_LEASE_TTL_SECS: u64 = 1800; +/// Label applied to claimed issues. +const CLAIM_LABEL: &str = "automation-claimed"; + +/// Metadata stored alongside a claim to enable lease expiry and audit. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClaimLease { + pub issue_number: u64, + pub repo: String, + pub run_id: String, + pub claimed_at: DateTime, + pub renewed_at: DateTime, + pub lease_ttl_secs: u64, +} + +impl ClaimLease { + /// Returns true if the lease has expired based on `renewed_at + lease_ttl_secs`. + pub fn is_expired(&self) -> bool { + let deadline = self.renewed_at + chrono::Duration::seconds(self.lease_ttl_secs as i64); + Utc::now() > deadline + } + + /// Seconds remaining on the lease, or 0 if expired. + pub fn remaining_secs(&self) -> i64 { + let deadline = self.renewed_at + chrono::Duration::seconds(self.lease_ttl_secs as i64); + (deadline - Utc::now()).num_seconds().max(0) + } +} + +// --------------------------------------------------------------------------- +// State persistence +// --------------------------------------------------------------------------- + +fn claims_dir(project_root: &Path) -> PathBuf { + project_root.join(".tutti").join("state").join("claims") +} + +fn claim_path(project_root: &Path, issue_number: u64) -> PathBuf { + claims_dir(project_root).join(format!("{}.json", issue_number)) +} + +/// Persist a claim lease to disk. +pub fn save_claim(project_root: &Path, lease: &ClaimLease) -> Result<()> { + let dir = claims_dir(project_root); + std::fs::create_dir_all(&dir)?; + let path = claim_path(project_root, lease.issue_number); + let json = serde_json::to_string_pretty(lease)?; + std::fs::write(path, json)?; + Ok(()) +} + +/// Load a claim lease from disk, if one exists. +pub fn load_claim(project_root: &Path, issue_number: u64) -> Result> { + let path = claim_path(project_root, issue_number); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read_to_string(&path)?; + let lease: ClaimLease = serde_json::from_str(&data)?; + Ok(Some(lease)) +} + +/// Remove a claim lease file from disk. +fn remove_claim_file(project_root: &Path, issue_number: u64) -> Result<()> { + let path = claim_path(project_root, issue_number); + if path.exists() { + std::fs::remove_file(path)?; + } + Ok(()) +} + +/// List all persisted claim leases. +pub fn list_claims(project_root: &Path) -> Result> { + let dir = claims_dir(project_root); + if !dir.exists() { + return Ok(Vec::new()); + } + let mut claims = Vec::new(); + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) == Some("json") { + let data = std::fs::read_to_string(&path)?; + if let Ok(lease) = serde_json::from_str::(&data) { + claims.push(lease); + } + } + } + Ok(claims) +} + +// --------------------------------------------------------------------------- +// GitHub interactions via `gh` CLI +// --------------------------------------------------------------------------- + +fn gh_command() -> Command { + Command::new("gh") +} + +/// Add the `automation-claimed` label to an issue. +fn add_claim_label(repo: &str, issue_number: u64) -> Result<()> { + let output = gh_command() + .args([ + "issue", + "edit", + &issue_number.to_string(), + "--repo", + repo, + "--add-label", + CLAIM_LABEL, + ]) + .output()?; + if !output.status.success() { + return Err(TuttiError::State(format!( + "failed to add claim label to issue #{}: {}", + issue_number, + String::from_utf8_lossy(&output.stderr) + ))); + } + Ok(()) +} + +/// Remove the `automation-claimed` label from an issue. +fn remove_claim_label(repo: &str, issue_number: u64) -> Result<()> { + let output = gh_command() + .args([ + "issue", + "edit", + &issue_number.to_string(), + "--repo", + repo, + "--remove-label", + CLAIM_LABEL, + ]) + .output()?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + // Tolerate the label already being absent. + if !stderr.contains("not found") && !stderr.contains("does not have") { + return Err(TuttiError::State(format!( + "failed to remove claim label from issue #{}: {}", + issue_number, stderr + ))); + } + } + Ok(()) +} + +/// Post an audit comment on the issue. +fn post_claim_comment(repo: &str, issue_number: u64, body: &str) -> Result<()> { + let output = gh_command() + .args([ + "issue", + "comment", + &issue_number.to_string(), + "--repo", + repo, + "--body", + body, + ]) + .output()?; + if !output.status.success() { + // Non-fatal: log but don't fail the claim operation. + eprintln!( + "warning: failed to comment on issue #{}: {}", + issue_number, + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Public claim lifecycle +// --------------------------------------------------------------------------- + +/// Acquire a claim on an issue: add the label, persist metadata, post a comment. +pub fn acquire_claim( + project_root: &Path, + repo: &str, + issue_number: u64, + run_id: &str, + lease_ttl_secs: Option, +) -> Result { + let now = Utc::now(); + let ttl = lease_ttl_secs.unwrap_or(DEFAULT_LEASE_TTL_SECS); + + let lease = ClaimLease { + issue_number, + repo: repo.to_string(), + run_id: run_id.to_string(), + claimed_at: now, + renewed_at: now, + lease_ttl_secs: ttl, + }; + + add_claim_label(repo, issue_number)?; + save_claim(project_root, &lease)?; + + let comment = format!( + "🤖 **Claim acquired** by run `{}` — lease {}s (expires ~{})", + run_id, + ttl, + now + chrono::Duration::seconds(ttl as i64), + ); + let _ = post_claim_comment(repo, issue_number, &comment); + + eprintln!( + "claim: acquired issue #{} (run={}, ttl={}s)", + issue_number, run_id, ttl + ); + Ok(lease) +} + +/// Renew an existing claim lease, extending the expiry window. +pub fn renew_claim(project_root: &Path, issue_number: u64) -> Result { + let mut lease = load_claim(project_root, issue_number)? + .ok_or_else(|| TuttiError::State(format!("no claim found for issue #{}", issue_number)))?; + + lease.renewed_at = Utc::now(); + save_claim(project_root, &lease)?; + + eprintln!( + "claim: renewed issue #{} (remaining={}s)", + issue_number, + lease.remaining_secs() + ); + Ok(lease) +} + +/// Release a claim: remove the label, delete state, post a comment. +pub fn release_claim(project_root: &Path, issue_number: u64, reason: &str) -> Result<()> { + let lease = load_claim(project_root, issue_number)?; + let repo = match &lease { + Some(l) => l.repo.clone(), + None => { + eprintln!( + "claim: no local lease for issue #{}, attempting label removal from env", + issue_number + ); + resolve_repo()? + } + }; + + remove_claim_label(&repo, issue_number)?; + remove_claim_file(project_root, issue_number)?; + + let comment = format!( + "🤖 **Claim released** — reason: {}{}", + reason, + lease + .as_ref() + .map(|l| format!(" (run `{}`)", l.run_id)) + .unwrap_or_default(), + ); + let _ = post_claim_comment(&repo, issue_number, &comment); + + eprintln!("claim: released issue #{} ({})", issue_number, reason); + Ok(()) +} + +/// Sweep all persisted claims, releasing any whose lease has expired. +/// Returns the list of issue numbers that were released. +pub fn sweep_stale_claims(project_root: &Path) -> Result> { + let claims = list_claims(project_root)?; + let mut released = Vec::new(); + + for lease in &claims { + if lease.is_expired() { + eprintln!( + "claim: sweeping stale claim on issue #{} (run={}, expired {}s ago)", + lease.issue_number, + lease.run_id, + -lease.remaining_secs(), + ); + if let Err(e) = + release_claim(project_root, lease.issue_number, "lease expired (sweeper)") + { + eprintln!( + "claim: failed to release stale claim on issue #{}: {}", + lease.issue_number, e + ); + } else { + released.push(lease.issue_number); + } + } + } + + if released.is_empty() { + eprintln!("claim: sweep complete — no stale claims found"); + } else { + eprintln!( + "claim: sweep complete — released {} stale claim(s)", + released.len() + ); + } + Ok(released) +} + +/// Try to resolve the current GitHub repo from `GITHUB_REPOSITORY` env or `gh`. +fn resolve_repo() -> Result { + if let Ok(repo) = std::env::var("GITHUB_REPOSITORY") { + return Ok(repo); + } + let output = gh_command() + .args([ + "repo", + "view", + "--json", + "nameWithOwner", + "-q", + ".nameWithOwner", + ]) + .output()?; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } else { + Err(TuttiError::State( + "cannot resolve GitHub repo: set GITHUB_REPOSITORY or run from a gh-authenticated checkout" + .to_string(), + )) + } +} + +/// Load the selected issue from the standard state file and return its number. +pub fn load_selected_issue_number(project_root: &Path) -> Result> { + let path = project_root + .join(".tutti") + .join("state") + .join("auto") + .join("selected_issue.json"); + if !path.exists() { + return Ok(None); + } + let data = std::fs::read_to_string(&path)?; + let val: serde_json::Value = serde_json::from_str(&data)?; + Ok(val.get("issue_number").and_then(|v| v.as_u64())) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Duration; + + #[test] + fn lease_expiry_logic() { + let now = Utc::now(); + let lease = ClaimLease { + issue_number: 42, + repo: "owner/repo".to_string(), + run_id: "run-1".to_string(), + claimed_at: now - Duration::seconds(3600), + renewed_at: now - Duration::seconds(3600), + lease_ttl_secs: 1800, + }; + assert!(lease.is_expired()); + assert_eq!(lease.remaining_secs(), 0); + } + + #[test] + fn lease_not_expired() { + let now = Utc::now(); + let lease = ClaimLease { + issue_number: 42, + repo: "owner/repo".to_string(), + run_id: "run-1".to_string(), + claimed_at: now, + renewed_at: now, + lease_ttl_secs: 1800, + }; + assert!(!lease.is_expired()); + assert!(lease.remaining_secs() > 1790); + } + + #[test] + fn save_and_load_claim() { + let tmp = std::env::temp_dir().join(format!("tutti-claim-test-{}", std::process::id())); + std::fs::create_dir_all(&tmp).unwrap(); + + let lease = ClaimLease { + issue_number: 7, + repo: "owner/repo".to_string(), + run_id: "run-abc".to_string(), + claimed_at: Utc::now(), + renewed_at: Utc::now(), + lease_ttl_secs: 600, + }; + + save_claim(&tmp, &lease).unwrap(); + let loaded = load_claim(&tmp, 7).unwrap().expect("should exist"); + assert_eq!(loaded.issue_number, 7); + assert_eq!(loaded.run_id, "run-abc"); + assert_eq!(loaded.lease_ttl_secs, 600); + + remove_claim_file(&tmp, 7).unwrap(); + assert!(load_claim(&tmp, 7).unwrap().is_none()); + + let _ = std::fs::remove_dir_all(&tmp); + } + + #[test] + fn list_claims_empty_dir() { + let tmp = std::env::temp_dir().join(format!("tutti-claim-list-{}", std::process::id())); + // Dir doesn't exist yet + let claims = list_claims(&tmp).unwrap(); + assert!(claims.is_empty()); + let _ = std::fs::remove_dir_all(&tmp); + } +} diff --git a/src/main.rs b/src/main.rs index 5552025..381aba3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,7 @@ mod automation; mod budget; +#[allow(dead_code)] +mod claim; mod cli; mod config; mod error;