diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index d50d023bdf..e431e45a36 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -102,6 +102,8 @@ Use `/save-todo` after planning. Auto-detects complexity: **Dependencies**: `blocked-by:t001`, `blocks:t002`, `t001.1` (subtask) +**GitHub issue sync**: Issue titles MUST be prefixed with `t{NNN}:` (followed by a space, e.g., `t{NNN}: `). TODO.md tasks MUST include `ref:GH#{issue}`. Always maintain both directions. See `workflows/plans.md` "GitHub Issue Sync" section. + **After ANY TODO/planning edit**: Commit and push immediately. Planning-only files (TODO.md, todo/) go directly to main — no branch, no PR. Mixed changes (planning + non-exception files) use a worktree. NEVER `git checkout -b` in the main repo. See `workflows/plans.md` "Commit and Push After TODO Changes" section. **Full docs**: `workflows/plans.md`, `tools/task-management/beads.md` diff --git a/.agents/scripts/commands/log-issue-aidevops.md b/.agents/scripts/commands/log-issue-aidevops.md index c7712e9904..2871bef304 100644 --- a/.agents/scripts/commands/log-issue-aidevops.md +++ b/.agents/scripts/commands/log-issue-aidevops.md @@ -129,11 +129,25 @@ Labels: bug (or enhancement, question, documentation) 4. Cancel ``` +### Step 5b: Assign Task ID and Sync (Maintainers Only) + +> **Note**: This step is for aidevops maintainers with repository write access. External users reporting issues can skip to Step 6. + +Before creating the issue, assign a TODO.md task ID: + +1. Read TODO.md to find the highest existing t-number +2. Assign the next sequential t-number (e.g., if highest is t148, use t149) +3. Prefix the issue title with `t{NNN}:` (followed by a space) +4. After issue creation, add the task to TODO.md with `ref:GH#{issue_number}` +5. Commit and push TODO.md + +This ensures bidirectional sync between GitHub issues and TODO.md. See `workflows/plans.md` "GitHub Issue Sync" section. + ### Step 6: Create the Issue ```bash gh issue create -R marcusquinn/aidevops \ - --title "TITLE" \ + --title "t{NNN}: TITLE" \ --body "$(cat <<'EOF' BODY_CONTENT EOF diff --git a/.agents/scripts/session-checkpoint-helper.sh b/.agents/scripts/session-checkpoint-helper.sh new file mode 100755 index 0000000000..7b96f51a07 --- /dev/null +++ b/.agents/scripts/session-checkpoint-helper.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# session-checkpoint-helper.sh - Persist session state to survive context compaction +# Part of aidevops framework: https://aidevops.sh +# +# Usage: +# session-checkpoint-helper.sh [command] [options] +# +# Commands: +# save Save current session checkpoint +# load Load and display current checkpoint +# clear Remove checkpoint file +# status Show checkpoint age and summary +# help Show this help +# +# Options: +# --task <id> Current task ID (e.g., t135.9) +# --next <ids> Comma-separated next task IDs +# --worktree <path> Active worktree path +# --branch <name> Active branch name +# --batch <name> Batch/PR name +# --note <text> Free-form context note +# --elapsed <mins> Minutes elapsed in session +# --target <mins> Target session duration in minutes +# +# The checkpoint file is written to: +# ~/.aidevops/.agent-workspace/tmp/session-checkpoint.md +# +# Design: AI agent writes checkpoint after each task completion and reads it +# before starting the next task. Survives context compaction because state +# is on disk, not in context window. + +set -euo pipefail + +readonly CHECKPOINT_DIR="${HOME}/.aidevops/.agent-workspace/tmp" +readonly CHECKPOINT_FILE="${CHECKPOINT_DIR}/session-checkpoint.md" + +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly RED='\033[0;31m' +readonly BOLD='\033[1m' +readonly DIM='\033[2m' +readonly NC='\033[0m' + +print_success() { printf "${GREEN}%s${NC}\n" "$1"; } +print_warning() { printf "${YELLOW}%s${NC}\n" "$1"; } +print_error() { printf "${RED}%s${NC}\n" "$1" >&2; } +print_info() { printf "${DIM}%s${NC}\n" "$1"; } + +ensure_dir() { + if [[ ! -d "$CHECKPOINT_DIR" ]]; then + mkdir -p "$CHECKPOINT_DIR" + fi + return 0 +} + +cmd_save() { + local current_task="" + local next_tasks="" + local worktree_path="" + local branch_name="" + local batch_name="" + local note="" + local elapsed_mins="" + local target_mins="" + + while [[ $# -gt 0 ]]; do + case "$1" in + --task) [[ $# -lt 2 ]] && { print_error "--task requires a value"; return 1; }; current_task="$2"; shift 2 ;; + --next) [[ $# -lt 2 ]] && { print_error "--next requires a value"; return 1; }; next_tasks="$2"; shift 2 ;; + --worktree) [[ $# -lt 2 ]] && { print_error "--worktree requires a value"; return 1; }; worktree_path="$2"; shift 2 ;; + --branch) [[ $# -lt 2 ]] && { print_error "--branch requires a value"; return 1; }; branch_name="$2"; shift 2 ;; + --batch) [[ $# -lt 2 ]] && { print_error "--batch requires a value"; return 1; }; batch_name="$2"; shift 2 ;; + --note) [[ $# -lt 2 ]] && { print_error "--note requires a value"; return 1; }; note="$2"; shift 2 ;; + --elapsed) [[ $# -lt 2 ]] && { print_error "--elapsed requires a value"; return 1; }; elapsed_mins="$2"; shift 2 ;; + --target) [[ $# -lt 2 ]] && { print_error "--target requires a value"; return 1; }; target_mins="$2"; shift 2 ;; + *) print_error "Unknown option: $1"; return 1 ;; + esac + done + + ensure_dir + + local timestamp + timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + # Auto-detect git state if not provided + if [[ -z "$branch_name" ]]; then + branch_name="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")" + fi + + # Build checkpoint file + cat > "$CHECKPOINT_FILE" <<EOF +# Session Checkpoint + +Updated: ${timestamp} + +## Current State + +| Field | Value | +|-------|-------| +| Current Task | ${current_task:-none} | +| Branch | ${branch_name} | +| Worktree | ${worktree_path:-not set} | +| Batch/PR | ${batch_name:-not set} | +| Elapsed | ${elapsed_mins:-unknown} min | +| Target | ${target_mins:-unknown} min | + +## Next Tasks + +${next_tasks:-No next tasks specified} + +## Context Note + +${note:-No additional context} + +## Git Status + +$(git status --short 2>/dev/null || echo "Not in a git repo") + +## Recent Commits (this branch) + +$(git log --oneline -5 2>/dev/null || echo "No commits") + +## Open Worktrees + +$(git worktree list 2>/dev/null || echo "No worktrees") +EOF + + print_success "Checkpoint saved: ${CHECKPOINT_FILE}" + print_info "Task: ${current_task:-none} | Branch: ${branch_name} | ${timestamp}" + return 0 +} + +cmd_load() { + if [[ ! -f "$CHECKPOINT_FILE" ]]; then + print_warning "No checkpoint found at ${CHECKPOINT_FILE}" + print_info "Run: session-checkpoint-helper.sh save --task <id> --next <ids>" + return 1 + fi + + cat "$CHECKPOINT_FILE" + return 0 +} + +cmd_clear() { + if [[ -f "$CHECKPOINT_FILE" ]]; then + rm "$CHECKPOINT_FILE" + print_success "Checkpoint cleared" + else + print_info "No checkpoint to clear" + fi + return 0 +} + +cmd_status() { + if [[ ! -f "$CHECKPOINT_FILE" ]]; then + print_warning "No active checkpoint" + return 1 + fi + + local file_age_seconds + local now + local file_mtime + + now="$(date +%s)" + if [[ "$(uname)" == "Darwin" ]]; then + file_mtime="$(stat -f %m "$CHECKPOINT_FILE")" + else + file_mtime="$(stat -c %Y "$CHECKPOINT_FILE")" + fi + file_age_seconds=$(( now - file_mtime )) + + local age_display + if [[ $file_age_seconds -lt 60 ]]; then + age_display="${file_age_seconds}s ago" + elif [[ $file_age_seconds -lt 3600 ]]; then + age_display="$(( file_age_seconds / 60 ))m ago" + else + age_display="$(( file_age_seconds / 3600 ))h $(( (file_age_seconds % 3600) / 60 ))m ago" + fi + + # Extract key fields + local current_task + current_task="$(awk -F'|' '/Current Task/ {gsub(/^[ \t]+|[ \t]+$/, "", $3); print $3; exit}' "$CHECKPOINT_FILE" || echo "unknown")" + local branch + branch="$(awk -F'|' '/Branch/ {gsub(/^[ \t]+|[ \t]+$/, "", $3); print $3; exit}' "$CHECKPOINT_FILE" || echo "unknown")" + + printf '%b\n' "${BOLD}Checkpoint Status${NC}" + printf " Age: %s\n" "$age_display" + printf " Task: %s\n" "$current_task" + printf " Branch: %s\n" "$branch" + printf " File: %s\n" "$CHECKPOINT_FILE" + + if [[ $file_age_seconds -gt 1800 ]]; then + print_warning " Warning: Checkpoint is stale (>30min). Consider updating." + fi + return 0 +} + +cmd_help() { + # Extract header comment block as help text + sed -n '2,/^$/p' "$0" | sed 's/^# \?//' + return 0 +} + +# Main dispatch +main() { + local command="${1:-help}" + shift 2>/dev/null || true + + case "$command" in + save) cmd_save "$@" ;; + load) cmd_load ;; + clear) cmd_clear ;; + status) cmd_status ;; + help|-h|--help) cmd_help ;; + *) + print_error "Unknown command: ${command}" + cmd_help + return 1 + ;; + esac +} + +main "$@" diff --git a/.agents/scripts/supervisor-helper.sh b/.agents/scripts/supervisor-helper.sh index 559baaf199..18643a7223 100755 --- a/.agents/scripts/supervisor-helper.sh +++ b/.agents/scripts/supervisor-helper.sh @@ -40,9 +40,9 @@ # -> failed (dispatch failure / unrecoverable) # # Post-PR lifecycle (t128.8): -# complete -> pr_review -> merging -> merged -> deploying -> deployed +# complete -> pr_review -> review_triage -> merging -> merged -> deploying -> deployed # Workers exit after PR creation. Supervisor handles remaining stages: -# wait for CI, merge (squash), postflight, deploy, worktree cleanup. +# wait for CI, triage review threads (t148), merge (squash), postflight, deploy, worktree cleanup. # # Outcome evaluation (3-tier): # 1. Deterministic: FULL_LOOP_COMPLETE/TASK_COMPLETE signals, EXIT codes @@ -79,7 +79,7 @@ readonly MEMORY_HELPER="${SCRIPT_DIR}/memory-helper.sh" # Used by pulse comman export MAIL_HELPER MEMORY_HELPER # Valid states for the state machine -readonly VALID_STATES="queued dispatched running evaluating retrying complete pr_review merging merged deploying deployed blocked failed cancelled" +readonly VALID_STATES="queued dispatched running evaluating retrying complete pr_review review_triage merging merged deploying deployed blocked failed cancelled" # Valid state transitions (from:to pairs) # Format: "from_state:to_state" - checked by validate_transition() @@ -105,9 +105,14 @@ readonly -a VALID_TRANSITIONS=( # Post-PR lifecycle transitions (t128.8) "complete:pr_review" "complete:deployed" + "pr_review:review_triage" "pr_review:merging" "pr_review:blocked" "pr_review:cancelled" + # Review triage transitions (t148) - triage unresolved review threads before merge + "review_triage:merging" + "review_triage:blocked" + "review_triage:pr_review" "merging:merged" "merging:blocked" "merging:failed" @@ -365,7 +370,7 @@ CREATE TABLE tasks ( repo TEXT NOT NULL, description TEXT, status TEXT NOT NULL DEFAULT 'queued' - CHECK(status IN ('queued','dispatched','running','evaluating','retrying','complete','pr_review','merging','merged','deploying','deployed','blocked','failed','cancelled')), + CHECK(status IN ('queued','dispatched','running','evaluating','retrying','complete','pr_review','review_triage','merging','merged','deploying','deployed','blocked','failed','cancelled')), session_id TEXT, worktree TEXT, branch TEXT, @@ -391,6 +396,44 @@ MIGRATE log_success "Database schema migrated for post-PR lifecycle states" fi + # Migrate: add review_triage to status CHECK constraint (t148) + if [[ -n "$check_sql" ]] && ! echo "$check_sql" | grep -q 'review_triage'; then + log_info "Migrating database schema for review_triage state (t148)..." + db "$SUPERVISOR_DB" << 'MIGRATE_TRIAGE' +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +ALTER TABLE tasks RENAME TO tasks_old_t148; +CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + repo TEXT NOT NULL, + description TEXT, + status TEXT NOT NULL DEFAULT 'queued' + CHECK(status IN ('queued','dispatched','running','evaluating','retrying','complete','pr_review','review_triage','merging','merged','deploying','deployed','blocked','failed','cancelled')), + session_id TEXT, + worktree TEXT, + branch TEXT, + log_file TEXT, + retries INTEGER NOT NULL DEFAULT 0, + max_retries INTEGER NOT NULL DEFAULT 3, + model TEXT DEFAULT 'anthropic/claude-opus-4-6', + error TEXT, + pr_url TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + started_at TEXT, + completed_at TEXT, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')) +); +INSERT INTO tasks SELECT * FROM tasks_old_t148; +DROP TABLE tasks_old_t148; +CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status); +CREATE INDEX IF NOT EXISTS idx_tasks_repo ON tasks(repo); +CREATE INDEX IF NOT EXISTS idx_tasks_created ON tasks(created_at); +COMMIT; +PRAGMA foreign_keys=ON; +MIGRATE_TRIAGE + log_success "Database schema migrated for review_triage state" + fi + # Migrate: add max_load_factor column to batches if missing (t135.15.4) local has_max_load has_max_load=$(db "$SUPERVISOR_DB" "SELECT count(*) FROM pragma_table_info('batches') WHERE name='max_load_factor';" 2>/dev/null || echo "0") @@ -426,7 +469,7 @@ CREATE TABLE IF NOT EXISTS tasks ( repo TEXT NOT NULL, description TEXT, status TEXT NOT NULL DEFAULT 'queued' - CHECK(status IN ('queued','dispatched','running','evaluating','retrying','complete','pr_review','merging','merged','deploying','deployed','blocked','failed','cancelled')), + CHECK(status IN ('queued','dispatched','running','evaluating','retrying','complete','pr_review','review_triage','merging','merged','deploying','deployed','blocked','failed','cancelled')), session_id TEXT, worktree TEXT, branch TEXT, @@ -2541,13 +2584,48 @@ cmd_reprompt() { if [[ -n "$prompt_override" ]]; then reprompt_msg="$prompt_override" else - reprompt_msg="The previous attempt for task $task_id encountered an issue: ${terror:-unknown error}. + # Detect if this was a compaction/context-limit exit (clean exit, no signal) + # and tailor the reprompt accordingly + local is_compaction_retry=false + if [[ "$terror" == *"clean_exit_no_signal"* || "$terror" == *"context"* ]]; then + is_compaction_retry=true + fi + + if [[ "$is_compaction_retry" == "true" ]]; then + # Compaction-specific reprompt: the worker likely ran out of context + # window and exited cleanly without completing. Give it targeted guidance. + local worktree_status="" + if [[ -n "$tworktree" && -d "$tworktree" ]]; then + worktree_status=$(git -C "$tworktree" status --short 2>/dev/null | head -20 || echo "") + local recent_commits + recent_commits=$(git -C "$tworktree" log --oneline -5 2>/dev/null || echo "") + if [[ -n "$recent_commits" ]]; then + worktree_status="${worktree_status} + +Recent commits on this branch: +${recent_commits}" + fi + fi + + reprompt_msg="The previous attempt for task $task_id likely hit context compaction and exited without completing. + +/full-loop $task_id + +IMPORTANT: Check the worktree for partial progress before starting fresh. The previous session may have made commits or uncommitted changes. +${worktree_status:+ +Current worktree state: +${worktree_status}} + +Task description: ${tdesc:-$task_id}" + else + reprompt_msg="The previous attempt for task $task_id encountered an issue: ${terror:-unknown error}. Please continue the /full-loop for $task_id. Pick up where the previous attempt left off. If the task was partially completed, verify what's done and continue from there. If it failed entirely, start fresh with /full-loop $task_id. Task description: ${tdesc:-$task_id}" + fi fi # Set up log file for this retry attempt @@ -2778,6 +2856,157 @@ check_pr_status() { return 0 } +####################################### +# Check for unresolved review threads on a PR (t148) +# Returns JSON: {"total": N, "unresolved": N, "threads": [...]} +# Each thread: {"author": "...", "body": "...", "is_bot": bool, "path": "...", "line": N} +####################################### +check_review_threads() { + local task_id="$1" + + ensure_db + + local escaped_id + escaped_id=$(sql_escape "$task_id") + local pr_url + pr_url=$(db "$SUPERVISOR_DB" "SELECT pr_url FROM tasks WHERE id = '$escaped_id';") + + if [[ -z "$pr_url" || "$pr_url" == "no_pr" || "$pr_url" == "task_only" ]]; then + echo '{"total":0,"unresolved":0,"threads":[]}' + return 0 + fi + + local pr_number + pr_number=$(echo "$pr_url" | grep -oE '[0-9]+$' || echo "") + local repo_slug + repo_slug=$(echo "$pr_url" | grep -oE 'github\.com/[^/]+/[^/]+' | sed 's|github\.com/||' || echo "") + + if [[ -z "$pr_number" || -z "$repo_slug" ]]; then + echo '{"total":0,"unresolved":0,"threads":[]}' + return 0 + fi + + local owner repo_name + owner="${repo_slug%%/*}" + repo_name="${repo_slug##*/}" + + # Fetch review threads via GraphQL (REST API doesn't expose thread resolution) + local graphql_result + graphql_result=$(gh api graphql -f query=' + query($owner: String!, $repo: String!, $pr: Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $pr) { + reviewThreads(first: 100) { # NOTE: max 100 per page; pagination not yet implemented (rare to exceed 100 threads) + totalCount + nodes { + isResolved + comments(first: 1) { + nodes { + author { login } + body + path + line + } + } + } + } + } + } + } + ' -f owner="$owner" -f repo="$repo_name" -F pr="$pr_number" 2>/dev/null || echo "") + + if [[ -z "$graphql_result" ]]; then + echo '{"total":0,"unresolved":0,"threads":[],"error":"graphql_failed"}' + return 0 + fi + + # Known bot accounts that leave COMMENTED reviews + local bot_pattern='coderabbitai|github-actions|dependabot|renovate|copilot|gemini|sonarcloud|codecov' + + # Parse and classify threads + local result + result=$(echo "$graphql_result" | jq --arg bots "$bot_pattern" ' + .data.repository.pullRequest.reviewThreads as $threads | + { + total: $threads.totalCount, + unresolved: [$threads.nodes[] | select(.isResolved == false)] | length, + threads: [ + $threads.nodes[] + | select(.isResolved == false) + | .comments.nodes[0] as $c + | { + author: ($c.author.login // "unknown"), + body: ($c.body // "" | .[0:500]), + is_bot: (($c.author.login // "") | test($bots; "i")), + path: ($c.path // ""), + line: ($c.line // 0) + } + ] + } + ' 2>/dev/null || echo '{"total":0,"unresolved":0,"threads":[]}') + + echo "$result" + return 0 +} + +####################################### +# Triage review feedback for a PR (t148) +# Classifies unresolved threads by severity and source (bot vs human) +# Returns: triage_pass (safe to merge), triage_block (needs attention), triage_skip (no threads) +####################################### +triage_review_feedback() { + local task_id="$1" + local skip_triage="${2:-false}" + + if [[ "$skip_triage" == "true" ]]; then + echo "triage_skip" + return 0 + fi + + local threads_json + threads_json=$(check_review_threads "$task_id") + + local unresolved + unresolved=$(echo "$threads_json" | jq -r '.unresolved // 0' 2>/dev/null || echo "0") + + if [[ "$unresolved" -eq 0 ]]; then + log_info "Review triage for $task_id: no unresolved threads" + echo "triage_pass" + return 0 + fi + + # Count bot vs human threads + local bot_count human_count + bot_count=$(echo "$threads_json" | jq '[.threads[] | select(.is_bot == true)] | length' 2>/dev/null || echo "0") + human_count=$(echo "$threads_json" | jq '[.threads[] | select(.is_bot == false)] | length' 2>/dev/null || echo "0") + + log_info "Review triage for $task_id: $unresolved unresolved ($bot_count bot, $human_count human)" + + # Human review threads always block + if [[ "$human_count" -gt 0 ]]; then + log_warn " $human_count human review thread(s) unresolved - blocking merge" + echo "$threads_json" | jq -r '.threads[] | select(.is_bot == false) | " - \(.author) on \(.path):\(.line): \(.body[0:120])"' 2>/dev/null || true + echo "triage_block" + return 0 + fi + + # Bot-only threads: classify by severity keywords + local high_severity_count + high_severity_count=$(echo "$threads_json" | jq '[.threads[] | select(.is_bot == true) | select(.body | test("\\b(bug|security|vulnerability|critical|error|crash|data.loss|injection|XSS|CSRF)\\b"; "i"))] | length' 2>/dev/null || echo "0") + + if [[ "$high_severity_count" -gt 0 ]]; then + log_warn " $high_severity_count high-severity bot finding(s) - blocking merge" + echo "$threads_json" | jq -r '.threads[] | select(.is_bot == true) | select(.body | test("\\b(bug|security|vulnerability|critical|error|crash|data.loss|injection|XSS|CSRF)\\b"; "i")) | " - \(.author) on \(.path):\(.line): \(.body[0:120])"' 2>/dev/null || true + echo "triage_block" + return 0 + fi + + # Bot-only, low severity (style, naming, docs suggestions) - pass with warning + log_info " $bot_count bot thread(s) are low-severity - safe to merge" + echo "triage_pass" + return 0 +} + ####################################### # Command: pr-check - check PR CI/review status for a task ####################################### @@ -3033,7 +3262,7 @@ cleanup_after_merge() { # Checks CI, merges, runs postflight, deploys, cleans up worktree ####################################### cmd_pr_lifecycle() { - local task_id="" dry_run="false" + local task_id="" dry_run="false" skip_review_triage="false" if [[ $# -gt 0 && ! "$1" =~ ^-- ]]; then task_id="$1" @@ -3043,6 +3272,7 @@ cmd_pr_lifecycle() { while [[ $# -gt 0 ]]; do case "$1" in --dry-run) dry_run=true; shift ;; + --skip-review-triage) skip_review_triage=true; shift ;; *) log_error "Unknown option: $1"; return 1 ;; esac done @@ -3115,10 +3345,38 @@ cmd_pr_lifecycle() { case "$pr_status" in ready_to_merge) - if [[ "$dry_run" == "false" ]]; then - cmd_transition "$task_id" "merging" 2>/dev/null || true + # Review triage gate (t148): check unresolved threads before merge + if [[ "$skip_review_triage" == "false" && "$dry_run" == "false" ]]; then + cmd_transition "$task_id" "review_triage" 2>/dev/null || true + local triage_result + triage_result=$(triage_review_feedback "$task_id") + case "$triage_result" in + triage_pass|triage_skip) + log_info "Review triage passed for $task_id" + cmd_transition "$task_id" "merging" 2>/dev/null || true + tstatus="merging" + ;; + triage_block) + log_warn "Review triage blocked $task_id - unresolved review threads need attention" + cmd_transition "$task_id" "blocked" --error "Unresolved review threads (t148 triage)" 2>/dev/null || true + send_task_notification "$task_id" "blocked" "Unresolved review threads need attention before merge" 2>/dev/null || true + return 1 + ;; + *) + log_warn "Review triage returned unexpected result: $triage_result" + cmd_transition "$task_id" "merging" 2>/dev/null || true + tstatus="merging" + ;; + esac + else + if [[ "$skip_review_triage" == "true" ]]; then + log_warn "Review triage bypassed via --skip-review-triage for $task_id" + fi + if [[ "$dry_run" == "false" ]]; then + cmd_transition "$task_id" "merging" 2>/dev/null || true + fi + tstatus="merging" fi - tstatus="merging" ;; already_merged) if [[ "$dry_run" == "false" ]]; then @@ -3160,10 +3418,11 @@ cmd_pr_lifecycle() { ;; no_pr) # Track consecutive no_pr failures to avoid infinite retry loop - local no_pr_key="no_pr_retries_${task_id}" local no_pr_count - no_pr_count=$(db "SELECT COALESCE( - (SELECT CAST(json_extract(error, '$.no_pr_retries') AS INTEGER) + no_pr_count=$(db "$SUPERVISOR_DB" "SELECT COALESCE( + (SELECT CAST(json_extract( + CASE WHEN json_valid(error) THEN error ELSE '{}' END, + '$.no_pr_retries') AS INTEGER) FROM tasks WHERE id='$task_id'), 0);" 2>/dev/null || echo "0") no_pr_count=$((no_pr_count + 1)) @@ -3180,12 +3439,45 @@ cmd_pr_lifecycle() { log_warn "No PR found for $task_id (attempt $no_pr_count/5)" # Store retry count in error field as JSON - db "UPDATE tasks SET error = json_set(COALESCE(error, '{}'), '$.no_pr_retries', $no_pr_count), updated_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') WHERE id='$task_id';" 2>/dev/null || true + db "$SUPERVISOR_DB" "UPDATE tasks SET error = json_set( + CASE WHEN json_valid(error) THEN error ELSE '{}' END, + '$.no_pr_retries', $no_pr_count), + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ','now') + WHERE id='$task_id';" 2>/dev/null || true return 0 ;; esac fi + # Step 2b: Re-triage if already in review_triage state (t148) + if [[ "$tstatus" == "review_triage" ]]; then + local triage_result + triage_result=$(triage_review_feedback "$task_id" "$skip_review_triage") + case "$triage_result" in + triage_pass|triage_skip) + log_info "Review triage passed for $task_id" + if [[ "$dry_run" == "false" ]]; then + cmd_transition "$task_id" "merging" 2>/dev/null || true + fi + tstatus="merging" + ;; + triage_block) + log_warn "Review triage still blocking $task_id" + if [[ "$dry_run" == "false" ]]; then + cmd_transition "$task_id" "blocked" --error "Unresolved review threads (t148 triage)" 2>/dev/null || true + fi + return 1 + ;; + *) + log_warn "Review triage returned unexpected: $triage_result - proceeding" + if [[ "$dry_run" == "false" ]]; then + cmd_transition "$task_id" "merging" 2>/dev/null || true + fi + tstatus="merging" + ;; + esac + fi + # Step 3: Merge if [[ "$tstatus" == "merging" ]]; then if [[ "$dry_run" == "true" ]]; then @@ -3245,7 +3537,7 @@ process_post_pr_lifecycle() { ensure_db # Find tasks eligible for post-PR processing - local where_clause="t.status IN ('complete', 'pr_review', 'merging', 'merged', 'deploying')" + local where_clause="t.status IN ('complete', 'pr_review', 'review_triage', 'merging', 'merged', 'deploying')" if [[ -n "$batch_id" ]]; then where_clause="$where_clause AND EXISTS (SELECT 1 FROM batch_tasks bt WHERE bt.task_id = t.id AND bt.batch_id = '$(sql_escape "$batch_id")')" fi @@ -3549,7 +3841,7 @@ cmd_pulse() { local total_complete total_complete=$(db "$SUPERVISOR_DB" "SELECT count(*) FROM tasks WHERE status IN ('complete', 'deployed');") local total_pr_review - total_pr_review=$(db "$SUPERVISOR_DB" "SELECT count(*) FROM tasks WHERE status IN ('pr_review', 'merging', 'merged', 'deploying');") + total_pr_review=$(db "$SUPERVISOR_DB" "SELECT count(*) FROM tasks WHERE status IN ('pr_review', 'review_triage', 'merging', 'merged', 'deploying');") local total_failed total_failed=$(db "$SUPERVISOR_DB" "SELECT count(*) FROM tasks WHERE status IN ('failed', 'blocked');") @@ -3632,6 +3924,14 @@ cmd_pulse() { log_info " Cleaned: $orphan_killed stale worker processes" fi + # Phase 5: Persist pulse checkpoint for orchestrator compaction resilience + # Writes state to disk so an AI orchestrating session can re-orient after + # context compaction by reading this file. Cron pulses are stateless bash + # and don't need this, but interactive orchestrators do. + save_pulse_checkpoint "${batch_id:-}" "$total_running" "$total_queued" \ + "$total_complete" "$total_tasks" "$total_failed" "$total_pr_review" \ + "$completed_count" "$dispatched_count" 2>/dev/null || true + return 0 } @@ -3824,6 +4124,80 @@ update_todo_on_complete() { } log_success "Committed and pushed TODO.md update for $task_id" + + # Sync matching GitHub issue (close on completion) + sync_github_issue "$task_id" "complete" 2>/dev/null || true + + return 0 +} + +####################################### +# Sync GitHub issue state for a task +# Closes matching issue on completion, adds blocked comment on failure +# Convention: issue titles prefixed with "t{NNN}: " +# TODO.md tasks reference issues with "ref:GH#{NNN}" +####################################### +sync_github_issue() { + local task_id="$1" + local event_type="$2" # complete, blocked + local detail="${3:-}" + + # Require gh CLI + if ! command -v gh &>/dev/null; then + log_info "gh CLI not available, skipping GitHub issue sync" + return 0 + fi + + ensure_db + + local escaped_id + escaped_id=$(sql_escape "$task_id") + local trepo + trepo=$(db "$SUPERVISOR_DB" "SELECT repo FROM tasks WHERE id = '$escaped_id';" 2>/dev/null || echo "") + + if [[ -z "$trepo" ]]; then + return 0 + fi + + # Determine repo slug from git remote + local repo_slug + repo_slug=$(git -C "$trepo" remote get-url origin 2>/dev/null | grep -oE '[^/:]+/[^/.]+' | tail -1 || echo "") + if [[ -z "$repo_slug" ]]; then + return 0 + fi + + # Search for matching issue by t-number prefix in title + local issue_number + issue_number=$(gh issue list --repo "$repo_slug" --state open --search "${task_id}:" --json number,title --jq ".[] | select(.title | startswith(\"${task_id}:\") or startswith(\"${task_id} \")) | .number" 2>/dev/null | head -1 || echo "") + + if [[ -z "$issue_number" ]]; then + log_info "No matching GitHub issue found for $task_id" + return 0 + fi + + case "$event_type" in + complete) + local pr_url + pr_url=$(db "$SUPERVISOR_DB" "SELECT pr_url FROM tasks WHERE id = '$escaped_id';" 2>/dev/null || echo "") + local close_comment="Completed by supervisor." + if [[ -n "$pr_url" && "$pr_url" != "no_pr" && "$pr_url" != "task_only" ]]; then + close_comment="Completed via ${pr_url}" + fi + gh issue close "$issue_number" --repo "$repo_slug" --comment "$close_comment" 2>/dev/null || { + log_warn "Failed to close GitHub issue #$issue_number for $task_id" + return 1 + } + log_success "Closed GitHub issue #$issue_number for $task_id" + ;; + blocked) + gh issue comment "$issue_number" --repo "$repo_slug" --body "Blocked by supervisor: ${detail}" 2>/dev/null || { + log_warn "Failed to comment on GitHub issue #$issue_number for $task_id" + return 1 + } + log_info "Added blocked comment to GitHub issue #$issue_number for $task_id" + ;; + esac + return 0 } @@ -3917,6 +4291,10 @@ ${notes_line}" "$todo_file" } log_success "Committed and pushed TODO.md blocked update for $task_id" + + # Sync matching GitHub issue (add blocked comment) + sync_github_issue "$task_id" "blocked" "$reason" 2>/dev/null || true + return 0 } @@ -3999,6 +4377,134 @@ send_task_notification() { return 0 } +####################################### +# Persist pulse state to disk for orchestrator compaction resilience +# An AI session orchestrating batch tasks can read this file after context +# compaction to re-orient without losing track of progress. +# File: $SUPERVISOR_DIR/pulse-checkpoint.md +####################################### +save_pulse_checkpoint() { + local batch_id="$1" + local running="$2" + local queued="$3" + local complete="$4" + local total="$5" + local failed="$6" + local pr_review="$7" + local just_completed="$8" + local just_dispatched="$9" + + local checkpoint_file="$SUPERVISOR_DIR/pulse-checkpoint.md" + local timestamp + timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + # Collect per-task state for running/queued tasks + local running_details + running_details=$(db -separator '|' "$SUPERVISOR_DB" " + SELECT id, description, worktree, status + FROM tasks WHERE status IN ('running', 'dispatched', 'evaluating') + ORDER BY started_at ASC; + " 2>/dev/null || echo "") + + local queued_details + queued_details=$(db -separator '|' "$SUPERVISOR_DB" " + SELECT id, description + FROM tasks WHERE status = 'queued' + ORDER BY rowid ASC LIMIT 10; + " 2>/dev/null || echo "") + + local blocked_details + blocked_details=$(db -separator '|' "$SUPERVISOR_DB" " + SELECT id, description, error + FROM tasks WHERE status IN ('blocked', 'failed') + ORDER BY updated_at DESC LIMIT 5; + " 2>/dev/null || echo "") + + local pr_details + pr_details=$(db -separator '|' "$SUPERVISOR_DB" " + SELECT id, description, status, pr_url + FROM tasks WHERE status IN ('pr_review', 'review_triage', 'merging', 'merged', 'deploying') + ORDER BY updated_at DESC; + " 2>/dev/null || echo "") + + # Build checkpoint + { + echo "# Supervisor Pulse Checkpoint" + echo "" + echo "Updated: ${timestamp}" + echo "Batch: ${batch_id:-all}" + echo "" + echo "## Summary" + echo "" + echo "| Metric | Count |" + echo "|--------|-------|" + echo "| Running | ${running} |" + echo "| Queued | ${queued} |" + echo "| Complete | ${complete} |" + echo "| Failed/Blocked | ${failed} |" + echo "| Post-PR | ${pr_review} |" + echo "| Total | ${total} |" + echo "| Just Completed | ${just_completed} |" + echo "| Just Dispatched | ${just_dispatched} |" + echo "" + + if [[ -n "$running_details" ]]; then + echo "## Running Tasks" + echo "" + while IFS='|' read -r tid tdesc tworktree tstatus; do + echo "- **${tid}** (${tstatus}): ${tdesc}" + if [[ -n "$tworktree" ]]; then + echo " Worktree: ${tworktree}" + fi + done <<< "$running_details" + echo "" + fi + + if [[ -n "$queued_details" ]]; then + echo "## Next Queued (up to 10)" + echo "" + while IFS='|' read -r tid tdesc; do + echo "- **${tid}**: ${tdesc}" + done <<< "$queued_details" + echo "" + fi + + if [[ -n "$pr_details" ]]; then + echo "## Post-PR Lifecycle" + echo "" + while IFS='|' read -r tid tdesc tstatus tpr; do + echo "- **${tid}** (${tstatus}): ${tdesc}" + if [[ -n "$tpr" ]]; then + echo " PR: ${tpr}" + fi + done <<< "$pr_details" + echo "" + fi + + if [[ -n "$blocked_details" ]]; then + echo "## Blocked/Failed (recent)" + echo "" + while IFS='|' read -r tid tdesc terror; do + echo "- **${tid}**: ${tdesc}" + if [[ -n "$terror" ]]; then + echo " Error: ${terror}" + fi + done <<< "$blocked_details" + echo "" + fi + + echo "## Re-orientation Instructions" + echo "" + echo "If you are an AI orchestrator reading this after context compaction:" + echo "1. Run \`supervisor-helper.sh status\` for full current state" + echo "2. Run \`supervisor-helper.sh pulse\` to trigger the next cycle" + echo "3. Check \`supervisor-helper.sh cron status\` to verify cron is running" + echo "4. Blocked/failed tasks may need manual intervention" + } > "$checkpoint_file" + + return 0 +} + ####################################### # Send a macOS notification for batch progress milestones # Called from pulse summary when notable progress occurs @@ -4768,10 +5274,12 @@ State Machine (worker lifecycle): -> blocked (needs human input / max retries exceeded) -> failed (dispatch failure / unrecoverable) -Post-PR Lifecycle (t128.8 - supervisor handles directly, no worker needed): - complete -> pr_review -> merging -> merged -> deploying -> deployed +Post-PR Lifecycle (t128.8 + t148 review triage): + complete -> pr_review -> review_triage -> merging -> merged -> deploying -> deployed Workers exit after PR creation. Supervisor detects complete tasks with PR URLs - and handles: CI wait, merge (squash), postflight, deploy, worktree cleanup. + and handles: CI wait, review thread triage, merge (squash), postflight, deploy, + worktree cleanup. Review triage checks unresolved threads via GraphQL and blocks + merge if human reviews or high-severity bot findings are unaddressed. Outcome Evaluation (3-tier): 1. Deterministic: FULL_LOOP_COMPLETE/TASK_COMPLETE signals, EXIT codes @@ -4816,6 +5324,10 @@ Options for 'evaluate': Options for 'pulse': --batch <batch_id> Only pulse tasks in this batch +Options for 'pr-lifecycle': + --dry-run Show what would happen without doing it + --skip-review-triage Skip review thread triage (merge without checking threads) + Options for 'cleanup': --dry-run Show what would be cleaned without doing it diff --git a/.agents/scripts/x-helper.sh b/.agents/scripts/x-helper.sh new file mode 100644 index 0000000000..16476f16f2 --- /dev/null +++ b/.agents/scripts/x-helper.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +# x-helper.sh - Fetch X/Twitter posts via fxtwitter API (no auth required) +# Part of aidevops framework: https://aidevops.sh +# +# Usage: +# x-helper.sh [command] [options] +# +# Commands: +# fetch <url> Fetch a tweet/post by URL (x.com or twitter.com) +# thread <url> Fetch a thread starting from the given post +# user <handle> Fetch recent posts from a user (limited) +# help Show this help +# +# Options: +# --json Output raw JSON +# --format md Output as markdown (default) +# --format text Output as plain text +# +# Uses fxtwitter.com API (no authentication required). +# Ref: https://github.com/FixTweet/FxTwitter + +set -euo pipefail + +readonly FXTWITTER_API="https://api.fxtwitter.com" + +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly RED='\033[0;31m' +readonly NC='\033[0m' + +print_success() { printf "${GREEN}%s${NC}\n" "$1"; } +print_warning() { printf "${YELLOW}%s${NC}\n" "$1"; } +print_error() { printf "${RED}%s${NC}\n" "$1" >&2; } + +extract_tweet_path() { + local url="$1" + # Extract username/status/id from various URL formats + # Handles: x.com/user/status/123, twitter.com/user/status/123 + local path + path=$(echo "$url" | sed -E 's|https?://(x\.com|twitter\.com)/||' | sed 's|[?#].*||') + echo "$path" +} + +extract_username() { + local url="$1" + echo "$url" | sed -E 's|https?://(x\.com|twitter\.com)/||' | sed 's|/.*||' | sed 's|^@||' +} + +cmd_fetch() { + local url="${1:-}" + local output_format="${2:-md}" + local raw_json="${3:-false}" + + if [[ -z "$url" ]]; then + print_error "URL required. Usage: x-helper.sh fetch <url>" + return 1 + fi + + local path + path=$(extract_tweet_path "$url") + + local response + response=$(curl -fsS --max-time 20 --retry 2 --retry-connrefused \ + "${FXTWITTER_API}/${path}" 2>/dev/null) || { + print_error "Failed to fetch tweet" + return 1 + } + + if [[ "$raw_json" == "true" ]]; then + echo "$response" | jq . 2>/dev/null || echo "$response" + return 0 + fi + + # Parse JSON and format output + local author author_handle text created_at likes retweets replies + author=$(echo "$response" | jq -r '.tweet.author.name // "Unknown"' 2>/dev/null || echo "Unknown") + author_handle=$(echo "$response" | jq -r '.tweet.author.screen_name // "unknown"' 2>/dev/null || echo "unknown") + text=$(echo "$response" | jq -r '.tweet.text // ""' 2>/dev/null || echo "") + created_at=$(echo "$response" | jq -r '.tweet.created_at // ""' 2>/dev/null || echo "") + likes=$(echo "$response" | jq -r '.tweet.likes // 0' 2>/dev/null || echo "0") + retweets=$(echo "$response" | jq -r '.tweet.retweets // 0' 2>/dev/null || echo "0") + replies=$(echo "$response" | jq -r '.tweet.replies // 0' 2>/dev/null || echo "0") + + if [[ "$output_format" == "text" ]]; then + echo "@${author_handle} (${author})" + echo "${created_at}" + echo "" + echo "${text}" + echo "" + echo "Likes: ${likes} | Retweets: ${retweets} | Replies: ${replies}" + else + # Markdown format + printf "**%s** (@%s)\n" "$author" "$author_handle" + printf "*%s*\n\n" "$created_at" + printf "%s\n\n" "$text" + printf "Likes: %s | Retweets: %s | Replies: %s\n" "$likes" "$retweets" "$replies" + printf "\nSource: %s\n" "$url" + fi + + return 0 +} + +cmd_thread() { + local url="${1:-}" + + if [[ -z "$url" ]]; then + print_error "URL required. Usage: x-helper.sh thread <url>" + return 1 + fi + + # fxtwitter doesn't have a native thread endpoint, but we can fetch + # the conversation by following reply chains + print_warning "Thread fetching is limited - fxtwitter returns individual posts" + print_warning "Fetching initial post..." + echo "" + cmd_fetch "$url" "md" "false" + + return 0 +} + +cmd_user() { + local handle="${1:-}" + + if [[ -z "$handle" ]]; then + print_error "Handle required. Usage: x-helper.sh user <handle>" + return 1 + fi + + # Remove @ prefix if present + handle="${handle#@}" + + local response + response=$(curl -fsS --max-time 20 --retry 2 --retry-connrefused \ + "${FXTWITTER_API}/${handle}" 2>/dev/null) || { + print_error "Failed to fetch user profile" + return 1 + } + + # Parse user info + local name followers following description + name=$(echo "$response" | jq -r '.user.name // "Unknown"' 2>/dev/null || echo "Unknown") + followers=$(echo "$response" | jq -r '.user.followers // 0' 2>/dev/null || echo "0") + following=$(echo "$response" | jq -r '.user.following // 0' 2>/dev/null || echo "0") + description=$(echo "$response" | jq -r '.user.description // ""' 2>/dev/null || echo "") + + printf "**%s** (@%s)\n" "$name" "$handle" + printf "Followers: %s | Following: %s\n\n" "$followers" "$following" + if [[ -n "$description" ]]; then + printf "%s\n" "$description" + fi + + return 0 +} + +cmd_help() { + sed -n '2,/^$/p' "$0" | sed 's/^# \?//' + return 0 +} + +main() { + local command="${1:-help}" + shift 2>/dev/null || true + + local output_format="md" + local raw_json="false" + local args=() + + while [[ $# -gt 0 ]]; do + case "$1" in + --json) raw_json="true"; shift ;; + --format) [[ $# -lt 2 ]] && { print_error "--format requires a value"; return 1; }; output_format="$2"; shift 2 ;; + *) args+=("$1"); shift ;; + esac + done + + case "$command" in + fetch) cmd_fetch "${args[0]:-}" "$output_format" "$raw_json" ;; + thread) cmd_thread "${args[0]:-}" ;; + user) cmd_user "${args[0]:-}" ;; + help|-h|--help) cmd_help ;; + *) + print_error "Unknown command: ${command}" + cmd_help + return 1 + ;; + esac +} + +main "$@" diff --git a/.agents/seo/backlink-checker.md b/.agents/seo/backlink-checker.md new file mode 100644 index 0000000000..08dcbee156 --- /dev/null +++ b/.agents/seo/backlink-checker.md @@ -0,0 +1,105 @@ +--- +description: Backlink monitoring and expired domain discovery for link reclamation +mode: subagent +tools: + read: true + write: true + edit: true + bash: true + glob: true + grep: true + webfetch: true + task: true +--- + +# Backlink & Expired Domain Checker + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Monitor backlinks, detect lost/broken links, find expired referring domains for purchase +- **Data Sources**: Ahrefs API, DataForSEO Backlinks API, WHOIS lookups +- **Helpers**: `scripts/seo-export-ahrefs.sh`, `scripts/seo-export-dataforseo.sh` (backlink data export) + +**Workflow**: Fetch backlink profile -> Identify lost/broken links -> Check domain expiry status -> Rank by DA/DR/traffic value -> Output purchase candidates + +<!-- AI-CONTEXT-END --> + +## Data Sources + +### Ahrefs API (Primary) + +> See `scripts/seo-export-ahrefs.sh` for the export implementation. + +Ahrefs endpoints used: +- `/v3/site-explorer/all-backlinks` - Full backlink list +- `/v3/site-explorer/backlinks-new-lost` - New/lost link changes +- `/v3/site-explorer/referring-domains` - Unique referring domains + +### DataForSEO Backlinks API (Alternative) + +> See `scripts/seo-export-dataforseo.sh` for the export implementation. + +DataForSEO endpoints: +- `/v3/backlinks/backlinks/live` - Live backlink data +- `/v3/backlinks/referring_domains/live` - Referring domains +- `/v3/backlinks/bulk_new_lost_backlinks/live` - Bulk new/lost + +## Expired Domain Detection + +### WHOIS Lookup + +```bash +# Check if a referring domain has expired +whois example-referrer.com | grep -i "expir" + +# Batch check (pipe from backlink export) +seo-helper.sh backlinks example.com --referring-domains-only | while read -r domain; do + expiry=$(whois "$domain" 2>/dev/null | grep -i "expiry\|expiration" | head -1) + echo "$domain: $expiry" +done +``` + +### Domain Availability Tools + +| Tool | Type | Notes | +|------|------|-------| +| `whois` CLI | Free | Rate-limited, varies by TLD | +| expired-domains.co | Web | Aggregates expired domain lists | +| expireddomains.net | Web | Filters by DA, backlinks, age | +| GoDaddy Auctions API | API | Auction/aftermarket domains | +| Namecheap API | API | Registration availability check | + +### GitHub Tools for Expired Domain Discovery + +- [peterprototypes/expired-domains](https://github.com/peterprototypes/expired-domains) - Rust CLI for checking domain expiry +- [Jeongseup/expired-domain-finder](https://github.com/Jeongseup/expired-domain-finder) - Python bulk checker + +## Reclamation Workflow + +1. **Export referring domains** with DR/DA scores +2. **Filter lost/broken** links from last 90 days +3. **WHOIS check** each lost referring domain +4. **Score candidates** by: + - Domain Rating (DR) or Domain Authority (DA) + - Number of backlinks the domain had + - Traffic estimate (Ahrefs/SimilarWeb) + - Registration cost vs. link value +5. **Output ranked list** of purchase candidates + +The full reclamation pipeline combines backlink export with WHOIS checks. Use the Ahrefs or DataForSEO export scripts to get lost backlinks, then pipe through WHOIS lookups above. + +## Integration with aidevops + +- Pairs with `seo/ahrefs.md` for API access +- Pairs with `seo/dataforseo.md` for alternative data source +- Pairs with `seo/domain-research.md` for DNS intelligence on candidates +- Results can feed into `seo/link-building.md` workflows + +## Related + +- `seo/ahrefs.md` - Ahrefs API integration +- `seo/dataforseo.md` - DataForSEO API integration +- `seo/domain-research.md` - DNS reconnaissance +- `seo/link-building.md` - Link-building strategies diff --git a/.agents/subagent-index.toon b/.agents/subagent-index.toon index ae76630ac9..3b26dbb473 100644 --- a/.agents/subagent-index.toon +++ b/.agents/subagent-index.toon @@ -99,6 +99,7 @@ todo-ready.sh,Show tasks with no open blockers ralph-loop-helper.sh,Iterative AI development loops full-loop-helper.sh,End-to-end development loop (task to PR to deploy) session-review-helper.sh,Gather session context for review +session-checkpoint-helper.sh,Persist session state to survive context compaction mail-helper.sh,TOON-based inter-agent mailbox system memory-helper.sh,Cross-session memory (SQLite FTS5) memory-embeddings-helper.sh,Semantic memory search with vector embeddings (opt-in) diff --git a/.agents/tools/accounts/subscription-audit.md b/.agents/tools/accounts/subscription-audit.md new file mode 100644 index 0000000000..787f7bd061 --- /dev/null +++ b/.agents/tools/accounts/subscription-audit.md @@ -0,0 +1,147 @@ +--- +description: Subscription audit - discover, track, and optimize recurring payments +mode: subagent +tools: + read: true + write: true + edit: true + bash: true + glob: true + grep: true + webfetch: true + task: true +--- + +# Subscription Audit + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Discover all active subscriptions, calculate total spend, identify savings +- **Command**: `/subscription-audit` or `@subscription-audit` +- **Data Sources**: Email receipts, bank statements (CSV), manual inventory + +**Quick start**: Use the `/subscription-audit` command to start an interactive audit. The agent guides you through discovery, analysis, and optimization phases. + +<!-- AI-CONTEXT-END --> + +## Audit Workflow + +### Phase 1: Discovery + +Identify all recurring charges from multiple sources: + +#### Email Receipt Scanning + +Search email for common subscription receipt patterns: + +```bash +# Gmail search queries (via IMAP or Gmail API) +# Recurring payment receipts +"receipt" OR "invoice" OR "subscription" OR "renewal" OR "billing" + +# Common SaaS senders +from:(noreply@github.com OR billing@stripe.com OR receipts@paddle.com) + +# Subscription keywords +"your subscription" OR "monthly charge" OR "annual renewal" OR "auto-renewal" +``` + +#### Bank Statement Import + +Import CSV bank statements (auto-detects format for Chase, Amex, Barclays, Monzo, Revolut, generic CSV). The agent identifies recurring patterns (same merchant, similar amount, regular interval). + +#### Manual Inventory + +Common categories to check: + +| Category | Examples | +|----------|----------| +| **Dev Tools** | GitHub, GitLab, JetBrains, Vercel, Netlify, Railway, Fly.io | +| **Cloud/Infra** | AWS, GCP, Azure, DigitalOcean, Hetzner, Cloudflare | +| **AI/ML** | OpenAI, Anthropic, Google AI, Replicate, HuggingFace | +| **Domains** | Namecheap, Cloudflare, GoDaddy, Porkbun | +| **Email** | Google Workspace, Fastmail, Proton, Mailgun, SendGrid | +| **Monitoring** | Datadog, Sentry, PagerDuty, UptimeRobot, BetterStack | +| **Security** | 1Password, Bitwarden, NordVPN, Tailscale | +| **Productivity** | Notion, Linear, Slack, Zoom, Figma, Miro | +| **Media** | Spotify, YouTube Premium, Netflix, Audible | +| **Storage** | iCloud, Dropbox, Backblaze, Wasabi | + +### Phase 2: Analysis + +The analysis phase generates a report that includes: +- **Monthly total**: Sum of all recurring charges +- **Annual projection**: Monthly * 12 + annual-only subscriptions +- **Category breakdown**: Spend per category +- **Unused detection**: Subscriptions with no recent login/API activity +- **Duplicate detection**: Multiple tools serving same purpose +- **Price increase alerts**: Charges that increased vs. last period + +### Phase 3: Optimization + +| Strategy | Savings Potential | +|----------|-------------------| +| Cancel unused subscriptions | 10-30% | +| Downgrade overprovisioned tiers | 5-15% | +| Switch to annual billing | 15-20% per service | +| Consolidate overlapping tools | 10-25% | +| Use open-source alternatives | Variable | +| Negotiate enterprise discounts | 10-40% | + +### Open-Source Alternatives + +| Paid Tool | Free Alternative | +|-----------|------------------| +| GitHub Copilot | Claude Code (free tier), Cody | +| Notion | Obsidian, Logseq | +| Slack | Mattermost, Zulip | +| Datadog | Grafana + Prometheus | +| PagerDuty | Grafana OnCall | +| 1Password | Bitwarden (self-hosted) | +| Vercel | Coolify (self-hosted) | +| Linear | Plane (self-hosted) | + +## Report Format + +```text +Subscription Audit Report +========================= +Generated: 2026-02-07 +Period: Monthly + +Active Subscriptions: 23 +Monthly Total: $847.50 +Annual Projection: $10,170.00 + +By Category: + Cloud/Infra: $312.00 (36.8%) + Dev Tools: $189.00 (22.3%) + AI/ML: $145.00 (17.1%) + Productivity: $89.50 (10.6%) + Security: $52.00 (6.1%) + Other: $60.00 (7.1%) + +Recommendations: +1. [UNUSED] Datadog Pro - no dashboards viewed in 60 days ($99/mo) +2. [DUPLICATE] Both Sentry and BetterStack for error tracking ($38/mo overlap) +3. [DOWNGRADE] GitHub Team → Free (only 2 private repos) ($4/mo) +4. [ANNUAL] Switch Vercel to annual billing (save $36/yr) + +Potential Monthly Savings: $141.00 (16.6%) +``` + +## Data Storage + +Subscription data stored in SQLite at `~/.aidevops/.agent-workspace/work/subscriptions/subscriptions.db`. + +Schema: +- `subscriptions` - Active subscriptions (name, category, amount, interval, provider) +- `audit_history` - Past audit snapshots for trend tracking +- `recommendations` - Generated recommendations with status (applied/dismissed) + +## Related + +- `accounts.md` - Financial operations agent +- `tools/credentials/api-key-setup.md` - API key management (related spend) diff --git a/.agents/tools/context/rapidfuzz.md b/.agents/tools/context/rapidfuzz.md new file mode 100644 index 0000000000..ac482ced8f --- /dev/null +++ b/.agents/tools/context/rapidfuzz.md @@ -0,0 +1,121 @@ +--- +description: RapidFuzz - fast fuzzy string matching library for Python and C++ +mode: subagent +tools: + read: true + write: false + edit: false + bash: true + glob: true + grep: true +--- + +# RapidFuzz - Fuzzy String Matching + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Fast fuzzy string matching (Levenshtein, Jaro-Winkler, token-based) +- **Install**: `pip install rapidfuzz` or `conda install -c conda-forge rapidfuzz` +- **Repo**: https://github.com/rapidfuzz/RapidFuzz (2.8k+ stars, MIT) +- **Docs**: https://rapidfuzz.github.io/RapidFuzz/ + +**When to use**: Deduplication, typo-tolerant search, record linkage, autocomplete, entity matching. 10-100x faster than `fuzzywuzzy` (C++ backend, no Python-only fallback). + +<!-- AI-CONTEXT-END --> + +## Core Functions + +### Simple Ratio (character-level similarity) + +```python +from rapidfuzz import fuzz + +fuzz.ratio("fuzzy wuzzy", "fuzzy wuzzy") # 100.0 +fuzz.ratio("fuzzy wuzzy", "fzzy wzzy") # ~84.2 +``` + +### Partial Ratio (substring matching) + +```python +fuzz.partial_ratio("fuzzy wuzzy", "wuzzy") # 100.0 +``` + +### Token Sort Ratio (order-independent) + +```python +fuzz.token_sort_ratio("New York Mets", "Mets New York") # 100.0 +``` + +### Token Set Ratio (handles duplicates and extras) + +```python +fuzz.token_set_ratio( + "fuzzy wuzzy was a bear", + "fuzzy fuzzy was a bear" +) # High score despite differences +``` + +## Process Module (batch matching) + +```python +from rapidfuzz import process + +choices = ["Atlanta Falcons", "New York Jets", "New York Giants", "Dallas Cowboys"] + +# Best match +process.extractOne("new york jets", choices) +# ('New York Jets', 100.0, 1) + +# Top N matches +process.extract("new york jets", choices, limit=2) +# [('New York Jets', 100.0, 1), ('New York Giants', 78.57, 2)] + +# Score cutoff +process.extract("new york jets", choices, score_cutoff=80) +``` + +## Distance Functions + +```python +from rapidfuzz.distance import Levenshtein, Jaro, JaroWinkler + +Levenshtein.distance("kitten", "sitting") # 3 +Levenshtein.normalized_similarity("kitten", "sitting") # ~0.571 + +Jaro.similarity("martha", "marhta") # ~0.944 +JaroWinkler.similarity("martha", "marhta") # ~0.961 +``` + +## Performance Tips + +- Use `score_cutoff` parameter to skip low-scoring comparisons early +- Use `process.cdist()` for pairwise distance matrices (NumPy integration) +- Pre-process strings with `rapidfuzz.utils.default_process` (lowercase, strip) +- For large datasets (>10k items), use `process.extract` with `workers=-1` for parallelism + +## Common Patterns in aidevops + +```python +# Fuzzy-match user input to known commands +from rapidfuzz import process +commands = ["deploy", "status", "update", "rollback", "logs"] +match, score, idx = process.extractOne(user_input, commands) +if score > 80: + execute(match) + +# Deduplicate similar entries +from rapidfuzz import fuzz +def deduplicate(items, threshold=85): + unique = [] + for item in items: + if not any(fuzz.ratio(item, u) > threshold for u in unique): + unique.append(item) + return unique +``` + +## Related + +- `tools/context/mcp-discovery.md` - Uses fuzzy matching for tool search +- `memory-helper.sh` - Could use RapidFuzz for similar memory deduplication diff --git a/.agents/tools/document/document-extraction.md b/.agents/tools/document/document-extraction.md new file mode 100644 index 0000000000..6a69dd8397 --- /dev/null +++ b/.agents/tools/document/document-extraction.md @@ -0,0 +1,200 @@ +--- +description: Privacy-preserving document extraction with Docling, ExtractThinker, and Presidio +mode: subagent +tools: + read: true + write: true + edit: true + bash: true + glob: true + grep: true + webfetch: true + task: true +--- + +# Document Extraction + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Extract structured data from documents (PDF, DOCX, images) with PII redaction +- **Stack**: Docling (parsing) + ExtractThinker (LLM extraction) + Presidio (PII detection) +- **Privacy**: Fully local processing via Ollama or Cloudflare Workers AI +- **PRD**: `todo/tasks/prd-document-extraction.md` + +**Status**: Planned (see `todo/tasks/prd-document-extraction.md` for PRD). The Python components below are available for direct use; a CLI helper script is not yet implemented. + +<!-- AI-CONTEXT-END --> + +## Architecture + +```text +Document Input (PDF/DOCX/Image/HTML) + │ + ┌────┴────┐ + │ Docling │ ← Document parsing (layout, tables, OCR) + └────┬────┘ + │ + ┌────┴──────────┐ + │ ExtractThinker │ ← LLM-powered structured extraction + └────┬──────────┘ + │ + ┌────┴────────┐ + │ Presidio │ ← PII detection and redaction (optional) + └────┬────────┘ + │ + Structured Output (JSON/CSV/Markdown) +``` + +## Components + +### Docling (Document Parsing) + +IBM's document conversion library. Handles complex layouts, tables, and OCR. + +```bash +pip install docling + +# Python usage +from docling.document_converter import DocumentConverter +converter = DocumentConverter() +result = converter.convert("document.pdf") +print(result.document.export_to_markdown()) +``` + +- **Formats**: PDF, DOCX, PPTX, XLSX, HTML, images, AsciiDoc +- **Features**: Table extraction, OCR (EasyOCR/Tesseract), layout analysis +- **Repo**: https://github.com/DS4SD/docling + +### ExtractThinker (LLM Extraction) + +Pydantic-based structured extraction using LLMs. + +```python +from extract_thinker import Extractor, Contract +from pydantic import BaseModel + +class Invoice(BaseModel): + vendor: str + date: str + total: float + items: list[dict] + +extractor = Extractor() +extractor.load_document_loader("docling") +extractor.load_llm("ollama/llama3.2") # Local model + +result = extractor.extract("invoice.pdf", Invoice) +print(result.model_dump_json(indent=2)) +``` + +- **Repo**: https://github.com/enoch3712/ExtractThinker +- **LLM backends**: Ollama (local), OpenAI, Anthropic, Google, Cloudflare Workers AI + +### Presidio (PII Redaction) + +Microsoft's PII detection and anonymization. + +```python +from presidio_analyzer import AnalyzerEngine +from presidio_anonymizer import AnonymizerEngine + +analyzer = AnalyzerEngine() +anonymizer = AnonymizerEngine() + +text = "John Smith's SSN is 123-45-6789" +results = analyzer.analyze(text=text, language="en") +anonymized = anonymizer.anonymize(text=text, analyzer_results=results) +print(anonymized.text) # "<PERSON>'s SSN is <US_SSN>" +``` + +- **Entities**: PERSON, EMAIL, PHONE, SSN, CREDIT_CARD, IBAN, IP_ADDRESS, etc. +- **Repo**: https://github.com/microsoft/presidio + +## Extraction Schemas (Templates) + +> These are example/template schemas for common document types. Customize for your project. + +### Invoice + +```python +class Invoice(BaseModel): + vendor_name: str + vendor_address: str | None + invoice_number: str + invoice_date: str + due_date: str | None + subtotal: float + tax: float | None + total: float + currency: str + line_items: list[LineItem] +``` + +### Receipt + +```python +class Receipt(BaseModel): + merchant: str + date: str + total: float + payment_method: str | None + items: list[ReceiptItem] +``` + +### Contract + +```python +class ContractSummary(BaseModel): + parties: list[str] + effective_date: str + termination_date: str | None + key_terms: list[str] + obligations: list[str] +``` + +## Privacy Modes + +| Mode | LLM | PII Handling | Use Case | +|------|-----|-------------|----------| +| **Local** | Ollama (llama3.2) | Presidio redact before LLM | Maximum privacy | +| **Edge** | Cloudflare Workers AI | Presidio redact before API | Good privacy, faster | +| **Cloud** | OpenAI/Anthropic | Presidio redact before API | Best quality | +| **None** | Any | No redaction | Non-sensitive documents | + +## Installation + +```bash +# Core +pip install docling extract-thinker + +# PII (optional) +pip install presidio-analyzer presidio-anonymizer +python -m spacy download en_core_web_lg + +# Local LLM (optional) +brew install ollama && ollama pull llama3.2 + +# OCR backends (optional) +pip install easyocr # or: brew install tesseract +``` + +## When to Use (vs Unstract) + +| Feature | This Stack (Docling+ExtractThinker) | Unstract MCP | +|---------|-------------------------------------|-------------| +| **Privacy** | Full local processing via Ollama | Cloud or self-hosted | +| **Schema control** | Pydantic models, custom schemas | Pre-built extractors | +| **PII redaction** | Built-in via Presidio | Manual | +| **Setup** | pip install, no server | Docker/server required | +| **Best for** | Custom extraction pipelines | Quick document processing | + +Use this stack when you need custom schemas, PII redaction, or fully local processing. Use Unstract when you need quick setup with pre-built extractors. + +## Related + +- `tools/document/pandoc-helper.sh` - Document format conversion +- `tools/document/mineru.md` - Alternative PDF extraction (MinerU) +- `services/document-processing/unstract.md` - Self-hosted document processing (alternative) +- `todo/tasks/prd-document-extraction.md` - Full PRD diff --git a/.agents/tools/terminal/terminal-optimization.md b/.agents/tools/terminal/terminal-optimization.md new file mode 100644 index 0000000000..f99f107436 --- /dev/null +++ b/.agents/tools/terminal/terminal-optimization.md @@ -0,0 +1,182 @@ +--- +description: Terminal environment optimization - shell config, aliases, performance tuning +mode: subagent +tools: + read: true + write: true + edit: true + bash: true + glob: true + grep: true +--- + +# Terminal Optimization + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Audit and optimize terminal environment (shell startup, aliases, PATH, tools) +- **Command**: `/terminal-optimize` or `@terminal-optimization` +- **Scope**: Shell config (.bashrc/.zshrc), PATH dedup, tool recommendations, startup profiling + +**Quick audit**: + +```bash +# Profile shell startup time +time zsh -i -c exit # or: time bash -i -c exit + +# Count PATH entries (duplicates waste lookup time) +echo "$PATH" | tr ':' '\n' | sort | uniq -c | sort -rn | head + +# Check shell config size +wc -l ~/.zshrc ~/.bashrc ~/.profile 2>/dev/null +``` + +<!-- AI-CONTEXT-END --> + +## Optimization Checklist + +### 1. Shell Startup Time + +Target: <200ms for interactive shell. + +```bash +# Profile zsh startup +zsh -xvs 2>&1 | ts -i '%.s' | head -50 + +# Profile bash startup +bash --norc --noprofile -c 'time bash -i -c exit' + +# Common slow culprits: +# - nvm (Node Version Manager) - lazy-load instead +# - conda init - lazy-load instead +# - rbenv/pyenv init - lazy-load instead +# - oh-my-zsh plugins - reduce to essentials +# - compinit (zsh completion) - cache with zcompdump +``` + +### Lazy-Loading Pattern + +```bash +# Instead of: eval "$(nvm init)" +# Use lazy-load: +nvm() { + unset -f nvm node npm npx + export NVM_DIR="$HOME/.nvm" + [ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh" + nvm "$@" +} +``` + +### 2. PATH Optimization + +```bash +# Deduplicate PATH +typeset -U PATH # zsh built-in +# or for bash: +PATH=$(echo "$PATH" | tr ':' '\n' | awk '!seen[$0]++' | tr '\n' ':' | sed 's/:$//') + +# Remove non-existent directories +PATH=$(echo "$PATH" | tr ':' '\n' | while read -r dir; do + [ -d "$dir" ] && echo "$dir" +done | tr '\n' ':' | sed 's/:$//') +``` + +### 3. Modern Tool Replacements + +| Classic | Modern | Speedup | Install | +|---------|--------|---------|---------| +| `find` | `fd` | 5-10x | `brew install fd` | +| `grep` | `ripgrep (rg)` | 5-20x | `brew install ripgrep` | +| `cat` | `bat` | Syntax highlighting | `brew install bat` | +| `ls` | `eza` | Better output | `brew install eza` | +| `du` | `dust` | Visual tree | `brew install dust` | +| `top` | `btop` | Better UI | `brew install btop` | +| `diff` | `delta` | Syntax-aware | `brew install git-delta` | +| `sed` | `sd` | Simpler syntax | `brew install sd` | +| `curl` | `xh` | Colorized HTTP | `brew install xh` | +| `man` | `tldr` | Quick examples | `brew install tldr` | + +### 4. Shell Aliases + +```bash +# Git shortcuts +alias gs='git status' +alias gd='git diff' +alias gl='git log --oneline -20' +alias gp='git pull --rebase' + +# Navigation +alias ..='cd ..' +alias ...='cd ../..' +alias ll='eza -la --git' + +# Safety +alias rm='rm -i' +alias cp='cp -i' +alias mv='mv -i' + +# aidevops +alias ad='aidevops' +alias ads='aidevops status' +alias adu='aidevops update' +``` + +### 5. Completion and History + +```bash +# Zsh: better history search +bindkey '^R' history-incremental-search-backward +HISTSIZE=50000 +SAVEHIST=50000 +setopt SHARE_HISTORY +setopt HIST_IGNORE_DUPS +setopt HIST_IGNORE_SPACE + +# fzf integration (fuzzy history search) +# brew install fzf && $(brew --prefix)/opt/fzf/install +``` + +### 6. Terminal Multiplexer + +```bash +# tmux essentials +brew install tmux + +# Minimal .tmux.conf +set -g mouse on +set -g history-limit 50000 +set -g default-terminal "screen-256color" +bind | split-window -h -c "#{pane_current_path}" +bind - split-window -v -c "#{pane_current_path}" +``` + +## Audit Report Format + +When running `/terminal-optimize`, output: + +```text +Terminal Optimization Report +============================ +Shell: zsh 5.9 +Startup time: 342ms (target: <200ms) +PATH entries: 23 (4 duplicates, 2 missing) +Config size: .zshrc 187 lines + +Issues Found: +1. [SLOW] nvm init adds 180ms - suggest lazy-load +2. [DUP] /usr/local/bin appears 3x in PATH +3. [MISSING] /opt/homebrew/bin not in PATH but Homebrew installed +4. [TOOL] Using grep instead of ripgrep (5-20x slower) + +Recommendations: +1. Lazy-load nvm (saves ~180ms) +2. Deduplicate PATH (saves ~5ms lookup) +3. Install: fd, ripgrep, bat, eza +4. Add fzf for fuzzy history search +``` + +## Related + +- `tools/terminal/terminal-title.md` - Terminal title management diff --git a/.agents/tools/voice/transcription.md b/.agents/tools/voice/transcription.md new file mode 100644 index 0000000000..50f19284ca --- /dev/null +++ b/.agents/tools/voice/transcription.md @@ -0,0 +1,204 @@ +--- +description: Audio/video transcription with local and cloud models +mode: subagent +tools: + read: true + write: true + edit: true + bash: true + glob: true + grep: true + webfetch: true + task: true +--- + +# Audio/Video Transcription + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: Transcribe audio/video from YouTube, URLs, or local files +- **Helper**: `transcription-helper.sh [transcribe|models|configure] [options]` +- **Default model**: Whisper Large v3 Turbo (best speed/accuracy tradeoff) +- **Dependencies**: `yt-dlp` (YouTube), `ffmpeg` (audio extraction), `faster-whisper` or `whisper.cpp` (local) + +**Quick Commands**: + +```bash +# Transcribe YouTube video +transcription-helper.sh transcribe "https://youtu.be/dQw4w9WgXcQ" + +# Transcribe local file +transcription-helper.sh transcribe recording.mp3 + +# Transcribe with specific model +transcription-helper.sh transcribe recording.mp3 --model large-v3-turbo + +# List available models +transcription-helper.sh models +``` + +<!-- AI-CONTEXT-END --> + +## Input Sources + +| Source | Detection | Extraction | +|--------|-----------|------------| +| YouTube URL | `youtu.be/` or `youtube.com/watch` | `yt-dlp -x --audio-format wav` | +| Direct media URL | HTTP(S) with media extension | `curl` + `ffmpeg` if video | +| Local audio | `.wav`, `.mp3`, `.flac`, `.ogg`, `.m4a` | Direct input | +| Local video | `.mp4`, `.mkv`, `.webm`, `.avi` | `ffmpeg -i input -vn -acodec pcm_s16le` | + +## Local Models (Whisper Family) + +### via faster-whisper (Recommended) + +CTranslate2-based, 4x faster than OpenAI Whisper with same accuracy. +See `voice-bridge.py:99-115` for the repo's `FasterWhisperSTT` implementation. + +```bash +pip install faster-whisper +``` + +Official usage: https://github.com/SYSTRAN/faster-whisper#usage + +### via whisper.cpp (C++ native) + +Optimized for Apple Silicon and CPU inference. + +```bash +# Build from source (no Homebrew formula) +git clone https://github.com/ggml-org/whisper.cpp.git +cd whisper.cpp && cmake -B build && cmake --build build -j --config Release + +# Download model +sh ./models/download-ggml-model.sh large-v3-turbo + +# Transcribe +./build/bin/whisper-cli -f audio.wav -otxt -osrt +``` + +### Model Comparison + +| Model | Size | Speed | Accuracy | Notes | +|-------|------|-------|----------|-------| +| Tiny | 75MB | 9.5 | 6.0-6.5 | Draft/preview only | +| Base | 142MB | 8.5 | 7.2-7.5 | Quick transcription | +| Small | 461MB | 7.0 | 8.5 | Good balance | +| Medium | 1.5GB | 5.0 | 9.0 | Solid quality | +| Large v2 | 2.9GB | 3.0 | 9.6 | High quality | +| Large v3 | 2.9GB | 3.0 | 9.8 | Best quality | +| **Large v3 Turbo** | **1.5GB** | **7.5** | **9.7** | **Recommended default** | +| Large v3 Turbo Q | 547MB | 7.5 | 9.5 | Quantized, smaller | + +### Other Local Models + +| Model | Size | Speed | Accuracy | Notes | +|-------|------|-------|----------|-------| +| NVIDIA Parakeet V2 | 474MB | 9.9 | 9.4 | English-only, fastest | +| NVIDIA Parakeet V3 | 494MB | 9.9 | 9.4 | Multilingual, experimental | +| Apple Speech | Built-in | 9.0 | 9.0 | macOS 26+, on-device | + +## Cloud APIs + +| Provider | Model | Accuracy | Speed | Cost | +|----------|-------|----------|-------|------| +| **Groq** | Whisper Large v3 Turbo | 9.6 | Lightning | Free tier available | +| **ElevenLabs** | Scribe v2 | 9.9 | Fast | Pay per minute | +| **ElevenLabs** | Scribe v1 | 9.8 | Fast | Pay per minute | +| **Mistral** | Voxtral Mini | 9.7 | Fast | Pay per token | +| **Deepgram** | Nova-2 | 9.5 | Real-time | Pay per minute | +| **Deepgram** | Nova-3 Medical | 9.6 | Real-time | English-only, clinical | +| **OpenAI** | Whisper API | 9.5 | Fast | $0.006/min | +| **Google** | Gemini 3 Pro | 9.7 | Fast | Multimodal input | +| **Google** | Gemini 3 Flash | 9.5 | Fastest | Low latency | +| **Soniox** | stt-async-v3 | 9.6 | Async | Batch processing | + +### Groq (Fastest Cloud) + +```bash +curl https://api.groq.com/openai/v1/audio/transcriptions \ + -H "Authorization: Bearer ${GROQ_API_KEY}" \ + -H "Content-Type: multipart/form-data" \ + -F "file=@audio.wav" \ + -F "model=whisper-large-v3" \ + -F "response_format=verbose_json" +``` + +Docs: https://console.groq.com/docs/speech-text + +### OpenAI Whisper API + +```bash +curl https://api.openai.com/v1/audio/transcriptions \ + -H "Authorization: Bearer ${OPENAI_API_KEY}" \ + -F "file=@audio.wav" \ + -F "model=whisper-1" \ + -F "response_format=srt" +``` + +Docs: https://platform.openai.com/docs/api-reference/audio/createTranscription + +### ElevenLabs Scribe + +```bash +curl -X POST "https://api.elevenlabs.io/v1/speech-to-text" \ + -H "xi-api-key: ${ELEVENLABS_API_KEY}" \ + -F "file=@audio.wav" \ + -F "model_id=scribe_v1" +``` + +Docs: https://elevenlabs.io/docs/api-reference/speech-to-text + +## Output Formats + +| Format | Extension | Use Case | +|--------|-----------|----------| +| Plain text | `.txt` | Reading, search indexing | +| SRT | `.srt` | Video subtitles | +| VTT | `.vtt` | Web video subtitles | +| JSON | `.json` | Programmatic access, timestamps | +| TSV | `.tsv` | Spreadsheet analysis | + +## Transcription Pipeline + +```text +Input Source + │ + ├── YouTube URL ──→ yt-dlp -x ──→ audio.wav + ├── Video URL ────→ curl + ffmpeg ──→ audio.wav + ├── Video file ───→ ffmpeg -vn ──→ audio.wav + └── Audio file ───→ (direct) ──→ audio.wav + │ + Model Selection + │ + ┌─────────┴─────────┐ + │ │ + Local Model Cloud API + (faster-whisper) (Groq/OpenAI/etc) + │ │ + └─────────┬──────────┘ + │ + Output Format + (txt/srt/vtt/json) +``` + +## Dependencies + +```bash +# Core +brew install yt-dlp ffmpeg # macOS +apt install yt-dlp ffmpeg # Ubuntu/Debian + +# Local inference (pick one) +pip install faster-whisper # Python (recommended) +# whisper.cpp: build from source (see above) +``` + +## Related + +- `tools/voice/voice-models.md` - TTS models for speech generation +- `tools/voice/speech-to-speech.md` - Full voice pipeline +- `tools/content/summarize.md` - Can summarize transcribed content +- `voice-helper.sh` - CLI for voice operations diff --git a/.agents/tools/voice/voice-models.md b/.agents/tools/voice/voice-models.md new file mode 100644 index 0000000000..7ab980055c --- /dev/null +++ b/.agents/tools/voice/voice-models.md @@ -0,0 +1,103 @@ +--- +description: Voice AI models for speech generation (TTS) and transcription (STT) +mode: subagent +tools: + read: true + write: false + edit: false + bash: true + glob: true + grep: true + webfetch: true + task: true +--- + +# Voice AI Models + +<!-- AI-CONTEXT-START --> + +## Quick Reference + +- **Purpose**: TTS (text-to-speech) and STT (speech-to-text) model selection and usage +- **Local TTS**: EdgeTTS (default), macOS Say, FacebookMMS — see `voice-bridge.py:133-238` +- **Local STT**: See `tools/voice/transcription.md` for full transcription guide +- **Cloud APIs**: ElevenLabs, OpenAI TTS — see official API docs + +**When to use**: Voice interfaces, content narration, accessibility, voice cloning, podcast generation, phone bots (with Twilio via `speech-to-speech.md`). + +<!-- AI-CONTEXT-END --> + +## Text-to-Speech (TTS) Models + +### Implemented in This Repo + +The voice bridge (`voice-bridge.py`) implements three TTS engines: + +#### EdgeTTS (Default) + +Microsoft Edge TTS via `edge-tts` package. See `voice-bridge.py:133-179`. + +- Free, no API key needed +- 300+ voices across 70+ languages +- Streaming support, adjustable rate +- Default voice: `en-GB-SoniaNeural` + +```bash +# Use via voice bridge +voice-helper.sh talk +``` + +#### macOS Say + +Native macOS speech synthesis. See `voice-bridge.py:182-205`. + +- Built-in, zero dependencies +- Default voice: `Samantha` +- macOS only + +#### FacebookMMS TTS + +Meta's Massively Multilingual Speech. See `voice-bridge.py:208-238`. + +- 1,100+ languages +- Requires `transformers` package +- CPU-friendly + +### Other Local Models (Not Implemented) + +These are available but not integrated into the voice bridge: + +| Model | Notes | Docs | +|-------|-------|------| +| Qwen3-TTS | Multilingual, voice cloning | https://github.com/QwenLM/Qwen3-TTS | +| Piper TTS | Lightweight, CPU-friendly, 100+ voices | https://github.com/rhasspy/piper | +| Bark (Suno) | Expressive, non-speech sounds | https://github.com/suno-ai/bark | +| Coqui TTS | Multi-model toolkit, voice cloning | https://github.com/coqui-ai/TTS | + +### Cloud APIs (Not Implemented) + +These require API keys and are not integrated into the voice bridge: + +| Provider | Docs | +|----------|------| +| ElevenLabs | https://elevenlabs.io/docs/api-reference/text-to-speech | +| OpenAI TTS | https://platform.openai.com/docs/api-reference/audio/createSpeech | +| Hugging Face Inference | https://huggingface.co/docs/api-inference/tasks/text-to-speech | + +## Model Selection Guide + +| Priority | Implemented | Not Yet Integrated | +|----------|-------------|-------------------| +| Default (free) | EdgeTTS | — | +| macOS native | macOS Say | — | +| Multilingual | FacebookMMS | Qwen3-TTS | +| Voice clone | — | Qwen3-TTS, ElevenLabs | +| Expressiveness | — | Bark | +| CPU-only | All three | Piper | +| Highest quality | — | ElevenLabs | + +## Related + +- `tools/voice/speech-to-speech.md` - Full voice pipeline (VAD+STT+LLM+TTS) +- `tools/voice/transcription.md` - STT/transcription models +- `voice-helper.sh` - CLI for voice operations diff --git a/.agents/workflows/plans.md b/.agents/workflows/plans.md index fe7f76cb76..ff70421c15 100644 --- a/.agents/workflows/plans.md +++ b/.agents/workflows/plans.md @@ -730,6 +730,52 @@ Uncommitted TODO changes are invisible to other sessions, agents, and the `/read | Status update | `chore: update task t{NNN} status` | | Plan creation | `chore: add plan for {title}` | +## GitHub Issue Sync + +Tasks and GitHub issues MUST stay in sync with matching identifiers. + +### Convention + +- **GitHub issue titles** MUST be prefixed with their TODO.md task ID: `t{NNN}: {title}` +- **TODO.md tasks** MUST reference their GitHub issue: `ref:GH#{NNN}` +- Both directions must be maintained whenever either is created or updated + +### When Creating a GitHub Issue + +```bash +# Always prefix with t-number +gh issue create --title "t146: bug: supervisor no_pr retry counter non-functional" ... +``` + +### When Creating a TODO.md Task from an Issue + +```markdown +- [ ] t146 bug: supervisor no_pr retry counter #bugfix ~15m logged:2026-02-07 ref:GH#439 +``` + +### When Creating Both Together + +1. Assign the next available t-number +2. Create the GitHub issue with `t{NNN}: ` prefix in the title +3. Add the task to TODO.md with `ref:GH#{issue_number}` +4. Commit and push TODO.md immediately + +### Automated Enforcement + +The supervisor's `update_todo_on_complete()` and `send_task_notification()` functions should maintain this sync. When the supervisor creates issues or updates TODO.md, it must: + +1. Check if a matching GitHub issue exists (search by `t{NNN}` in title) +2. If not, create one with the `t{NNN}: ` prefix +3. If the TODO.md task lacks `ref:GH#`, add it after issue creation +4. When closing a task, close the matching GitHub issue with a comment + +### Why This Matters + +Without consistent t-number prefixes on issues, there's no way to: +- Quickly find the GitHub issue for a TODO.md task (or vice versa) +- Automate status sync between the two systems +- Track which issues correspond to which batch of work + ## Integration with Other Workflows | Workflow | Integration | diff --git a/.agents/workflows/session-manager.md b/.agents/workflows/session-manager.md index ecaf1dd766..f7f077c6e6 100644 --- a/.agents/workflows/session-manager.md +++ b/.agents/workflows/session-manager.md @@ -243,6 +243,56 @@ Suggestions: --- ``` +## Compaction Resilience (Long Autonomous Sessions) + +During long autonomous sessions (1h+), context compaction can cause loss of task state. Use the checkpoint system to persist state to disk. + +### Checkpoint Workflow + +```bash +# After completing each task, save checkpoint: +session-checkpoint-helper.sh save \ + --task "t135.9" \ + --next "t135.11,t014,t025" \ + --worktree "/path/to/worktree" \ + --batch "batch2-quality" \ + --note "Completed trap cleanup for 29 scripts" \ + --elapsed "90" \ + --target "240" + +# Before starting any new task (especially after compaction), reload: +session-checkpoint-helper.sh load + +# Check if checkpoint is stale: +session-checkpoint-helper.sh status +``` + +### When to Checkpoint + +| Event | Action | +|-------|--------| +| Task completed | `save` with updated --task and --next | +| PR created/merged | `save` with --note describing PR state | +| Batch of files committed | `save` with --note listing what changed | +| Before large operation | `save` as recovery point | +| After context compaction | `load` to re-orient | + +### Self-Prompting Loop Pattern + +For autonomous multi-hour sessions, follow this loop after each task: + +1. Mark task complete in TODO.md +2. Save checkpoint to disk +3. Re-read checkpoint file (forces re-orientation after compaction) +4. Read TODO.md for next task +5. Start next task + +This ensures the agent always has current state even if context was compacted between steps 2 and 3. + +### Continuation Prompt Generation + +If a session must be resumed manually (new conversation), the checkpoint file at `~/.aidevops/.agent-workspace/tmp/session-checkpoint.md` contains enough state to generate a continuation prompt. The user can paste it or the agent can read it on startup. + ## Related **AGENTS.md is the single source of truth for agent behavior.** This document is supplementary and defers to AGENTS.md where they differ.