diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6af8dca9..b3dcfe811 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,3 +132,4 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} + diff --git a/.github/workflows/issue-triage-runner.yml b/.github/workflows/issue-triage-runner.yml new file mode 100644 index 000000000..e2ca17771 --- /dev/null +++ b/.github/workflows/issue-triage-runner.yml @@ -0,0 +1,82 @@ +name: Issue Triage + +on: + issues: + types: [opened, reopened] + +# Two-job pattern: triage runs with read-only issue access and produces a +# staged JSON artifact; apply runs with write access and calls safe-output. +jobs: + triage: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: read + outputs: + result: ${{ steps.run.outputs.result }} + skip: ${{ steps.run.outputs.skip }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install claude CLI + uses: anthropics/claude-code-action/setup@24492741e0ccfdef4c1d19da8e11e0f373d07494 # main + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + - name: Install Python dependencies + run: pip install "pyyaml==6.0.3" --only-binary :all: --quiet + + - name: Run issue-triage (staged) + id: run + env: + ISSUE_NUMBER: ${{ github.event.issue.number }} + ISSUE_TITLE: ${{ github.event.issue.title }} + ISSUE_BODY: ${{ github.event.issue.body }} + ISSUE_LABELS: ${{ join(github.event.issue.labels.*.name, ',') }} + ISSUE_URL: ${{ github.event.issue.html_url }} + ISSUE_ACTION: ${{ github.event.action }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + result=$(bash scripts/aw.sh run issue-triage --staged) + skip=$(AW_RESULT="$result" python3 - <<'PYEOF' +import json, os +d = json.loads(os.environ['AW_RESULT']) +print('true' if d.get('skip') else 'false') +PYEOF +) + # Store encoded to survive multi-line JSON in GITHUB_OUTPUT + encoded=$(printf '%s\n' "$result" | base64 -w0) + echo "result=$encoded" >> "$GITHUB_OUTPUT" + echo "skip=$skip" >> "$GITHUB_OUTPUT" + + apply: + needs: triage + if: needs.triage.outputs.skip != 'true' + runs-on: ubuntu-latest + timeout-minutes: 3 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Python dependencies + run: pip install "pyyaml==6.0.3" --only-binary :all: --quiet + + - name: Apply triage output (safe-output) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + ISSUE_NUMBER: ${{ github.event.issue.number }} + TRIAGE_RESULT: ${{ needs.triage.outputs.result }} + run: | + result_file=$(mktemp) + printf '%s\n' "$TRIAGE_RESULT" | base64 -d > "$result_file" + bash scripts/aw.sh safe-output apply issue-triage "$result_file" + rm -f "$result_file" diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md new file mode 100644 index 000000000..c5420fe86 --- /dev/null +++ b/.github/workflows/issue-triage.md @@ -0,0 +1,76 @@ +--- +on: + issues: + types: [opened] +engine: claude-sonnet-4-6 +permissions: + issues: read +safe-outputs: + add-labels: + allowed: + - bug + - enhancement + - documentation + - question + - needs-triage + - good-first-issue + - security + max: 3 +--- + +# Issue Triage + +You are triaging a new GitHub issue for the `${REPO}` repository. + +## Context + +- **Repository:** `${REPO}` +- **Issue:** [#${ISSUE_NUMBER}](${ISSUE_URL}): ${ISSUE_TITLE} +- **Labels already applied:** ${ISSUE_LABELS} + +**Issue body:** + +${ISSUE_BODY} + +## Task + +If the issue already has **2 or more labels**, output `{"skip": true}` and stop — do not classify, do not comment. + +Otherwise: + +1. **Classify** the issue into the best-fit category: + - `bug` — something is not working as expected + - `enhancement` — request for new or extended functionality + - `documentation` — unclear, missing, or incorrect docs + - `question` — user asking how to do something + - `security` — potential security vulnerability + - `good-first-issue` — simple enough for a first-time contributor + +2. **Select ≤ 3 labels** from the allowed set: + `bug`, `enhancement`, `documentation`, `question`, `needs-triage`, + `good-first-issue`, `security`. + - Add `needs-triage` for bugs and ambiguous reports that need human review. + - Add `good-first-issue` only when the scope is clearly small and + self-contained. + +3. **Write one welcoming comment** that: + - Opens warmly and thanks the contributor. + - Asks the **single most important** clarifying question for this issue type: + - Bug → steps to reproduce + expected vs actual behaviour + - Feature → use case / motivation + - Question → what docs they checked and where they got stuck + - Documentation → which page or section is unclear and what they expected + - Security → (do not ask for details publicly; ask them to use the + private security advisory channel) + - Does **not** ask multiple unrelated questions. + - Is concise (≤ 5 sentences). + +## Output format + +Output **exactly one** JSON object — no markdown fences, no preamble: + +For a normal triage: +{"labels": ["label1", "label2"], "comment": "Your welcoming comment here."} + +For a skip (issue already has 2+ labels): +{"skip": true} diff --git a/docs/aw/issue-triage.md b/docs/aw/issue-triage.md new file mode 100644 index 000000000..3b0db88c1 --- /dev/null +++ b/docs/aw/issue-triage.md @@ -0,0 +1,99 @@ +# Issue Triage — Agentic Workflow + +Part of the GitHub Agentic Workflows rollout (internal discussion: [petry-projects/.github-private#228](https://github.com/petry-projects/.github-private/discussions/228) — requires repo access). + +## What it does + +When a new issue is opened or reopened, this workflow: + +1. Checks the existing label count. If the issue already has **2 or more labels** it exits silently (no-op). +2. Calls Claude (`claude-sonnet-4-6`) with the issue title and body to classify the issue. +3. Selects up to 3 labels from the allowed set and writes a single welcoming comment that asks the most relevant clarifying question. + +All of this happens within 60 seconds of the issue being opened. + +## Allowed labels + +`bug`, `enhancement`, `documentation`, `question`, `needs-triage`, `good-first-issue`, `security` + +## Files + +| File | Purpose | +|------|---------| +| `.github/workflows/issue-triage.md` | Workflow definition (frontmatter + Claude prompt template) | +| `.github/workflows/issue-triage-runner.yml` | GitHub Actions runner (two-job: triage → apply) | +| `scripts/aw.sh` | `gh aw` CLI — compile, run, safe-output | +| `tests/aw/issue-triage/scenarios.md` | Scenario spec used to validate behaviour | + +## Architecture + +``` +issues: opened/reopened + │ + ▼ + triage job (permissions: issues: read) + │ bash scripts/aw.sh run issue-triage --staged + │ → Claude classifies, returns JSON + ▼ + apply job (permissions: issues: write) + │ bash scripts/aw.sh safe-output apply issue-triage result.json + │ → validates labels, posts comment, adds labels + ▼ + issue updated +``` + +The two-job split is deliberate: Claude runs with the minimum permission it +needs (read), and writes are applied by a separate job after the output has +been validated by `safe-output`. + +## Validation + +`gh aw compile` validates every `.md` workflow definition file. It checks: + +- Required frontmatter fields: `name`, `trigger`, `engine`, `permissions` +- Engine is a known Claude model identifier +- `output` is `staged` or `live` (if present) +- Trigger is a non-empty event mapping +- Workflow body (instructions) is not empty + +Run locally: + +```bash +bash scripts/aw.sh compile .github/workflows/issue-triage.md +``` + +## Staged vs live mode + +The `output` frontmatter field controls how results are applied: + +| Value | Behaviour | +|-------|-----------| +| `staged` | `aw run` emits JSON to stdout; a separate `safe-output apply` step writes to GitHub | +| `live` (default when omitted) | `aw run` applies labels and posts the comment inline | + +The runner workflow always uses the two-job pattern regardless of this setting, +so `safe-output` always validates before writing. + +## Skip condition + +If the issue already has **2 or more labels** when the workflow fires, `aw run` +returns `{"skip": true}` immediately and the apply job is skipped. No API +writes occur. + +## Smoke-test procedure + +To validate against the scenario spec before going live: + +```bash +# Create a test issue in a non-critical repo and capture the number +ISSUE_NUMBER= \ +ISSUE_TITLE="Login button is broken" \ +ISSUE_BODY="When I click nothing happens." \ +ISSUE_LABELS="" \ +ISSUE_URL="https://github.com/petry-projects/.github-private/issues/" \ +GITHUB_REPOSITORY="petry-projects/.github-private" \ +bash scripts/aw.sh run issue-triage --staged +``` + +Compare the JSON output against the expected labels and comment style in +`tests/aw/issue-triage/scenarios.md`. diff --git a/scripts/aw.sh b/scripts/aw.sh new file mode 100755 index 000000000..6f7131785 --- /dev/null +++ b/scripts/aw.sh @@ -0,0 +1,549 @@ +#!/usr/bin/env bash +# gh aw — GitHub Agentic Workflows CLI +# Subcommands: compile, run, safe-output +set -euo pipefail + +WORKFLOWS_DIR=".github/workflows" +COMMAND="${1:-}" +shift || true + +usage() { + cat >&2 <<'EOF' +Usage: + aw.sh compile [|--all] + aw.sh run [--staged] + aw.sh safe-output apply +EOF + exit 1 +} + +# --------------------------------------------------------------------------- +# compile — validate one or all workflow definition files +# --------------------------------------------------------------------------- +cmd_compile() { + local target="${1:---all}" + local files=() + + if [[ "$target" == "--all" ]]; then + while IFS= read -r -d '' f; do + files+=("$f") + done < <(find "$WORKFLOWS_DIR" -name '*.md' -print0 2>/dev/null) + if [[ ${#files[@]} -eq 0 ]]; then + echo "aw compile: no workflow definitions found in $WORKFLOWS_DIR" >&2 + exit 0 + fi + else + files=("$target") + fi + + local errors=0 + for file in "${files[@]}"; do + if ! validate_workflow "$file"; then + errors=$(( errors + 1 )) + fi + done + + if [[ $errors -gt 0 ]]; then + echo "aw compile: $errors file(s) failed validation" >&2 + exit 1 + fi + + echo "aw compile: all workflow definitions are valid" +} + +# Validate a single workflow .md file. +# Uses python3 to parse the YAML frontmatter — always available on GitHub Actions runners. +validate_workflow() { + local file="$1" + + if [[ ! -f "$file" ]]; then + echo "aw compile: $file: file not found" >&2 + return 1 + fi + + python3 - "$file" <<'PYEOF' +import sys, re + +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() + +# Extract YAML frontmatter +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +if not m: + print(f"aw compile: {path}: missing YAML frontmatter", file=sys.stderr) + sys.exit(1) + +import yaml +try: + fm = yaml.safe_load(m.group(1)) +except yaml.YAMLError as e: + print(f"aw compile: {path}: invalid YAML frontmatter: {e}", file=sys.stderr) + sys.exit(1) + +errors = [] + +# Required fields +for field in ("name", "trigger", "engine", "permissions"): + if field not in fm: + errors.append(f"missing required field: {field!r}") + +# Validate engine — warn on unknown but don't fail (new models can be added without a code change) +known_engines = { + "claude-sonnet-4-6", "claude-haiku-4-5-20251001", "claude-opus-4-7", + "claude-haiku-4-5", +} +engine = fm.get("engine", "") +if engine and engine not in known_engines: + print(f"aw compile: {path}: warning: unknown engine {engine!r}", file=sys.stderr) + +# Validate output mode if present +output_mode = fm.get("output") +if output_mode and output_mode not in ("staged", "live"): + errors.append(f"output must be 'staged' or 'live', got {output_mode!r}") + +# Validate trigger has at least one event +trigger = fm.get("trigger") or {} +if not isinstance(trigger, dict) or not trigger: + errors.append("trigger must be a non-empty mapping of event types") + +# Instructions body must not be empty +body = text[m.end():].strip() +if not body: + errors.append("workflow body (instructions) must not be empty") + +if errors: + for e in errors: + print(f"aw compile: {path}: {e}", file=sys.stderr) + sys.exit(1) + +print(f"aw compile: {path}: ok") +PYEOF +} + +# --------------------------------------------------------------------------- +# run — execute a workflow against GitHub context (or a fixture) +# --------------------------------------------------------------------------- +cmd_run() { + local name="" + local staged=false + local fixture="" + + while [[ $# -gt 0 ]]; do + case "$1" in + --staged) staged=true ;; + --fixture) fixture="${2:-}"; shift ;; + -*) echo "aw run: unknown flag $1" >&2; exit 1 ;; + *) name="$1" ;; + esac + shift + done + + if [[ -z "$name" ]]; then + echo "aw run: workflow name required" >&2 + exit 1 + fi + + # Validate workflow name to prevent path traversal + if [[ ! "$name" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]]; then + echo "aw run: invalid workflow name: $name" >&2 + exit 1 + fi + + local wf_file="$WORKFLOWS_DIR/${name}.md" + if [[ ! -f "$wf_file" ]]; then + echo "aw run: workflow file not found: $wf_file" >&2 + exit 1 + fi + + # Build context from environment or fixture + local issue_number issue_title issue_body issue_labels issue_url repo issue_action + if [[ -n "$fixture" ]]; then + # Pass path via env var to avoid injecting user-controlled paths into Python source + issue_number="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d['issue']['number']) +PYEOF +)" + issue_title="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d['issue']['title']) +PYEOF +)" + issue_body="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d['issue']['body'] or '') +PYEOF +)" + issue_labels="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(','.join(l['name'] for l in d['issue'].get('labels', []))) +PYEOF +)" + issue_url="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d['issue']['html_url']) +PYEOF +)" + repo="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d['repository']['full_name']) +PYEOF +)" + issue_action="$(AW_FIXTURE="$fixture" python3 - <<'PYEOF' +import json, os +with open(os.environ['AW_FIXTURE'], encoding='utf-8') as fh: + d = json.load(fh) +print(d.get('action', 'opened')) +PYEOF +)" + else + issue_number="${ISSUE_NUMBER:-}" + issue_title="${ISSUE_TITLE:-}" + issue_body="${ISSUE_BODY:-}" + issue_labels="${ISSUE_LABELS:-}" + issue_url="${ISSUE_URL:-https://github.com/${GITHUB_REPOSITORY:-unknown}/issues/${ISSUE_NUMBER:-0}}" + repo="${GITHUB_REPOSITORY:-unknown}" + issue_action="${ISSUE_ACTION:-opened}" + fi + + # Skip guard: read threshold from frontmatter (default 2), skip if label count >= threshold + local label_count skip_threshold + label_count="$(AW_LABELS="$issue_labels" python3 - <<'PYEOF' +import os +raw = os.environ.get('AW_LABELS', '').strip() +labels = [l for l in raw.split(',') if l] +print(len(labels)) +PYEOF +)" + skip_threshold="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(fm.get('skip-if-labels-ge', 2)) +PYEOF +)" + if [[ "$label_count" -ge "$skip_threshold" ]]; then + echo '{"skip": true}' + return 0 + fi + + # Skip if any existing label is in the allowed-labels set (prevents re-triage on reopen) + local allowed_json + allowed_json="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml, json +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(json.dumps(fm.get('allowed-labels', []))) +PYEOF +)" + local has_triage_label + has_triage_label="$(AW_ALLOWED="$allowed_json" AW_LABELS="$issue_labels" python3 - <<'PYEOF' +import os, json +current = set(l for l in os.environ.get('AW_LABELS', '').split(',') if l) +allowed = set(json.loads(os.environ.get('AW_ALLOWED', '[]'))) +print('true' if current & allowed else 'false') +PYEOF +)" + if [[ "$has_triage_label" == "true" && "$issue_action" == "reopened" ]]; then + echo '{"skip": true}' + return 0 + fi + + # Substitute variables into workflow body. + # Use a quoted heredoc and env vars so user-controlled issue data cannot + # escape into Python source code (code injection hotspot). + local prompt + prompt="$(AW_ISSUE_NUMBER="$issue_number" \ + AW_ISSUE_TITLE="$issue_title" \ + AW_ISSUE_BODY="$issue_body" \ + AW_ISSUE_LABELS="$issue_labels" \ + AW_ISSUE_URL="$issue_url" \ + AW_REPO="$repo" \ + python3 - "$wf_file" <<'PYEOF' +import sys, re, os + +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n.*?\n---\n', text, re.DOTALL) +body = text[m.end():] if m else text + +labels_val = os.environ.get('AW_ISSUE_LABELS', '') +subs = { + 'ISSUE_NUMBER': os.environ.get('AW_ISSUE_NUMBER', ''), + 'ISSUE_TITLE': os.environ.get('AW_ISSUE_TITLE', ''), + 'ISSUE_BODY': os.environ.get('AW_ISSUE_BODY', ''), + 'ISSUE_LABELS': labels_val if labels_val else '(none)', + 'ISSUE_URL': os.environ.get('AW_ISSUE_URL', ''), + 'REPO': os.environ.get('AW_REPO', ''), +} +pattern = re.compile('|'.join(re.escape('${' + k + '}') for k in subs)) +body = pattern.sub(lambda m: subs[m.group(0)[2:-1]], body) +print(body) +PYEOF +)" + + # Select engine from frontmatter + local engine + engine="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) +print(fm.get("engine", "claude-sonnet-4-6")) +PYEOF +)" + + # Validate engine to prevent command injection before passing to claude + if [[ ! "$engine" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then + echo "aw run: invalid engine value: $engine" >&2 + exit 1 + fi + + # Call Claude — write prompt to a temp file to avoid echo with user-controlled data + local result rc=0 prompt_file + prompt_file="$(mktemp)" + printf '%s\n' "$prompt" > "$prompt_file" + result="$(claude --model "$engine" --print --output-format text < "$prompt_file" 2>/dev/null)" || rc=$? + rm -f "$prompt_file" + if [[ $rc -ne 0 ]]; then + echo "aw run: claude invocation failed" >&2 + exit 1 + fi + + # Validate JSON output + if ! AW_RESULT="$result" python3 - 2>/dev/null <<'PYEOF'; then +import json, os +json.loads(os.environ['AW_RESULT']) +PYEOF + echo "aw run: claude returned non-JSON output" >&2 + echo "$result" >&2 + exit 1 + fi + + # Reject skip flag from model — only the pre-Claude label-count guard may emit skip + if AW_RESULT="$result" python3 - 2>/dev/null <<'PYEOF'; then +import json, os, sys +d = json.loads(os.environ['AW_RESULT']) +sys.exit(1) if d.get('skip') else None +PYEOF + : # no skip flag from model, ok + else + echo "aw run: model returned skip flag — possible prompt injection, rejecting" >&2 + exit 1 + fi + + # Honor output:staged frontmatter when --staged was not explicitly passed + if [[ "$staged" == false ]]; then + local frontmatter_output + frontmatter_output="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(fm.get('output', 'live')) +PYEOF +)" + if [[ "$frontmatter_output" == "staged" ]]; then + staged=true + fi + fi + + if [[ "$staged" == true ]]; then + # Staged mode: emit JSON to stdout for review; safe-output handles writes + echo "$result" + else + # Live mode: apply immediately + apply_result "$name" "$result" "$issue_number" + fi +} + +# --------------------------------------------------------------------------- +# safe-output apply — validate and apply a staged result artifact +# --------------------------------------------------------------------------- +cmd_safe_output() { + local subcommand="${1:-}" + shift || true + + case "$subcommand" in + apply) cmd_safe_output_apply "$@" ;; + *) echo "aw safe-output: unknown subcommand ${subcommand}" >&2; exit 1 ;; + esac +} + +cmd_safe_output_apply() { + local name="${1:-}" + local result_file="${2:-}" + + if [[ -z "$name" || -z "$result_file" ]]; then + echo "aw safe-output apply: requires " >&2 + exit 1 + fi + if [[ ! -f "$result_file" ]]; then + echo "aw safe-output apply: result file not found: $result_file" >&2 + exit 1 + fi + + local wf_file="$WORKFLOWS_DIR/${name}.md" + local allowed_labels max_labels + allowed_labels="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml, json +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(json.dumps(fm.get("allowed-labels", []))) +PYEOF +)" + max_labels="$(python3 - "$wf_file" <<'PYEOF' +import sys, re, yaml +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(fm.get('max-labels', 3)) +PYEOF +)" + + # Validate and apply via GitHub API + python3 - "$result_file" "$allowed_labels" "$max_labels" <<'PYEOF' +import sys, json + +result_file = sys.argv[1] +allowed_set = set(json.loads(sys.argv[2])) +max_labels = int(sys.argv[3]) + +with open(result_file, encoding='utf-8') as f: + result = json.load(f) + +if result.get("skip"): + print("safe-output: skip flag set — no writes applied") + sys.exit(0) + +labels = result.get("labels", []) +comment = result.get("comment", "") + +# Validate labels is a JSON array of strings before any further checks +if not isinstance(labels, list) or not all(isinstance(l, str) for l in labels): + print(f"safe-output: rejected — labels must be a JSON array of strings, got {type(labels).__name__}", file=sys.stderr) + sys.exit(1) + +# Validate comment is a non-empty string before any writes occur +if not isinstance(comment, str): + print(f"safe-output: rejected — comment must be a string, got {type(comment).__name__}", file=sys.stderr) + sys.exit(1) +if not comment: + print("safe-output: rejected — empty comment", file=sys.stderr) + sys.exit(1) + +# Validate labels against allowed set +invalid = [l for l in labels if l not in allowed_set] +if invalid: + print(f"safe-output: rejected — labels not in allowed set: {invalid}", file=sys.stderr) + sys.exit(1) +if len(labels) > max_labels: + print(f"safe-output: rejected — more than {max_labels} labels: {labels}", file=sys.stderr) + sys.exit(1) + +print(f"safe-output: labels={labels}, comment length={len(comment)}") +print("safe-output: validation passed — ready for API writes") +PYEOF + + # Apply writes via gh CLI + local issue_number="${ISSUE_NUMBER:-}" + local repo="${GITHUB_REPOSITORY:-}" + + if [[ -z "$issue_number" || -z "$repo" ]]; then + echo "safe-output apply: ISSUE_NUMBER and GITHUB_REPOSITORY must be set for live writes" >&2 + exit 1 + fi + + # Validate inputs before passing to gh CLI to prevent command injection + if [[ ! "$issue_number" =~ ^[0-9]+$ ]]; then + echo "safe-output apply: ISSUE_NUMBER must be a positive integer, got: $issue_number" >&2 + exit 1 + fi + if [[ ! "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + echo "safe-output apply: GITHUB_REPOSITORY has unexpected format: $repo" >&2 + exit 1 + fi + + local result_json + result_json="$(cat "$result_file")" + + # Apply labels — pass as comma-separated string so the argument stays quoted + local labels_csv + labels_csv="$(AW_RESULT_JSON="$result_json" python3 - <<'PYEOF' +import json, os +r = json.loads(os.environ['AW_RESULT_JSON']) +if not r.get('skip'): + print(','.join(r.get('labels', []))) +PYEOF +)" + if [[ -n "$labels_csv" ]]; then + gh issue edit "$issue_number" --repo "$repo" --add-label "$labels_csv" + fi + + # Post comment via temp file to avoid ARG_MAX limits on large comment bodies + local comment_file + comment_file="$(mktemp)" + python3 - "$result_file" <<'PYEOF' > "$comment_file" +import json, sys +with open(sys.argv[1], encoding='utf-8') as f: + r = json.load(f) +c = r.get('comment', '') if not r.get('skip') else '' +sys.stdout.write(c) +PYEOF + if [[ -s "$comment_file" ]]; then + gh issue comment "$issue_number" --repo "$repo" --body-file "$comment_file" + fi + rm -f "$comment_file" + + echo "safe-output apply: done" +} + +# Apply result inline (live mode, called from cmd_run) +apply_result() { + local name="$1" result="$2" issue_number="$3" + local tmp + tmp="$(mktemp)" + printf '%s\n' "$result" > "$tmp" + ISSUE_NUMBER="$issue_number" cmd_safe_output_apply "$name" "$tmp" + rm -f "$tmp" +} + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- +case "$COMMAND" in + compile) cmd_compile "$@" ;; + run) cmd_run "$@" ;; + safe-output) cmd_safe_output "$@" ;; + "") usage ;; + *) echo "aw.sh: unknown command '$COMMAND'" >&2; usage ;; +esac diff --git a/tests/aw/issue-triage/fixtures/scenario-1-bug.json b/tests/aw/issue-triage/fixtures/scenario-1-bug.json new file mode 100644 index 000000000..b956770af --- /dev/null +++ b/tests/aw/issue-triage/fixtures/scenario-1-bug.json @@ -0,0 +1,12 @@ +{ + "issue": { + "number": 101, + "title": "Login button is broken", + "body": "When I click the login button nothing happens.", + "labels": [], + "html_url": "https://github.com/example/repo/issues/101" + }, + "repository": { + "full_name": "example/repo" + } +} diff --git a/tests/aw/issue-triage/fixtures/scenario-4-skip.json b/tests/aw/issue-triage/fixtures/scenario-4-skip.json new file mode 100644 index 000000000..cf4bb64a1 --- /dev/null +++ b/tests/aw/issue-triage/fixtures/scenario-4-skip.json @@ -0,0 +1,15 @@ +{ + "issue": { + "number": 102, + "title": "Something is wrong", + "body": "Not sure what's happening.", + "labels": [ + {"name": "bug"}, + {"name": "P1"} + ], + "html_url": "https://github.com/example/repo/issues/102" + }, + "repository": { + "full_name": "example/repo" + } +} diff --git a/tests/aw/issue-triage/scenarios.md b/tests/aw/issue-triage/scenarios.md new file mode 100644 index 000000000..446d59d4a --- /dev/null +++ b/tests/aw/issue-triage/scenarios.md @@ -0,0 +1,75 @@ +# Issue Triage — Scenario Spec + +This spec defines the expected triage behaviour for representative inputs. All +scenarios must be validated before the workflow is promoted to live. + +--- + +## Scenario 1 — Bug report (no labels) + +**Input:** +- Title: `Login button is broken` +- Body: `When I click the login button nothing happens.` +- Existing labels: _(none)_ + +**Expected output:** +- Labels applied: `bug`, `needs-triage` +- Comment posted: yes — asks for steps to reproduce and expected vs actual + behaviour (e.g. "Could you share the steps to reproduce this? What did you + expect to happen, and what actually happened instead?") + +--- + +## Scenario 2 — Feature request (no labels) + +**Input:** +- Title: `Add dark mode support` +- Body: `It would be great if the app supported dark mode.` +- Existing labels: _(none)_ + +**Expected output:** +- Labels applied: `enhancement` +- Comment posted: yes — acknowledges the feature request and asks about the + use case (e.g. "Thanks for the suggestion! Could you tell us a bit more + about your use case — how would dark mode help your workflow?") + +--- + +## Scenario 3 — Question (no labels) + +**Input:** +- Title: `How do I configure X?` +- Body: `I'm trying to configure X but can't find any documentation.` +- Existing labels: _(none)_ + +**Expected output:** +- Labels applied: `question` +- Comment posted: yes — points to docs and asks what the user has already + tried (e.g. "Have you had a chance to check the docs at …? If so, what + step are you getting stuck on?") + +--- + +## Scenario 4 — Already labelled (edge case: no-op) + +**Input:** +- Title: `Something is wrong` +- Body: `Not sure what's happening.` +- Existing labels: `bug`, `P1` _(2 labels — threshold met)_ + +**Expected output:** +- Labels applied: _(none)_ +- Comment posted: _(none)_ +- Workflow exits silently (skip, no changes, no comment) + +--- + +## Validation notes + +- The allowed label set is: `bug`, `enhancement`, `documentation`, `question`, + `needs-triage`, `good-first-issue`, `security`. +- At most **3 labels** may be applied in a single run. +- The comment must be a **single** welcoming message; it must not ask multiple + unrelated questions. +- For Scenario 4 the JSON output must be `{"skip": true}` and the safe-output + step must apply zero GitHub API writes. diff --git a/tests/aw/issue-triage/test_aw_run.sh b/tests/aw/issue-triage/test_aw_run.sh new file mode 100755 index 000000000..6f54b6b19 --- /dev/null +++ b/tests/aw/issue-triage/test_aw_run.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# tests/aw/issue-triage/test_aw_run.sh +# +# Unit tests for aw.sh paths that do not require a live Claude call. +# Covers: skip guard, safe-output label validation, safe-output skip no-op. +# +# Run: bash tests/aw/issue-triage/test_aw_run.sh +set -euo pipefail + +PASS=0 +FAIL=0 +ERRORS="" + +ok() { PASS=$((PASS + 1)); printf 'PASS %s\n' "$1"; } +fail() { FAIL=$((FAIL + 1)); ERRORS="${ERRORS}FAIL $1: $2\n"; printf 'FAIL %s: %s\n' "$1" "$2"; } + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +FIXTURES="$(dirname "$0")/fixtures" + +# --------------------------------------------------------------------------- +# Test 1: Scenario 4 — issue already has 2 labels → aw run emits {"skip":true} +# --------------------------------------------------------------------------- +result=$(bash "$REPO_ROOT/scripts/aw.sh" run issue-triage \ + --fixture "$FIXTURES/scenario-4-skip.json" --staged) +skip=$(AW_RESULT="$result" python3 - <<'PYEOF' +import json, os +print(json.loads(os.environ['AW_RESULT']).get('skip', False)) +PYEOF +) +if [[ "$skip" == "True" ]]; then + ok "scenario-4: 2 existing labels → skip=true" +else + fail "scenario-4: 2 existing labels → skip=true" "got skip=$skip, result=$result" +fi + +# --------------------------------------------------------------------------- +# Test 2: safe-output rejects result with more labels than max-labels (3) +# --------------------------------------------------------------------------- +tmp=$(mktemp) +printf '{"labels":["bug","needs-triage","question","enhancement"],"comment":"Hello!"}' > "$tmp" +output=$(ISSUE_NUMBER=1 GITHUB_REPOSITORY=example/repo \ + bash "$REPO_ROOT/scripts/aw.sh" safe-output apply issue-triage "$tmp" 2>&1 || true) +if echo "$output" | grep -q "rejected"; then + ok "safe-output: 4 labels exceeds max-labels:3 → rejected" +else + fail "safe-output: 4 labels exceeds max-labels:3 → rejected" "got: $output" +fi +rm -f "$tmp" + +# --------------------------------------------------------------------------- +# Test 3: safe-output rejects a label not in the allowed set +# --------------------------------------------------------------------------- +tmp=$(mktemp) +printf '{"labels":["not-a-real-label"],"comment":"Hello!"}' > "$tmp" +output=$(ISSUE_NUMBER=1 GITHUB_REPOSITORY=example/repo \ + bash "$REPO_ROOT/scripts/aw.sh" safe-output apply issue-triage "$tmp" 2>&1 || true) +if echo "$output" | grep -q "rejected"; then + ok "safe-output: disallowed label → rejected" +else + fail "safe-output: disallowed label → rejected" "got: $output" +fi +rm -f "$tmp" + +# --------------------------------------------------------------------------- +# Test 4: safe-output with skip flag → no-op, prints skip message +# --------------------------------------------------------------------------- +tmp=$(mktemp) +printf '{"skip":true}' > "$tmp" +output=$(ISSUE_NUMBER=1 GITHUB_REPOSITORY=example/repo \ + bash "$REPO_ROOT/scripts/aw.sh" safe-output apply issue-triage "$tmp" 2>&1) +if echo "$output" | grep -q "skip flag set"; then + ok "safe-output: skip flag → no writes" +else + fail "safe-output: skip flag → no writes" "got: $output" +fi +rm -f "$tmp" + +# --------------------------------------------------------------------------- +# Test 5: safe-output rejects result with empty comment +# --------------------------------------------------------------------------- +tmp=$(mktemp) +printf '{"labels":["bug"],"comment":""}' > "$tmp" +output=$(ISSUE_NUMBER=1 GITHUB_REPOSITORY=example/repo \ + bash "$REPO_ROOT/scripts/aw.sh" safe-output apply issue-triage "$tmp" 2>&1 || true) +if echo "$output" | grep -q "rejected"; then + ok "safe-output: empty comment → rejected" +else + fail "safe-output: empty comment → rejected" "got: $output" +fi +rm -f "$tmp" + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo "Results: $PASS passed, $FAIL failed" +if [[ "$FAIL" -gt 0 ]]; then + printf '%b' "$ERRORS" + exit 1 +fi