feat(inline-comment): add confirmed param + probe-pattern safety net

action.yml

@@ -89,6 +89,10 @@ inputs:
     description: "Use just one comment to deliver issue/PR comments"
     required: false
     default: "false"
+  classify_inline_comments:
+    description: "Buffer inline comments without confirmed=true and classify them (real review vs test/probe) before posting after the session ends. Set to 'false' to post all inline comments immediately (pre-buffering behavior)."
+    required: false
+    default: "true"
   use_commit_signing:
     description: "Enable commit signing using GitHub's commit signature verification. When false, Claude uses standard git commands"
     required: false
@@ -204,6 +208,7 @@ runs:
         EXCLUDE_COMMENTS_BY_ACTOR: ${{ inputs.exclude_comments_by_actor }}
         GITHUB_RUN_ID: ${{ github.run_id }}
         USE_STICKY_COMMENT: ${{ inputs.use_sticky_comment }}
+        CLASSIFY_INLINE_COMMENTS: ${{ inputs.classify_inline_comments }}
         DEFAULT_WORKFLOW_TOKEN: ${{ github.token }}
         USE_COMMIT_SIGNING: ${{ inputs.use_commit_signing }}
         SSH_SIGNING_KEY: ${{ inputs.ssh_signing_key }}
@@ -282,6 +287,18 @@ runs:
       run: |
         bun run ${GITHUB_ACTION_PATH}/src/entrypoints/cleanup-ssh-signing.ts
 
+    - name: Post buffered inline comments
+      if: always() && inputs.classify_inline_comments != 'false'
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ steps.run.outputs.github_token || inputs.github_token || github.token }}
+        REPO_OWNER: ${{ github.event.repository.owner.login }}
+        REPO_NAME: ${{ github.event.repository.name }}
+        PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }}
+        ANTHROPIC_API_KEY: ${{ inputs.anthropic_api_key }}
+      run: |
+        bun run ${GITHUB_ACTION_PATH}/src/entrypoints/post-buffered-inline-comments.ts
+
     - name: Revoke app token
       if: always() && inputs.github_token == '' && steps.run.outputs.skipped_due_to_workflow_validation_mismatch != 'true'
       shell: bash

docs/solutions.md

@@ -55,7 +55,7 @@ jobs:
             Note: The PR branch is already checked out in the current working directory.
 
             Use `gh pr comment` for top-level feedback.
-            Use `mcp__github_inline_comment__create_inline_comment` to highlight specific code issues.
+            Use `mcp__github_inline_comment__create_inline_comment` (with `confirmed: true`) to highlight specific code issues.
             Only post GitHub comments - don't submit review text as messages.
 
           claude_args: |
@@ -585,7 +585,7 @@ prompt: |
 ### Common Tool Permissions
 
 - **PR Comments**: `Bash(gh pr comment:*)`
-- **Inline Comments**: `mcp__github_inline_comment__create_inline_comment`
+- **Inline Comments**: `mcp__github_inline_comment__create_inline_comment` — pass `confirmed: true` to post immediately. When omitted, the comment is buffered and classified after the session ends (real review comments post, test/probe comments are filtered). This prevents subagent test comments from reaching PRs. To disable classification entirely, set `classify_inline_comments: 'false'` on the action.
 - **File Operations**: `Read,Write,Edit`
 - **Git Operations**: `Bash(git:*)`
 

src/entrypoints/collect-inputs.ts

@@ -23,6 +23,7 @@ export function collectActionInputsPresence(): string {
     github_token: "",
     max_turns: "",
     use_sticky_comment: "false",
+    classify_inline_comments: "true",
     use_commit_signing: "false",
     ssh_signing_key: "",
   };

src/entrypoints/post-buffered-inline-comments.ts

@@ -0,0 +1,233 @@
+#!/usr/bin/env bun
+/**
+ * Reads buffered inline-comment calls from /tmp/inline-comments-buffer.jsonl,
+ * classifies each as "real review" vs "test/probe" using Haiku, and posts
+ * only the real ones. Calls with confirmed=false are never posted.
+ *
+ * If the Anthropic API is unavailable (Bedrock/Vertex users without a direct
+ * key), falls back to posting everything with confirmed !== false. This
+ * preserves backward compatibility — before this change, all unconfirmed
+ * calls posted immediately.
+ */
+import { readFileSync } from "fs";
+import { createOctokit } from "../github/api/client";
+
+const BUFFER_PATH = "/tmp/inline-comments-buffer.jsonl";
+
+type BufferedComment = {
+  ts: string;
+  path: string;
+  line?: number;
+  startLine?: number;
+  side?: "LEFT" | "RIGHT";
+  commit_id?: string;
+  body: string;
+  confirmed?: boolean;
+};
+
+const CLASSIFICATION_PROMPT = `You are classifying PR inline comments as either REAL code review feedback or TEST/PROBE calls.
+
+A TEST/PROBE call is when an automated agent is checking whether a commenting tool works. These typically:
+- Start with phrases like "Test comment", "Testing if", "Can I", "Does this work", "Checking if"
+- Have generic/placeholder content not specific to any code
+- Exist to verify tool functionality, not to provide review feedback
+
+A REAL review comment:
+- Discusses specific code, logic, bugs, or style
+- Provides actionable feedback for the PR author
+- References concrete aspects of the change
+
+For each numbered comment body below, respond with ONLY a JSON array of booleans where true = REAL review comment, false = test/probe. No other text.
+
+Comments:
+`;
+
+async function classifyComments(bodies: string[]): Promise<boolean[] | null> {
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) {
+    console.log(
+      "ANTHROPIC_API_KEY not set — skipping classification, posting all unconfirmed comments",
+    );
+    return null;
+  }
+
+  const prompt =
+    CLASSIFICATION_PROMPT +
+    bodies.map((b, i) => `${i + 1}. ${JSON.stringify(b)}`).join("\n");
+
+  try {
+    const res = await fetch("https://api.anthropic.com/v1/messages", {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+      },
+      body: JSON.stringify({
+        model: "claude-haiku-4-5",
+        max_tokens: 1024,
+        messages: [{ role: "user", content: prompt }],
+      }),
+    });
+
+    if (!res.ok) {
+      console.log(
+        `Classification API returned ${res.status} — posting all unconfirmed comments`,
+      );
+      return null;
+    }
+
+    const data = (await res.json()) as {
+      content: { type: string; text: string }[];
+    };
+    const text = data.content.find((c) => c.type === "text")?.text ?? "";
+    const match = text.match(/\[[\s\S]*\]/);
+    if (!match) {
+      console.log(
+        "Could not parse classification response — posting all unconfirmed comments",
+      );
+      return null;
+    }
+    const parsed = JSON.parse(match[0]);
+    if (
+      !Array.isArray(parsed) ||
+      parsed.length !== bodies.length ||
+      !parsed.every((v) => typeof v === "boolean")
+    ) {
+      console.log(
+        "Classification response shape mismatch — posting all unconfirmed comments",
+      );
+      return null;
+    }
+    return parsed;
+  } catch (e) {
+    console.log(
+      `Classification failed (${e instanceof Error ? e.message : String(e)}) — posting all unconfirmed comments`,
+    );
+    return null;
+  }
+}
+
+async function postComment(
+  octokit: ReturnType<typeof createOctokit>["rest"],
+  owner: string,
+  repo: string,
+  pull_number: number,
+  headSha: string,
+  c: BufferedComment,
+): Promise<boolean> {
+  const params: Parameters<typeof octokit.rest.pulls.createReviewComment>[0] = {
+    owner,
+    repo,
+    pull_number,
+    body: c.body,
+    path: c.path,
+    side: c.side || "RIGHT",
+    commit_id: c.commit_id || headSha,
+  };
+  if (c.startLine) {
+    params.start_line = c.startLine;
+    params.start_side = c.side || "RIGHT";
+    params.line = c.line;
+  } else {
+    params.line = c.line;
+  }
+  try {
+    await octokit.rest.pulls.createReviewComment(params);
+    return true;
+  } catch (e) {
+    console.log(
+      `  failed ${c.path}:${c.line}: ${e instanceof Error ? e.message : String(e)}`,
+    );
+    return false;
+  }
+}
+
+async function main() {
+  let raw: string;
+  try {
+    raw = readFileSync(BUFFER_PATH, "utf8");
+  } catch {
+    console.log("No buffered inline comments");
+    return;
+  }
+
+  const comments: BufferedComment[] = raw
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => JSON.parse(line));
+
+  if (comments.length === 0) {
+    console.log("No buffered inline comments");
+    return;
+  }
+
+  console.log(`Found ${comments.length} buffered inline comment(s)`);
+
+  const githubToken = process.env.GITHUB_TOKEN;
+  const owner = process.env.REPO_OWNER;
+  const repo = process.env.REPO_NAME;
+  const prNumber = process.env.PR_NUMBER;
+
+  if (!githubToken || !owner || !repo || !prNumber) {
+    console.log(
+      "::warning::Missing GITHUB_TOKEN/REPO_OWNER/REPO_NAME/PR_NUMBER — cannot post buffered comments",
+    );
+    return;
+  }
+
+  // Partition: confirmed=false are never posted; the rest are candidates
+  const neverPost = comments.filter((c) => c.confirmed === false);
+  const candidates = comments.filter((c) => c.confirmed !== false);
+
+  if (neverPost.length > 0) {
+    console.log(`  ${neverPost.length} with confirmed=false — not posting`);
+  }
+
+  if (candidates.length === 0) {
+    return;
+  }
+
+  // Classify candidates
+  const verdicts = await classifyComments(candidates.map((c) => c.body));
+  const toPost =
+    verdicts === null
+      ? candidates
+      : candidates.filter((_, i) => verdicts[i] === true);
+  const filtered =
+    verdicts === null ? [] : candidates.filter((_, i) => verdicts[i] === false);
+
+  if (filtered.length > 0) {
+    console.log(
+      `::warning::${filtered.length} buffered comment(s) classified as test/probe — NOT posted:`,
+    );
+    for (const c of filtered) {
+      console.log(`  [${c.path}:${c.line}] ${c.body.slice(0, 120)}`);
+    }
+  }
+
+  if (toPost.length === 0) {
+    console.log("No real comments to post");
+    return;
+  }
+
+  const octokit = createOctokit(githubToken).rest;
+  const pull_number = parseInt(prNumber, 10);
+  const pr = await octokit.pulls.get({ owner, repo, pull_number });
+  const headSha = pr.data.head.sha;
+
+  console.log(`Posting ${toPost.length} classified-as-real comment(s)`);
+  let posted = 0;
+  for (const c of toPost) {
+    if (await postComment(octokit, owner, repo, pull_number, headSha, c)) {
+      console.log(`  posted ${c.path}:${c.line}`);
+      posted++;
+    }
+  }
+  console.log(`Posted ${posted}/${toPost.length}`);
+}
+
+main().catch((e) => {
+  console.error("post-buffered-inline-comments failed:", e);
+  process.exit(1);
+});

src/github/context.ts

@@ -90,6 +90,7 @@ type BaseContext = {
     branchPrefix: string;
     branchNameTemplate?: string;
     useStickyComment: boolean;
+    classifyInlineComments: boolean;
     useCommitSigning: boolean;
     sshSigningKey: string;
     botId: string;
@@ -150,6 +151,7 @@ export function parseGitHubContext(): GitHubContext {
       branchPrefix: process.env.BRANCH_PREFIX ?? "claude/",
       branchNameTemplate: process.env.BRANCH_NAME_TEMPLATE,
       useStickyComment: process.env.USE_STICKY_COMMENT === "true",
+      classifyInlineComments: process.env.CLASSIFY_INLINE_COMMENTS !== "false",
       useCommitSigning: process.env.USE_COMMIT_SIGNING === "true",
       sshSigningKey: process.env.SSH_SIGNING_KEY || "",
       botId: process.env.BOT_ID ?? String(CLAUDE_APP_BOT_ID),