feat: ads/judge n context

.gitignore

@@ -29,3 +29,6 @@ coverage/
 .vulnerability-spoiler-state.json
 
 # Note: dist/ is NOT ignored - it must be committed for the action to work
+
+# CLAUDE.md
+CLAUDE.md

README.md

@@ -68,6 +68,9 @@ jobs:
 | `issue-repo` | No | Current repo | Where to create issues (`owner/repo`) |
 | `model` | No | `claude-sonnet-4-20250514` | Claude model to use |
 | `max-commits` | No | `50` | Max commits to analyze per repo per run |
+| `enable-repo-context` | No | `false` | Fetch modified files for context (max 3 files, 3000 chars each) |
+| `enable-judge` | No | `false` | Use second model to review detections and reduce false positives |
+| `judge-model` | No | Same as `model` | Claude model for judge (only used if `enable-judge` is true) |
 
 ## Outputs
 
@@ -131,6 +134,18 @@ jobs:
     echo "${{ steps.scan.outputs.results }}" | jq .
 ```
 
+### Enhanced Detection with Context & Judge
+
+```yaml
+- uses: spaceraccoon/vulnerability-spoiler-alert-action@v1
+  with:
+    anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }}
+    repositories: '[{"owner": "expressjs", "repo": "express"}]'
+    enable-repo-context: true  # Fetch file contents for better analysis
+    enable-judge: true          # Second review to reduce false positives
+    judge-model: 'claude-sonnet-4-20250514'
+```
+
 ### Send to Slack
 
 ```yaml
@@ -251,6 +266,20 @@ Prompt injection could be used to:
    exploits, or severity ratings in created issues originate from LLM analysis
    of external data and should be independently verified.
 
+## API Usage and Costs
+
+### Anthropic API
+- **Primary analysis:** 1 API call per commit analyzed
+- **Judge model:** +1 API call per detected vulnerability (when `enable-judge: true`)
+- With judge enabled, costs can double for repositories with frequent vulnerability detections
+
+### GitHub API
+- **Repository context:** Up to 4 API calls per commit (1 for parent SHA + up to 3 for files, when `enable-repo-context: true`)
+- For default `max-commits: 50`, this could be 200+ API calls per run per repository
+- Monitor your rate limits when watching multiple repositories
+
+**Tip:** Start with lower `max-commits` values and monitor costs before scaling up.
+
 ## Limitations
 
 - Only analyzes public repositories (or private repos your token can access)

action.yml

@@ -37,6 +37,20 @@ inputs:
     description: 'Maximum number of commits to analyze per repository per run'
     required: false
     default: '50'
+  enable-repo-context:
+    description: 'Fetch modified files for context (max 3 files, 3000 chars each)'
+    required: false
+    type: boolean
+    default: false
+  enable-judge:
+    description: 'Review detections with second model (only runs on positives)'
+    required: false
+    type: boolean
+    default: false
+  judge-model:
+    description: 'Model for judge (empty = same as primary model)'
+    required: false
+    default: ''
 
 outputs:
   vulnerabilities-found:

dist/analyzer.d.ts

@@ -1,3 +1,4 @@
-import type { CommitInfo, VulnerabilityAnalysis } from "./types.js";
+import type { CommitInfo, VulnerabilityAnalysis, JudgeAnalysis } from "./types.js";
 export declare function initAnalyzer(apiKey: string, model: string): void;
-export declare function analyzeCommit(commit: CommitInfo): Promise<VulnerabilityAnalysis>;
+export declare function analyzeCommit(commit: CommitInfo, repoContext?: string): Promise<VulnerabilityAnalysis>;
+export declare function judgeAnalysis(commit: CommitInfo, primaryAnalysis: VulnerabilityAnalysis, judgeModel: string, repoContext?: string): Promise<JudgeAnalysis>;

dist/github.d.ts

@@ -7,3 +7,4 @@ export declare function createVulnerabilityIssue(issueRepo: {
     repo: string;
 }, repo: RepoConfig, commit: CommitInfo, analysis: VulnerabilityAnalysis): Promise<string>;
 export declare function truncateDiff(diff: string, maxLength: number): string;
+export declare function getModifiedFilesContent(repo: RepoConfig, diff: string, commitSha: string): Promise<string>;

dist/index.js

@@ -42106,9 +42106,66 @@ function truncateDiff(diff, maxLength) {
         return diff;
     return diff.substring(0, maxLength) + "\n\n... [diff truncated]";
 }
+function extractModifiedPaths(diff) {
+    // Handle both quoted paths (spaces/special chars) and unquoted paths
+    // Quoted:   diff --git "a/path with spaces" "b/path with spaces"
+    // Unquoted: diff --git a/path b/path
+    const regex = /^diff --git "?a\/(.+?)"? "?b\//gm;
+    return [...diff.matchAll(regex)].map((m) => m[1]);
+}
 function escapeCodeFence(text) {
     return text.replace(/`{3,}/g, (match) => "`\u200B".repeat(match.length));
 }
+async function getModifiedFilesContent(repo, diff, commitSha) {
+    // Extract file paths from diff
+    const paths = extractModifiedPaths(diff);
+    // Fetch parent commit SHA (GitHub API doesn't support ~1 syntax)
+    let parentSha;
+    try {
+        const { data: commit } = await octokit.repos.getCommit({
+            owner: repo.owner,
+            repo: repo.repo,
+            ref: commitSha,
+        });
+        if (!commit.parents || commit.parents.length === 0) {
+            return ""; // Initial commit, no parent
+        }
+        parentSha = commit.parents[0].sha;
+    }
+    catch {
+        return ""; // Failed to get parent
+    }
+    // Fetch each file at parent commit, max 3 files
+    const fileContents = [];
+    for (const path of paths.slice(0, 3)) {
+        try {
+            const { data } = await octokit.repos.getContent({
+                owner: repo.owner,
+                repo: repo.repo,
+                path,
+                ref: parentSha,
+            });
+            if (!Array.isArray(data) &&
+                data.type === "file" &&
+                data.encoding === "base64") {
+                const content = Buffer.from(data.content, "base64").toString("utf-8");
+                // Truncate at 3000 chars
+                const truncated = content.length > 3000
+                    ? content.substring(0, 3000) + "\n... [truncated]"
+                    : content;
+                const safeContent = escapeCodeFence(truncated);
+                fileContents.push(`**${path}** (before patch):\n\`\`\`\n${safeContent}\n\`\`\``);
+            }
+        }
+        catch {
+            // Skip files that don't exist or fail to fetch
+            continue;
+        }
+    }
+    return fileContents.length > 0
+        ? `## Modified Files Context\n\n${fileContents.join("\n\n")}\n\n`
+        : "";
+}
 function escapeMarkdown(text) {
     return text
         .replace(/\\/g, "\\\\")
@@ -46240,6 +46297,8 @@ Analyze the following commit and determine if it is patching an EXPLOITABLE secu
 **Message:**
 {message}
 {prSection}
+{repoContext}
+
 ## Diff
 {diff}
 
@@ -46285,7 +46344,7 @@ Example proofOfConcept formats:
 - For Command Injection: \`; rm -rf /\` appended to the filename
 
 If you cannot write a specific, concrete proof of concept, set isVulnerabilityPatch to false.`;
-async function analyzeCommit(commit) {
+async function analyzeCommit(commit, repoContext = "") {
     let prSection = "";
     if (commit.pullRequest) {
         const pr = commit.pullRequest;
@@ -46303,9 +46362,10 @@ ${pr.body ? `**Description:**\n${pr.body.substring(0, 1000)}${pr.body.length > 1
         "{date}": commit.date,
         "{message}": commit.message,
         "{prSection}": prSection,
+        "{repoContext}": repoContext,
         "{diff}": commit.diff,
     };
-    const prompt = ANALYSIS_PROMPT.replace(/\{sha\}|\{author\}|\{date\}|\{message\}|\{prSection\}|\{diff\}/g, (match) => replacements[match] ?? match);
+    const prompt = ANALYSIS_PROMPT.replace(/\{sha\}|\{author\}|\{date\}|\{message\}|\{prSection\}|\{repoContext\}|\{diff\}/g, (match) => replacements[match] ?? match);
     const response = await client.messages.create({
         model: modelId,
         max_tokens: 1024,
@@ -46333,6 +46393,55 @@ ${pr.body ? `**Description:**\n${pr.body.substring(0, 1000)}${pr.body.length > 1
         };
     }
 }
+async function judgeAnalysis(commit, primaryAnalysis, judgeModel, repoContext = "") {
+    const prompt = `You are reviewing a security vulnerability assessment. The primary analyzer detected a potential vulnerability.
+
+## Commit Being Reviewed
+**SHA:** ${commit.sha}
+**Message:** ${commit.message}
+
+## Primary Analysis Conclusion
+- **Vulnerability Type:** ${primaryAnalysis.vulnerabilityType}
+- **Severity:** ${primaryAnalysis.severity}
+- **Description:** ${primaryAnalysis.description}
+- **Proof of Concept:** ${primaryAnalysis.proofOfConcept}
+
+${repoContext}## Code Changes (Diff)
+${commit.diff}
+
+## Your Task
+Review the commit diff above and the primary analysis conclusion. Do you AGREE or DISAGREE that this commit patches a real, exploitable vulnerability?
+
+Consider:
+1. Does the diff show an actual security fix?
+2. Is the proof of concept realistic and specific?
+3. Could the vulnerability actually be exploited?
+
+Respond with JSON only:
+{
+  "agrees": boolean,
+  "reasoning": string (2-3 sentences explaining your decision)
+}`;
+    const response = await client.messages.create({
+        model: judgeModel,
+        max_tokens: 1024,
+        messages: [
+            { role: "user", content: prompt },
+            { role: "assistant", content: "{" },
+        ],
+    });
+    const content = response.content[0];
+    if (content.type !== "text") {
+        return { agrees: true, reasoning: "Judge failed to respond" };
+    }
+    try {
+        const analysis = JSON.parse("{" + content.text);
+        return analysis;
+    }
+    catch {
+        return { agrees: true, reasoning: "Judge parse failed" };
+    }
+}
 
 ;// CONCATENATED MODULE: ./src/index.ts
 
@@ -46403,6 +46512,9 @@ function getInputs() {
         issueRepo,
         model: core.getInput("model") || "claude-sonnet-4-20250514",
         maxCommits: parseInt(core.getInput("max-commits") || "50", 10),
+        enableRepoContext: core.getBooleanInput("enable-repo-context"),
+        enableJudge: core.getBooleanInput("enable-judge"),
+        judgeModel: core.getInput("judge-model") || "",
     };
 }
 function loadState(stateFile) {
@@ -46461,9 +46573,38 @@ async function run() {
                     core.info(`  Analyzing commit ${commit.sha.substring(0, 7)}...`);
                     outputs.analyzedCommits++;
                     try {
-                        const analysis = await analyzeCommit(commit);
+                        // Fetch repository context if enabled
+                        let repoContext = "";
+                        if (inputs.enableRepoContext) {
+                            try {
+                                repoContext = await getModifiedFilesContent(repo, commit.diff, commit.sha);
+                                if (repoContext) {
+                                    core.info(`    Fetched context for modified files`);
+                                }
+                            }
+                            catch (error) {
+                                core.warning(`    Failed to fetch repo context: ${error}`);
+                            }
+                        }
+                        const analysis = await analyzeCommit(commit, repoContext);
                         if (analysis.isVulnerabilityPatch) {
                             core.warning(`    VULNERABILITY DETECTED: ${analysis.vulnerabilityType} (${analysis.severity})`);
+                            // Run judge if enabled
+                            if (inputs.enableJudge) {
+                                const judgeModelToUse = inputs.judgeModel || inputs.model;
+                                try {
+                                    const judge = await judgeAnalysis(commit, analysis, judgeModelToUse, repoContext);
+                                    if (!judge.agrees) {
+                                        core.info(`    Judge DISAGREED: ${judge.reasoning}`);
+                                        core.info(`    Skipping issue creation`);
+                                        continue;
+                                    }
+                                    core.info(`    Judge CONFIRMED: ${judge.reasoning}`);
+                                }
+                                catch (error) {
+                                    core.warning(`    Judge failed: ${error}, proceeding anyway`);
+                                }
+                            }
                             const vulnerability = {
                                 repo,
                                 commit,

dist/types.d.ts

@@ -14,6 +14,9 @@ export interface ActionInputs {
     };
     model: string;
     maxCommits: number;
+    enableRepoContext: boolean;
+    enableJudge: boolean;
+    judgeModel: string;
 }
 export interface State {
     [repoKey: string]: string;
@@ -43,6 +46,10 @@ export interface VulnerabilityAnalysis {
     affectedCode: string | null;
     proofOfConcept: string | null;
 }
+export interface JudgeAnalysis {
+    agrees: boolean;
+    reasoning: string;
+}
 export interface DetectedVulnerability {
     repo: RepoConfig;
     commit: CommitInfo;