diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 953a5b7ec..1b9812ad6 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -205,6 +205,8 @@ Done when: policy match handles longer parser phrases without private command grammar. - [x] A synthetic workload corpus covers ordinary search, read, pipeline, redirect, and file-change commands without production command text. +- [x] Sanitized post-swap evidence classifies 51 prompts across 202 shell calls. + Eleven executable live cases pin the intended allow and prompt boundaries. - [x] A safe pipeline stage can compose with a stored grant for each stage that still requires approval. - [x] A prompt excludes a safe stage from the approval candidates that the user diff --git a/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json b/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json index 06a1edb6f..78e36d2e4 100644 --- a/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json +++ b/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json @@ -410,6 +410,195 @@ "expectedFinal": { "outcome": "RequiresApproval", "reason": "UncoveredCandidates", "approvalCandidates": ["gh pr close"], "isMessy": false, "agentCorrection": null } } ], + "liveRegressionCases": [ + { + "sourceEvidenceId": "S18", + "classification": "NetclawPolicyDebt", + "targetOutcome": "Allow", + "policyCase": { + "id": "L01", + "category": "BoundedGrepAndList", + "command": "grep -Rni 'TypeMarker' src | head; ls src/Observability", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "Allow", "denyReason": null, "approvalCandidates": null, "isMessy": null, "optionKeys": null, "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S22", + "classification": "NetclawPolicyDebt", + "targetOutcome": "Allow", + "policyCase": { + "id": "L02", + "category": "BoundedGrepPipeline", + "command": "grep -Rni 'TestMarker' tests | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "Allow", "denyReason": null, "approvalCandidates": null, "isMessy": null, "optionKeys": null, "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S40", + "classification": "NetclawPolicyDebt", + "targetOutcome": "Allow", + "policyCase": { + "id": "L03", + "category": "BoundedMultiRootGrep", + "command": "grep -Rni 'PolicyMarker' src tests | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "Allow", "denyReason": null, "approvalCandidates": null, "isMessy": null, "optionKeys": null, "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S16", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L04", + "category": "BroadFindDiscovery", + "command": "find . -maxdepth 3 -name '*.csproj'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["find"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S11", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L05", + "category": "DiskSummaryWithSort", + "command": "du -sh ./* | sort -rh | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["sort"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S10", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L06", + "category": "RemotePullRequestCreate", + "command": "gh pr create --repo example/project --title 'Example' --body 'Example body'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["gh pr create"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S24", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L07", + "category": "RemotePullRequestMerge", + "command": "gh pr merge 50 --repo example/project --auto --squash", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["gh pr merge"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S20", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L08", + "category": "RemoteGitPush", + "command": "git push origin feature/example", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["git push origin"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S13", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L09", + "category": "RecursiveDelete", + "command": "rm -rf ./cache/*", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["rm"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceId": "S21", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L10", + "category": "DynamicDiagnosticRange", + "command": "dotnet test > /tmp/test.log; sed -n \"$(grep -n 'FAIL' /tmp/test.log | cut -d: -f1),+4p\" /tmp/test.log", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceId": "S44", + "classification": "NetclawPolicyDebt", + "targetOutcome": "Allow", + "policyCase": { + "id": "L11", + "category": "BoundedResourceGrep", + "command": "grep -Rni 'ResourceMarker' src | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", + "projectDirectory": "/work", + "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "Allow", "denyReason": null, "approvalCandidates": null, "isMessy": null, "optionKeys": null, "actorCheckCount": 1 } + } + } + ], "adversarialCases": [ { "id": "A01", diff --git a/openspec/changes/structure-shell-approval-policy/evidence/post-1925-extended-approval-harvest.json b/openspec/changes/structure-shell-approval-policy/evidence/post-1925-extended-approval-harvest.json new file mode 100644 index 000000000..0705aa79f --- /dev/null +++ b/openspec/changes/structure-shell-approval-policy/evidence/post-1925-extended-approval-harvest.json @@ -0,0 +1,399 @@ +{ + "schemaVersion": 1, + "sourceRuntime": { + "version": "0.26.0", + "commit": "ba83530", + "windowStartUtc": "2026-08-13T20:29:54Z", + "windowEndUtc": "2026-08-13T22:22:43Z", + "shellCallCount": 140, + "approvalPromptCount": 42 + }, + "sanitization": { + "home": "/home/user", + "repositoryRoot": "/work/project", + "worktreeRoot": "/work/project-worktree", + "remoteRepository": "example/project", + "internalHost": "service.example.invalid", + "inlineBodies": "Private titles, bodies, messages, branches, and source identifiers were replaced by semantic placeholders." + }, + "cases": [ + { + "id": "S10", + "sourcePromptTimeUtc": "2026-08-13T20:30:00.488Z", + "commandShape": "gh pr create --repo example/project --title '' --body ''", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S11", + "sourcePromptTimeUtc": "2026-08-13T20:33:31.023Z", + "commandShape": "df -h /work; du -sh /work/project/* | sort -rh | head", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent combined broad disk traversal and sorting for a diagnostic summary." + }, + { + "id": "S12", + "sourcePromptTimeUtc": "2026-08-13T20:34:55.384Z", + "commandShape": "du -sh /work/project/* /work/project/.[!.]* | sort -rh | head", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent used broad repository traversal and authored globs for a size query." + }, + { + "id": "S13", + "sourcePromptTimeUtc": "2026-08-13T20:35:16.622Z", + "commandShape": "rm -rf /work/project/Trash/*; ls -la /work/project/Trash; df -h /work", + "observedResponse": "Session", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call recursively deletes files." + }, + { + "id": "S14", + "sourcePromptTimeUtc": "2026-08-13T20:37:24.116Z", + "commandShape": "gh pr create --repo example/project --title '' --body '<body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S15", + "sourcePromptTimeUtc": "2026-08-13T20:40:31.508Z", + "commandShape": "gh pr merge 49 --repo example/project --auto --squash; gh pr view 49 --repo example/project", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S16", + "sourcePromptTimeUtc": "2026-08-13T20:42:08.442Z", + "commandShape": "grep -Rni '<project marker>' /work/project; find /work/project -maxdepth 3 -name '*.csproj'", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent combined recursive search and filesystem discovery instead of using bounded repository queries." + }, + { + "id": "S17", + "sourcePromptTimeUtc": "2026-08-13T20:42:59.113Z", + "commandShape": "grep -Rni '<observability marker>' /work/project; find /work/project -maxdepth 4 -type d", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent combined recursive content search with broad directory discovery." + }, + { + "id": "S18", + "sourcePromptTimeUtc": "2026-08-13T20:43:43.563Z", + "commandShape": "grep -Rni '<type marker>' /work/project/src | head; ls /work/project/src/Observability", + "observedResponse": "Once", + "classification": "NetclawPolicyDebt", + "owner": "NetclawPolicy", + "reason": "The bounded source search and directory listing are read-only within the declared project." + }, + { + "id": "S19", + "sourcePromptTimeUtc": "2026-08-13T20:44:12.915Z", + "commandShape": "grep -Rni '<analyzer marker>' /work/project; find /work/project -maxdepth 4 -iname '*analyzer*'", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent mixed a repository search with broad filesystem discovery." + }, + { + "id": "S20", + "sourcePromptTimeUtc": "2026-08-13T20:46:21.027Z", + "commandShape": "git add <files>; git commit -m '<message>'; git push origin <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a commit and mutates a remote branch." + }, + { + "id": "S21", + "sourcePromptTimeUtc": "2026-08-13T20:48:05.240Z", + "commandShape": "dotnet test > /tmp/test.log; sed -n \"$(grep -n '<failure>' /tmp/test.log | cut -d: -f1),+4p\" /tmp/test.log", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call runs a build tool, writes temporary output, and constructs a dynamic range." + }, + { + "id": "S22", + "sourcePromptTimeUtc": "2026-08-13T20:55:22.245Z", + "commandShape": "grep -Rni '<test marker>' /work/project/tests | head", + "observedResponse": "Once", + "classification": "NetclawPolicyDebt", + "owner": "NetclawPolicy", + "reason": "The bounded test-source search is read-only within the declared project." + }, + { + "id": "S23", + "sourcePromptTimeUtc": "2026-08-13T20:58:21.870Z", + "commandShape": "gh pr create --repo example/project --title '<title>' --body '<body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S24", + "sourcePromptTimeUtc": "2026-08-13T21:06:01.313Z", + "commandShape": "gh pr merge 50 --repo example/project --auto --squash; gh pr view 50 --repo example/project", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S25", + "sourcePromptTimeUtc": "2026-08-13T21:22:21.625Z", + "commandShape": "git fetch origin; git merge origin/dev; git push origin <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call fetches remote state, merges it, and pushes a branch." + }, + { + "id": "S26", + "sourcePromptTimeUtc": "2026-08-13T21:23:09.549Z", + "commandShape": "git merge origin/dev; git push origin <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call mutates repository history and a remote branch." + }, + { + "id": "S27", + "sourcePromptTimeUtc": "2026-08-13T21:32:36.466Z", + "commandShape": "git fetch origin; git ls-remote --tags origin; git tag <version>; git push origin <version>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call contacts a remote and creates a remote tag." + }, + { + "id": "S28", + "sourcePromptTimeUtc": "2026-08-13T21:33:53.811Z", + "commandShape": "git show <ref>:<path>; git diff <refs> -- <path>", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent issued repository reads before declaring the project scope." + }, + { + "id": "S29", + "sourcePromptTimeUtc": "2026-08-13T21:34:51.893Z", + "commandShape": "gh pr create --repo example/project --title '<release title>' --body '<release body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S30", + "sourcePromptTimeUtc": "2026-08-13T21:35:03.577Z", + "commandShape": "gh pr merge <number> --repo example/project --auto --squash", + "observedResponse": "Session", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S31", + "sourcePromptTimeUtc": "2026-08-13T21:35:36.673Z", + "commandShape": "git status --short; git log -1 --oneline; git worktree list", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent issued repository diagnostics before declaring the project scope." + }, + { + "id": "S32", + "sourcePromptTimeUtc": "2026-08-13T21:35:56.411Z", + "commandShape": "git show --stat <ref>; git show <ref>:<path>", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent issued repository reads before declaring the project scope." + }, + { + "id": "S33", + "sourcePromptTimeUtc": "2026-08-13T21:40:10.994Z", + "commandShape": "git tag <version>; git push origin <version>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a local and remote tag." + }, + { + "id": "S34", + "sourcePromptTimeUtc": "2026-08-13T21:43:43.423Z", + "commandShape": "git remote -v", + "observedResponse": "Once", + "classification": "AgentAlignmentDebt", + "owner": "AgentGuidance", + "reason": "The agent queried repository configuration before declaring the project scope." + }, + { + "id": "S35", + "sourcePromptTimeUtc": "2026-08-13T21:43:53.009Z", + "commandShape": "gh pr create --repo example/project --title '<deployment title>' --body '<deployment body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S36", + "sourcePromptTimeUtc": "2026-08-13T21:44:43.720Z", + "commandShape": "git fetch origin", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call contacts a remote and updates repository metadata." + }, + { + "id": "S37", + "sourcePromptTimeUtc": "2026-08-13T21:54:08.405Z", + "commandShape": "git worktree remove --force /work/project-worktree; git branch -D <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call forcefully removes a worktree and deletes a branch." + }, + { + "id": "S38", + "sourcePromptTimeUtc": "2026-08-13T21:55:07.302Z", + "commandShape": "git branch -D <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call forcefully deletes a branch." + }, + { + "id": "S39", + "sourcePromptTimeUtc": "2026-08-13T21:55:21.035Z", + "commandShape": "for repo in /work/project/*; do git -C \"$repo\" branch -D <branch>; done", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call dynamically deletes branches across repositories." + }, + { + "id": "S40", + "sourcePromptTimeUtc": "2026-08-13T22:01:04.710Z", + "commandShape": "grep -Rni '<policy marker>' /work/project/src /work/project/tests | head", + "observedResponse": "Once", + "classification": "NetclawPolicyDebt", + "owner": "NetclawPolicy", + "reason": "The bounded source and test search is read-only within the declared project." + }, + { + "id": "S41", + "sourcePromptTimeUtc": "2026-08-13T22:02:32.082Z", + "commandShape": "gh pr create --repo example/project --title '<title>' --body '<body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S42", + "sourcePromptTimeUtc": "2026-08-13T22:02:52.528Z", + "commandShape": "gh pr merge <number> --repo example/project --auto --squash", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S43", + "sourcePromptTimeUtc": "2026-08-13T22:04:50.582Z", + "commandShape": "git add <files>; git commit -m '<message>'; git push; gh pr create --repo example/project --title '<title>' --body '<body>'", + "observedResponse": "Session", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a commit, mutates a remote branch, and creates a pull request." + }, + { + "id": "S44", + "sourcePromptTimeUtc": "2026-08-13T22:05:07.704Z", + "commandShape": "grep -Rni '<resource marker>' /work/project/src | head", + "observedResponse": "Once", + "classification": "NetclawPolicyDebt", + "owner": "NetclawPolicy", + "reason": "The bounded source search is read-only within the declared project." + }, + { + "id": "S45", + "sourcePromptTimeUtc": "2026-08-13T22:05:31.537Z", + "commandShape": "grep -Rni '<skill marker>' /work/project/tests /work/project/feeds | head", + "observedResponse": "Once", + "classification": "NetclawPolicyDebt", + "owner": "NetclawPolicy", + "reason": "The bounded test and skill search is read-only within the declared project." + }, + { + "id": "S46", + "sourcePromptTimeUtc": "2026-08-13T22:06:36.288Z", + "commandShape": "gh pr merge <number> --repo example/project --auto --squash", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S47", + "sourcePromptTimeUtc": "2026-08-13T22:09:00.656Z", + "commandShape": "gh pr merge <number> --repo example/project --auto --squash", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + }, + { + "id": "S48", + "sourcePromptTimeUtc": "2026-08-13T22:14:10.416Z", + "commandShape": "git fetch origin; git tag <version>; git push origin <version>; git ls-remote --tags origin; gh run list --repo example/project", + "observedResponse": "Session", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call contacts a remote and creates a remote tag." + }, + { + "id": "S49", + "sourcePromptTimeUtc": "2026-08-13T22:14:26.630Z", + "commandShape": "git fetch origin; git checkout <branch>; git merge origin/dev; git push origin <branch>; git worktree add /work/project-worktree <branch>", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call updates local history, pushes a remote branch, and creates a worktree." + }, + { + "id": "S50", + "sourcePromptTimeUtc": "2026-08-13T22:21:22.673Z", + "commandShape": "gh pr create --repo example/project --title '<deployment title>' --body '<deployment body>'", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call creates a remote pull request." + }, + { + "id": "S51", + "sourcePromptTimeUtc": "2026-08-13T22:22:42.720Z", + "commandShape": "gh pr merge <number> --repo example/project --auto --squash", + "observedResponse": "Once", + "classification": "ExpectedApproval", + "owner": "ApprovalPolicy", + "reason": "The call schedules a remote pull request merge." + } + ] +} diff --git a/openspec/changes/structure-shell-approval-policy/tasks.md b/openspec/changes/structure-shell-approval-policy/tasks.md index 2c03045dd..b239f3985 100644 --- a/openspec/changes/structure-shell-approval-policy/tasks.md +++ b/openspec/changes/structure-shell-approval-policy/tasks.md @@ -13,6 +13,7 @@ - [x] 1.5 Add adversarial dynamic identity, redirect, protected path, prefix collision, runtime loop, wrapper, provider, and unsafe-catalog cases. - [x] 1.6 Run the PII audit and manually inspect every command and fixture. +- [x] 1.7 Add sanitized post-swap evidence and executable live regression cases. ## 2. Typed coordinator and actor protocol diff --git a/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs b/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs index 8927a243f..c65d380a3 100644 --- a/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs +++ b/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs @@ -90,48 +90,71 @@ public async Task Adversarial_policy_fixtures_fail_closed_through_the_coordinato foreach (var policyCase in catalog.AdversarialCases) { - var invocation = CreateInvocation(catalog.FixtureDefaults, policyCase); - var approvals = CreateApprovals(policyCase.Available); - await using var harness = await ShellApprovalHarness.CreateAsync( - policyCase.Id, - invocation, - approvals, - fixture.ActorSystem, - TestContext.Current.CancellationToken, - timeProvider, - new ShellApprovalHarnessScope( - policyCase.ProjectDirectory, - policyCase.SessionDirectory, - catalog.FixtureDefaults.Session.SessionId, - policyCase.Available.OneTimeApprovalKeys), - policyCase.UseBundledSafeCatalog - ? null - : CreateSafeVerbs(policyCase.Available, invocation.CreateEnvironment())); + await AssertPolicyCaseAsync(catalog, timeProvider, policyCase); + } + } - var decision = await harness.EvaluateDecisionAsync(TestContext.Current.CancellationToken); - TestContext.Current.TestOutputHelper?.WriteLine( - $"{policyCase.Id} ({policyCase.Category}): outcome={decision.Outcome}; " - + $"deny={decision.DenyReason}; " - + $"candidates={string.Join(", ", decision.ApprovalContext?.CandidateVerbs ?? [])}; " - + $"messy={decision.ApprovalContext?.IsMessy}; " - + $"checks={harness.ApprovalService.CheckCount}; " - + $"allow={decision.AllowReason}; " - + $"matches={string.Join(", ", decision.ApprovalMatches.Select(item => item.Pattern))}; " - + $"trace={string.Join(", ", decision.ShellPolicyTrace.Rows.Select(row => $"{row.Stage}/{row.Reason}"))}"); - - Assert.Equal(ParseOutcome(policyCase.Expected.Outcome), decision.Outcome); - Assert.Equal(policyCase.Expected.DenyReason, decision.DenyReason); - Assert.Equal( - policyCase.Expected.ApprovalCandidates, - decision.ApprovalContext?.CandidateVerbs); - Assert.Equal(policyCase.Expected.IsMessy, decision.ApprovalContext?.IsMessy); - Assert.Equal( - policyCase.Expected.OptionKeys, - decision.ApprovalContext?.Options.Select(option => option.Key.Value).ToList()); - Assert.Equal(policyCase.Expected.ActorCheckCount, harness.ApprovalService.CheckCount); + [Fact] + public async Task Live_regression_fixtures_pin_current_policy_outcomes() + { + var catalog = JsonSerializer.Deserialize( + File.ReadAllBytes(EvidencePath()), + ShellPolicyFixtureJsonContext.Default.PolicyFixtureCatalog) + ?? throw new InvalidDataException("The policy fixture catalog has no root object."); + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse( + catalog.FixtureDefaults.ClockUtc, + CultureInfo.InvariantCulture)); + + foreach (var liveCase in catalog.LiveRegressionCases) + { + await AssertPolicyCaseAsync(catalog, timeProvider, liveCase.PolicyCase); } } + private async Task AssertPolicyCaseAsync( + PolicyFixtureCatalog catalog, + FakeTimeProvider timeProvider, + PolicyAdversarialCase policyCase) + { + var invocation = CreateInvocation(catalog.FixtureDefaults, policyCase); + var approvals = CreateApprovals(policyCase.Available); + await using var harness = await ShellApprovalHarness.CreateAsync( + policyCase.Id, + invocation, + approvals, + fixture.ActorSystem, + TestContext.Current.CancellationToken, + timeProvider, + new ShellApprovalHarnessScope( + policyCase.ProjectDirectory, + policyCase.SessionDirectory, + catalog.FixtureDefaults.Session.SessionId, + policyCase.Available.OneTimeApprovalKeys), + policyCase.UseBundledSafeCatalog + ? null + : CreateSafeVerbs(policyCase.Available, invocation.CreateEnvironment())); + + var decision = await harness.EvaluateDecisionAsync(TestContext.Current.CancellationToken); + TestContext.Current.TestOutputHelper?.WriteLine( + $"{policyCase.Id} ({policyCase.Category}): outcome={decision.Outcome}; " + + $"deny={decision.DenyReason}; " + + $"candidates={string.Join(", ", decision.ApprovalContext?.CandidateVerbs ?? [])}; " + + $"messy={decision.ApprovalContext?.IsMessy}; " + + $"checks={harness.ApprovalService.CheckCount}; " + + $"allow={decision.AllowReason}; " + + $"matches={string.Join(", ", decision.ApprovalMatches.Select(item => item.Pattern))}; " + + $"trace={string.Join(", ", decision.ShellPolicyTrace.Rows.Select(row => $"{row.Stage}/{row.Reason}"))}"); + + Assert.Equal(ParseOutcome(policyCase.Expected.Outcome), decision.Outcome); + Assert.Equal(policyCase.Expected.DenyReason, decision.DenyReason); + Assert.Equal(policyCase.Expected.ApprovalCandidates, decision.ApprovalContext?.CandidateVerbs); + Assert.Equal(policyCase.Expected.IsMessy, decision.ApprovalContext?.IsMessy); + Assert.Equal( + policyCase.Expected.OptionKeys, + decision.ApprovalContext?.Options.Select(option => option.Key.Value).ToList()); + Assert.Equal(policyCase.Expected.ActorCheckCount, harness.ApprovalService.CheckCount); + } + private static void AssertProjectedCandidates( PolicyFixtureCase policyCase, ShellApprovalInvocation invocation) diff --git a/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs b/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs index 1811319f0..cdf77d921 100644 --- a/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs +++ b/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs @@ -20,6 +20,7 @@ public sealed partial class ShellApprovalEvidenceContractTests private const string PolicyFixturesFile = "netclaw-policy-fixtures.json"; private const string PostMergeHarvestFile = "post-1890-approval-harvest.json"; private const string PostSwapHarvestFile = "post-1925-binary-swap-approval-harvest.json"; + private const string ExtendedPostSwapHarvestFile = "post-1925-extended-approval-harvest.json"; private const string ApprovalMatrixSha256 = "0169105efe87b345d9a82d777ef86909e31fa81a5255cc0cc30f32fbe4d0d6b0"; @@ -65,6 +66,31 @@ public void Policy_fixtures_load_exact_authority_and_trace_fields() Assert.Equal("/work", fixtures.FixtureDefaults.ProjectDirectory); Assert.Null(fixtures.FixtureDefaults.InheritedWorkingDirectory); Assert.Equal(10, fixtures.Cases.Count); + var sourceEvidence = new[] { PostSwapHarvestFile, ExtendedPostSwapHarvestFile } + .SelectMany(file => JsonSerializer.Deserialize( + File.ReadAllBytes(EvidencePath(file)), + ShellApprovalEvidenceJsonContext.Default.PostMergeApprovalHarvest)! + .Cases) + .ToDictionary(item => item.Id); + Assert.Equal( + Enumerable.Range(1, 11).Select(number => $"L{number:00}"), + fixtures.LiveRegressionCases.Select(item => item.PolicyCase.Id)); + Assert.Equal( + ["S18", "S22", "S40", "S16", "S11", "S10", "S24", "S20", "S13", "S21", "S44"], + fixtures.LiveRegressionCases.Select(item => item.SourceEvidenceId)); + Assert.All(fixtures.LiveRegressionCases, item => + { + Assert.Contains( + item.Classification, + new[] { "ExpectedApproval", "AgentAlignmentDebt", "NetclawPolicyDebt" }); + Assert.Contains(item.TargetOutcome, new[] { "Allow", "RequiresApproval" }); + Assert.Equal(item.TargetOutcome, item.PolicyCase.Expected.Outcome); + Assert.Equal(sourceEvidence[item.SourceEvidenceId].Classification, item.Classification); + }); + Assert.Equal(4, fixtures.LiveRegressionCases.Count(item => item.TargetOutcome == "Allow")); + Assert.Equal( + 7, + fixtures.LiveRegressionCases.Count(item => item.TargetOutcome == "RequiresApproval")); Assert.Equal( Enumerable.Range(1, 12).Select(number => $"A{number:00}"), fixtures.AdversarialCases.Select(item => item.Id)); @@ -379,6 +405,46 @@ public void Post_swap_harvest_classifies_every_prompt_in_the_frozen_window() }); } + [Fact] + public void Extended_post_swap_harvest_classifies_every_prompt_in_the_frozen_window() + { + var harvest = JsonSerializer.Deserialize( + File.ReadAllBytes(EvidencePath(ExtendedPostSwapHarvestFile)), + ShellApprovalEvidenceJsonContext.Default.PostMergeApprovalHarvest) + ?? throw new InvalidDataException( + $"{ExtendedPostSwapHarvestFile} has no root object."); + + Assert.Equal(1, harvest.SchemaVersion); + Assert.Equal("0.26.0", harvest.SourceRuntime.Version); + Assert.Equal("ba83530", harvest.SourceRuntime.Commit); + Assert.Equal(140, harvest.SourceRuntime.ShellCallCount); + Assert.Equal(42, harvest.SourceRuntime.ApprovalPromptCount); + Assert.Equal( + Enumerable.Range(10, 42).Select(number => $"S{number:00}"), + harvest.Cases.Select(item => item.Id)); + Assert.Equal( + harvest.Cases.Select(item => item.SourcePromptTimeUtc).Order(), + harvest.Cases.Select(item => item.SourcePromptTimeUtc)); + Assert.All(harvest.Cases, item => + { + Assert.InRange( + item.SourcePromptTimeUtc, + DateTimeOffset.Parse(harvest.SourceRuntime.WindowStartUtc), + DateTimeOffset.Parse(harvest.SourceRuntime.WindowEndUtc)); + }); + Assert.Equal(28, harvest.Cases.Count(item => item.Classification == "ExpectedApproval")); + Assert.Equal(9, harvest.Cases.Count(item => item.Classification == "AgentAlignmentDebt")); + Assert.Equal(5, harvest.Cases.Count(item => item.Classification == "NetclawPolicyDebt")); + Assert.DoesNotContain( + harvest.Cases, + item => item.Classification == "ShellSyntaxTreeFactGap"); + Assert.All(harvest.Cases, item => + { + Assert.False(string.IsNullOrWhiteSpace(item.CommandShape)); + Assert.False(string.IsNullOrWhiteSpace(item.Reason)); + }); + } + private static void AssertValuePart(PolicyValuePart expected, ShellValueDomain actual) { if (expected.Exact is { } exact) diff --git a/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs b/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs index 979de6e5a..629ecd5c7 100644 --- a/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs +++ b/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs @@ -15,9 +15,22 @@ internal sealed record PolicyFixtureCatalog public required List<PolicyFixtureCase> Cases { get; init; } + public required List<PolicyLiveRegressionCase> LiveRegressionCases { get; init; } + public required List<PolicyAdversarialCase> AdversarialCases { get; init; } } +internal sealed record PolicyLiveRegressionCase +{ + public required string SourceEvidenceId { get; init; } + + public required string Classification { get; init; } + + public required string TargetOutcome { get; init; } + + public required PolicyAdversarialCase PolicyCase { get; init; } +} + internal sealed record PolicyFixtureDefaults { public required string ToolName { get; init; }