diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 10f5c835f..c43e78da5 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -212,6 +212,8 @@ Done when: Eleven executable live cases pin the intended allow and prompt boundaries. - [x] Post-1952 live evidence froze 285 shell calls and 69 prompts. Twenty-one sanitized cases sample expected, guidance, and ShellSyntaxTree gaps. + The coordinator executes one identity-free regression for each sampled case. + Each regression locks its evidence source, approval shape, and actor contact. - [x] A safe pipeline stage can compose with a stored grant for each stage that still requires approval. - [x] A prompt excludes a safe stage from the approval candidates that the user diff --git a/openspec/changes/extend-live-approval-regression-corpus/.openspec.yaml b/openspec/changes/extend-live-approval-regression-corpus/.openspec.yaml new file mode 100644 index 000000000..4af864176 --- /dev/null +++ b/openspec/changes/extend-live-approval-regression-corpus/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/extend-live-approval-regression-corpus/design.md b/openspec/changes/extend-live-approval-regression-corpus/design.md new file mode 100644 index 000000000..64aa9bbdb --- /dev/null +++ b/openspec/changes/extend-live-approval-regression-corpus/design.md @@ -0,0 +1,121 @@ +## Context + +The existing policy fixture catalog executes D acceptance cases, A adversarial +cases, and eleven earlier live regressions through the real shell policy +coordinator. The post-1952 harvest adds 21 representative prompts, but its +`commandShape` values are display-safe evidence. Several contain angle-bracket +placeholders that would change Bash grammar if executed. Copying those strings +into the coordinator fixture would therefore test the redaction syntax rather +than the observed approval shape. + +This change affects only source-controlled evidence and tests. It does not +change an actor boundary, policy stage, grant store, session history, public +API, or persisted representation. The coordinator fixture continues to use an +in-process approval actor stub and the canonical bundled safe catalog. + +## Goals / Non-Goals + +**Goals:** + +- represent every T01-T21 harvested case with an identity-free command that + preserves the policy-relevant shell structure; +- execute each command through the real coordinator with exact expected + outcome, approval shape, and actor-contact count; +- bind each regression to its source file, source evidence ID, classification, + and target outcome; +- make accidental command or expectation drift visible through a locked digest; + and +- preserve current strict behavior for expected approvals, agent-alignment + cases, and unresolved ShellSyntaxTree facts. + +**Non-Goals:** + +- changing production policy or reviewed-safe catalogs; +- parsing `gh` operations, Docker behavior, or Bash arithmetic in Netclaw; +- granting authority from an evidence classification; +- claiming the 21 cases exhaust the source traffic window; or +- changing actor messages, persistence, recovery, or runtime failure handling. + +## Decisions + +### Curate executable commands instead of executing display redactions + +Each new live regression will retain the command's control flow, executable +chain, path boundaries, redirects, and dynamic constructs while replacing +identities with ordinary quoted literals. Angle-bracket placeholders will not +appear in executable commands because Bash treats them as redirects. + +The alternative was to execute `commandShape` directly. That would make cases +such as `` and `` semantically false and could turn a +read into a redirect or parse failure. + +### Bind every live row to an explicit evidence file + +`PolicyLiveRegressionCase` will add a required internal-only +`SourceEvidenceFile` field. Contract validation will resolve the pair +`(SourceEvidenceFile, SourceEvidenceId)` and compare the source classification +with the fixture classification. This avoids relying on globally unique S/T +identifiers and makes future harvest additions unambiguous. + +The field is test-only JSON. It does not affect a public or durable runtime +contract. + +### Lock the executable live-regression section as one evidence artifact + +The contract test will compute a deterministic digest over the serialized +`liveRegressionCases` section. The digest will also include each linked source +`commandShape` and classification. A command, source shape, evidence link, +classification, outcome, correction, option, or actor count change therefore +requires an explicit evidence review. Semantic tests will still execute every +row through the coordinator. The digest is a drift alarm, not a substitute for +behavior. + +The alternative was a large hard-coded command dictionary in C#. +That would duplicate the corpus and make review harder. + +### Preserve classifications without translating them into authority + +All new T rows currently target `RequiresApproval`. `ExpectedApproval` remains +a product-appropriate prompt. `AgentAlignmentDebt` remains promptable because +guidance, file tools, or session scratch are the preferred remedy. +`ShellSyntaxTreeFactGap` remains promptable until a general parser-owned fact +exists. No classification can directly cover a policy candidate. + +### Reuse the existing coordinator harness + +The new rows use `PolicyAdversarialCase` and +`ShellPolicyEvidenceFixtureTests.Live_regression_fixtures_pin_current_policy_outcomes`. +This preserves the real syntax analysis, path policy, safe-catalog, +coordinator, approval-context, and actor-check route without adding a second +evaluator or fixture-specific production seam. + +## Risks / Trade-offs + +- **Curated commands can diverge from live intent** -> Preserve the + policy-relevant structure, link every row to the source evidence, and review + the paired source and curated command together. +- **A locked digest can be mechanically refreshed** -> Require the semantic + coordinator assertions and exact source-classification linkage to pass too. +- **Platform-dependent paths can make fixtures flaky** -> Use the declared + Bash/Linux environment with canonical POSIX fixture roots and identity-free + external paths. +- **A future parser release can legitimately change outcomes** -> Treat the + resulting fixture failure as an explicit corpus review, then update the + expectation and digest together if the new behavior is intended. + +## Migration Plan + +1. Extend the test-only fixture schema and existing L rows with explicit source + evidence files. +2. Add L12-L32 for T01-T21 and record current coordinator results. +3. Add linkage, digest, uniqueness, classification, and mutation coverage. +4. Run focused coordinator and evidence suites plus the repository quality + gates. + +Rollback removes the new test-only rows and schema field. No runtime data or +authority requires migration. + +## Open Questions + +None. Production changes for any future general parser fact require a separate +ShellSyntaxTree and Netclaw change. diff --git a/openspec/changes/extend-live-approval-regression-corpus/proposal.md b/openspec/changes/extend-live-approval-regression-corpus/proposal.md new file mode 100644 index 000000000..9950f6c53 --- /dev/null +++ b/openspec/changes/extend-live-approval-regression-corpus/proposal.md @@ -0,0 +1,30 @@ +## Why + +PRD-002 SEC-009 requires shell authorization to remain fail-closed, while live approval-fatigue work requires evidence that ordinary diagnostics are not prompted unnecessarily. The post-1952 harvest classifies 21 representative prompts, but those sanitized command shapes are not executed through the coordinator and therefore cannot protect later policy simplification. + +## What Changes + +- Curate parse-preserving, identity-free commands for all 21 post-1952 evidence cases. +- Bind every curated command to its source evidence ID, classification, intended outcome, approval shape, and actor-contact count. +- Execute the cases through the real shell policy coordinator alongside the existing D, A, and L matrices. +- Keep expected approvals and current ShellSyntaxTree fact gaps strict; do not reinterpret executable-private arguments or convert agent-alignment guidance into authority. +- Add mutation checks for source shapes, links, classifications, commands, outcomes, corrections, approval shapes, and actor counts. + +Out of scope: changing production policy, widening a safe catalog, parsing `gh` operations in Netclaw, adding Bash arithmetic grammar, or claiming the sample represents every prompt in the source window. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `tool-approval-gates`: Extend the executable sanitized approval corpus with post-1952 live prompt regressions and exact evidence linkage. + +## Impact + +- **Tests and evidence:** `netclaw-policy-fixtures.json`, its source-generated models, coordinator fixture tests, evidence-contract tests, and the post-1952 harvest linkage. +- **Security:** The change adds no authority. Expected approvals and unresolved general parser facts remain promptable through the real coordinator. +- **Operations:** Future policy and refactor pull requests receive a broader should-prompt/should-allow regression gate derived from live traffic. +- **APIs and persistence:** No public API, actor protocol, approval store, session history, configuration, or dependency change. diff --git a/openspec/changes/extend-live-approval-regression-corpus/specs/tool-approval-gates/spec.md b/openspec/changes/extend-live-approval-regression-corpus/specs/tool-approval-gates/spec.md new file mode 100644 index 000000000..446cef134 --- /dev/null +++ b/openspec/changes/extend-live-approval-regression-corpus/specs/tool-approval-gates/spec.md @@ -0,0 +1,68 @@ +## ADDED Requirements + +### Requirement: Executable post-1952 live approval regression corpus + +The shell-policy evidence catalog SHALL contain one executable live regression +for each representative post-1952 evidence case T01 through T21. Each +regression SHALL identify its source evidence file and source evidence ID. It +SHALL retain the source classification and intended policy outcome. + +Executable commands SHALL be identity-free and SHALL preserve the +policy-relevant shell grammar of the source shape. Display-only redactions that +would become shell operators SHALL NOT be executed as literal fixture input. + +The real shell policy coordinator SHALL evaluate every regression. Each row +SHALL assert the final outcome, deny reason, approval candidates, messy status, +approval option keys, and approval-actor contact count that are applicable to +that outcome. Evidence classifications SHALL NOT grant authority. + +#### Scenario: Every representative post-1952 case executes once + +- **WHEN** the live regression fixture loads +- **THEN** source evidence IDs T01 through T21 each occur exactly once +- **AND** policy case IDs L12 through L32 each occur exactly once +- **AND** every case executes through the real coordinator + +#### Scenario: Source evidence remains exactly linked + +- **WHEN** the evidence contract validates a live regression +- **THEN** its source file and evidence ID resolve to one harvested case +- **AND** its digest includes the harvested command shape +- **AND** its classification equals the harvested classification +- **AND** its target outcome equals its executable policy expectation + +#### Scenario: Display redaction does not change executable grammar + +- **WHEN** a harvested command shape contains a display-only placeholder +- **THEN** the executable fixture uses an identity-free shell literal +- **AND** it preserves the original command chain, path boundary, redirect, + or dynamic construct under test +- **AND** it does not interpret an angle-bracket placeholder as a redirect + +#### Scenario: Current fact gaps remain strict + +- **WHEN** the coordinator evaluates the curated default-GET `gh api` cases + or the static Bash arithmetic echo case +- **THEN** it requires approval under the current parser facts +- **AND** Netclaw does not infer executable-private operation semantics + +#### Scenario: Agent-alignment cases do not gain authority + +- **WHEN** the coordinator evaluates a case classified as + `AgentAlignmentDebt` +- **THEN** the classification does not provide candidate coverage +- **AND** the current call remains approval-gated + +#### Scenario: Executable evidence drift is explicit + +- **WHEN** a source shape, executable command, evidence link, classification, + expected outcome, correction, approval shape, or actor-contact count changes +- **THEN** the locked live-regression digest changes +- **AND** the evidence contract fails until the new artifact is reviewed and + deliberately accepted + +#### Scenario: Corpus contains no source identity + +- **WHEN** the PII contract scans the added executable fixtures +- **THEN** it finds no local username, private repository, channel, thread, + host, email, token, or secret diff --git a/openspec/changes/extend-live-approval-regression-corpus/tasks.md b/openspec/changes/extend-live-approval-regression-corpus/tasks.md new file mode 100644 index 000000000..6b85d09cd --- /dev/null +++ b/openspec/changes/extend-live-approval-regression-corpus/tasks.md @@ -0,0 +1,27 @@ +## 1. Evidence Schema and Linkage + +- [x] 1.1 Add a source evidence file to every live regression fixture. +- [x] 1.2 Resolve each evidence pair and verify its harvested classification. +- [x] 1.3 Digest source shapes, commands, links, outcomes, corrections, options, and actor counts. + +## 2. Post-1952 Executable Corpus + +- [x] 2.1 Curate identity-free, parse-preserving commands for T01-T07 as L12-L18. +- [x] 2.2 Curate identity-free, parse-preserving commands for T08-T14 as L19-L25. +- [x] 2.3 Curate identity-free, parse-preserving commands for T15-T21 as L26-L32. +- [x] 2.4 Record exact coordinator outcomes, corrections, approval shapes, and actor counts for L12-L32. + +## 3. Executable and Adversarial Verification + +- [x] 3.1 Execute L01-L32 through the real coordinator and require declared outcomes. +- [x] 3.2 Keep three ShellSyntaxTree fact-gap rows strict without private parsing. +- [x] 3.3 Keep agent-alignment rows strict without classification-derived authority. +- [x] 3.4 Mutate source shapes, links, classifications, commands, outcomes, corrections, approval shapes, and actor counts. +- [x] 3.5 Prove display-only angle-bracket placeholders do not appear in executable L12-L32 commands. + +## 4. Quality Gates and Delivery + +- [x] 4.1 Run focused Security evidence-contract and Actors coordinator-fixture suites. +- [x] 4.2 Run OpenSpec, JSON, headers, diff, formatting, Slopwatch, and PII gates. +- [x] 4.3 Complete an adversarial review of fixture truthfulness, parser fidelity, and authority neutrality. +- [x] 4.4 Rebase, rerun interaction gates, deliver the pull request, and enable auto-merge. diff --git a/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json b/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json index 78e36d2e4..8ad0291c1 100644 --- a/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json +++ b/openspec/changes/structure-shell-approval-policy/evidence/netclaw-policy-fixtures.json @@ -1,5 +1,5 @@ { - "schemaVersion": 2, + "schemaVersion": 3, "fixtureDefaults": { "toolName": "shell_execute", "audience": "Personal", @@ -412,6 +412,7 @@ ], "liveRegressionCases": [ { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S18", "classification": "NetclawPolicyDebt", "targetOutcome": "Allow", @@ -429,6 +430,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S22", "classification": "NetclawPolicyDebt", "targetOutcome": "Allow", @@ -446,6 +448,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S40", "classification": "NetclawPolicyDebt", "targetOutcome": "Allow", @@ -463,6 +466,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S16", "classification": "AgentAlignmentDebt", "targetOutcome": "RequiresApproval", @@ -480,6 +484,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S11", "classification": "AgentAlignmentDebt", "targetOutcome": "RequiresApproval", @@ -497,6 +502,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S10", "classification": "ExpectedApproval", "targetOutcome": "RequiresApproval", @@ -514,6 +520,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S24", "classification": "ExpectedApproval", "targetOutcome": "RequiresApproval", @@ -531,6 +538,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S20", "classification": "ExpectedApproval", "targetOutcome": "RequiresApproval", @@ -548,6 +556,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S13", "classification": "ExpectedApproval", "targetOutcome": "RequiresApproval", @@ -565,6 +574,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S21", "classification": "ExpectedApproval", "targetOutcome": "RequiresApproval", @@ -582,6 +592,7 @@ } }, { + "sourceEvidenceFile": "post-1925-extended-approval-harvest.json", "sourceEvidenceId": "S44", "classification": "NetclawPolicyDebt", "targetOutcome": "Allow", @@ -597,6 +608,321 @@ "useBundledSafeCatalog": true, "expected": { "outcome": "Allow", "denyReason": null, "approvalCandidates": null, "isMessy": null, "optionKeys": null, "actorCheckCount": 1 } } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T01", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L12", "category": "RemoteCloneIntoSharedTemp", + "command": "mkdir -p /tmp/review-workspace; cd /tmp/review-workspace; git clone --depth 1 https://service.example.invalid/example/project.git", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T02", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L13", "category": "SharedTempLoopInspection", + "command": "cd /tmp/review-workspace; for project in project-a project-b; do ls -la \"$project\" | head; done", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T03", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L14", "category": "RemoteGitFetch", + "command": "cd /work/project; git remote -v; git fetch origin; git fetch upstream", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T04", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L15", "category": "SharedTempFindInspection", + "command": "cd /tmp/review-workspace; find project-a/src -type f -name '*.fs' | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T05", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L16", "category": "ContinueGitRebase", + "command": "cd /work/project-worktree; grep -rn 'ConflictMarker' src; git add -A; git rebase --continue", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T06", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L17", "category": "SharedTempDiffArtifacts", + "command": "cd /work/project-worktree; git diff --name-only origin/dev...HEAD | sort > /tmp/old-files; git diff --name-only origin/dev...feature/example | sort > /tmp/new-files; comm /tmp/old-files /tmp/new-files", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T07", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L18", "category": "InlineProjectListInspection", + "command": "cd /projects/project-worktree; ls src/*Tests*; ls src | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work/current", "projectDirectory": "/work/current", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T08", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L19", "category": "DynamicKnownFileInspection", + "command": "cd /work/project-worktree; file=$(find src -name 'KnownFile.cs' | head -1); grep -n -A 22 'KnownSymbol' \"$file\" | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T09", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L20", "category": "ShellHttpInlineInterpreter", + "command": "curl -s https://api.github.com/repos/example/project | python3 -c 'import json,sys; print(json.load(sys.stdin).get(\"name\"))'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["curl", "python3"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T10", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L21", "category": "InlineProjectHistoryInspection", + "command": "cd /projects/project; git log --oneline -15; git log --all --oneline | grep 'Marker'; grep -ri 'Marker' services", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work/current", "projectDirectory": "/work/current", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T11", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L22", "category": "SessionArtifactInlineInterpreter", + "command": "cd /session; python3 -c 'import json; print(json.load(open(\"response.json\"))[\"status\"])'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T12", + "classification": "ShellSyntaxTreeFactGap", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L23", "category": "DefaultGetGhApiPair", + "command": "gh api repos/example/project/commits --jq '.[].sha'; gh api repos/example/project/issues --jq '.[].number'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["gh api"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T13", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L24", "category": "ExternalCliExternalTempRedirect", + "command": "external-crm deals list --status won --json > /external-temp/records.json 2> /external-temp/records.err; jq 'length' /external-temp/records.json", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["external-crm deals list", "jq"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T14", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L25", "category": "RemoteSshInspection", + "command": "ssh -o ConnectTimeout=6 -o BatchMode=yes service.example.invalid 'uname -a' | head", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["ssh"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T15", + "classification": "ShellSyntaxTreeFactGap", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L26", "category": "StaticArithmeticEcho", + "command": "echo 'DELTA'; echo $((current - baseline)); echo 'REMAINING'; echo $((target - current))", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T16", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L27", "category": "MultiProjectDynamicLoop", + "command": "for repository in /projects/project-a /projects/project-b; do git -C \"$repository\" remote -v; git -C \"$repository\" branch --show-current; done", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work/current", "projectDirectory": "/work/current", "sessionDirectory": "/session", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T17", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L28", "category": "ExternalPackageCacheRead", + "command": "grep -rn 'PackageMarker' /work/project/src; grep -rn 'ApiMarker' /opt/packages; ls /opt/packages | grep 'example'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["grep", "ls"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T18", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L29", "category": "DockerComposeAndGitDiff", + "command": "cd /work/project-worktree; docker compose -f service.yml config -q; git diff --stat; git diff", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T19", + "classification": "AgentAlignmentDebt", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L30", "category": "CompoundKnownFileInspection", + "command": "cd /work/project; sed -n '40,80p' scripts/build.sh; grep -n 'TimeoutMarker' related/scripts; git show HEAD~1:scripts/build.sh | grep 'TimeoutMarker'", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T20", + "classification": "ExpectedApproval", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L31", "category": "RuntimeBuildLoop", + "command": "for run in $(seq 1 8); do dotnet test --filter 'Category=Focused'; if echo \"$output\" | grep -q FAIL; then inspect; break; fi; done", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": [], "isMessy": true, "optionKeys": ["approve_once", "deny"], "actorCheckCount": 0 } + } + }, + { + "sourceEvidenceFile": "post-1952-live-approval-harvest.json", + "sourceEvidenceId": "T21", + "classification": "ShellSyntaxTreeFactGap", + "targetOutcome": "RequiresApproval", + "policyCase": { + "id": "L32", "category": "PaginatedGetGhApi", + "command": "gh api repos/example/project/labels --paginate --jq '.[].name' | sort", + "environment": { "platform": "Linux", "executablePath": "/bin/bash", "commandArguments": ["-c"], "grammar": "Bash", "pathStyle": "Posix", "powerShellDialect": null }, + "initialWorkingDirectory": "/work", "projectDirectory": "/work", "sessionDirectory": "/work", + "available": { "oneTimeApprovalKeys": [], "sessionGrants": [], "persistentGrants": [], "safePhrases": [] }, + "useBundledSafeCatalog": true, + "expected": { "outcome": "RequiresApproval", "denyReason": null, "approvalCandidates": ["gh api", "sort"], "isMessy": false, "optionKeys": ["approve_once", "approve_session", "approve_everywhere", "deny"], "actorCheckCount": 1 } + } } ], "adversarialCases": [ diff --git a/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs b/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs index c65d380a3..bea49ef7c 100644 --- a/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs +++ b/src/Netclaw.Actors.Tests/Tools/ShellPolicyEvidenceFixtureTests.cs @@ -94,8 +94,12 @@ public async Task Adversarial_policy_fixtures_fail_closed_through_the_coordinato } } - [Fact] - public async Task Live_regression_fixtures_pin_current_policy_outcomes() + public static TheoryData LiveRegressionCaseIds => new( + Enumerable.Range(1, 32).Select(number => $"L{number:00}")); + + [Theory] + [MemberData(nameof(LiveRegressionCaseIds))] + public async Task Live_regression_fixtures_pin_current_policy_outcomes(string caseId) { var catalog = JsonSerializer.Deserialize( File.ReadAllBytes(EvidencePath()), @@ -105,10 +109,10 @@ public async Task Live_regression_fixtures_pin_current_policy_outcomes() catalog.FixtureDefaults.ClockUtc, CultureInfo.InvariantCulture)); - foreach (var liveCase in catalog.LiveRegressionCases) - { - await AssertPolicyCaseAsync(catalog, timeProvider, liveCase.PolicyCase); - } + var liveCase = Assert.Single( + catalog.LiveRegressionCases, + item => item.PolicyCase.Id == caseId); + await AssertPolicyCaseAsync(catalog, timeProvider, liveCase.PolicyCase); } private async Task AssertPolicyCaseAsync( @@ -140,6 +144,7 @@ private async Task AssertPolicyCaseAsync( + $"deny={decision.DenyReason}; " + $"candidates={string.Join(", ", decision.ApprovalContext?.CandidateVerbs ?? [])}; " + $"messy={decision.ApprovalContext?.IsMessy}; " + + $"correction={decision.ApprovalContext?.AgentCorrection?.GetType().Name}; " + $"checks={harness.ApprovalService.CheckCount}; " + $"allow={decision.AllowReason}; " + $"matches={string.Join(", ", decision.ApprovalMatches.Select(item => item.Pattern))}; " @@ -147,6 +152,9 @@ private async Task AssertPolicyCaseAsync( Assert.Equal(ParseOutcome(policyCase.Expected.Outcome), decision.Outcome); Assert.Equal(policyCase.Expected.DenyReason, decision.DenyReason); + Assert.Equal( + policyCase.Expected.AgentCorrection, + decision.ApprovalContext?.AgentCorrection?.GetType().Name); Assert.Equal(policyCase.Expected.ApprovalCandidates, decision.ApprovalContext?.CandidateVerbs); Assert.Equal(policyCase.Expected.IsMessy, decision.ApprovalContext?.IsMessy); Assert.Equal( @@ -312,20 +320,28 @@ private static ShellApprovalInvocation CreateInvocation( _ => throw new InvalidDataException($"Unsupported fixture environment: {policyCase.Id}.") }; - var expectedWorkingDirectory = host == ShellApprovalHost.Bash - ? "/work" - : @"C:\work"; + var pathStyle = host == ShellApprovalHost.Bash + ? ShellPathStyle.Posix + : ShellPathStyle.Windows; if (!string.Equals( policyCase.InitialWorkingDirectory, - expectedWorkingDirectory, + policyCase.ProjectDirectory, StringComparison.Ordinal) + || !CanonicalShellPath.TryCreate( + policyCase.ProjectDirectory, + pathStyle, + out var normalizedProjectDirectory) || !string.Equals( + normalizedProjectDirectory.Value, policyCase.ProjectDirectory, - expectedWorkingDirectory, StringComparison.Ordinal) + || !CanonicalShellPath.TryCreate( + policyCase.SessionDirectory, + pathStyle, + out var normalizedSessionDirectory) || !string.Equals( + normalizedSessionDirectory.Value, policyCase.SessionDirectory, - expectedWorkingDirectory, StringComparison.Ordinal)) { throw new InvalidDataException($"Unsupported fixture scope: {policyCase.Id}."); diff --git a/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs b/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs index 8ec2cc779..faa60dc08 100644 --- a/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs +++ b/src/Netclaw.Security.Tests/ShellApprovalEvidenceContractTests.cs @@ -4,6 +4,7 @@ // // ----------------------------------------------------------------------- using System.Security.Cryptography; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; @@ -24,6 +25,8 @@ public sealed partial class ShellApprovalEvidenceContractTests private const string Post1952HarvestFile = "post-1952-live-approval-harvest.json"; private const string ApprovalMatrixSha256 = "0169105efe87b345d9a82d777ef86909e31fa81a5255cc0cc30f32fbe4d0d6b0"; + private const string LiveRegressionCasesSha256 = + "684b89f8e01f6abc8d4b9cff49c1e1ab16d3df9cd6aaf028e2aa0822509c421a"; [Fact] public void Approval_matrix_matches_the_locked_cross_repository_artifact() @@ -55,7 +58,7 @@ public void Policy_fixtures_load_exact_authority_and_trace_fields() var fixtures = DeserializeFixtures(File.ReadAllBytes(EvidencePath(PolicyFixturesFile))); var commands = matrix.Cases.ToDictionary(item => item.Id, item => item.Command); - Assert.Equal(2, fixtures.SchemaVersion); + Assert.Equal(3, fixtures.SchemaVersion); Assert.Equal("shell_execute", fixtures.FixtureDefaults.ToolName); Assert.Equal("Personal", fixtures.FixtureDefaults.Audience); Assert.Equal("Approval", fixtures.FixtureDefaults.ApprovalMode); @@ -67,31 +70,53 @@ public void Policy_fixtures_load_exact_authority_and_trace_fields() Assert.Equal("/work", fixtures.FixtureDefaults.ProjectDirectory); Assert.Null(fixtures.FixtureDefaults.InheritedWorkingDirectory); Assert.Equal(10, fixtures.Cases.Count); - var sourceEvidence = new[] { PostSwapHarvestFile, ExtendedPostSwapHarvestFile } - .SelectMany(file => JsonSerializer.Deserialize( - File.ReadAllBytes(EvidencePath(file)), - ShellApprovalEvidenceJsonContext.Default.PostMergeApprovalHarvest)! - .Cases) - .ToDictionary(item => item.Id); + var sourceEvidence = LoadLiveRegressionEvidence(); Assert.Equal( - Enumerable.Range(1, 11).Select(number => $"L{number:00}"), + Enumerable.Range(1, 32).Select(number => $"L{number:00}"), fixtures.LiveRegressionCases.Select(item => item.PolicyCase.Id)); Assert.Equal( - ["S18", "S22", "S40", "S16", "S11", "S10", "S24", "S20", "S13", "S21", "S44"], + new[] { "S18", "S22", "S40", "S16", "S11", "S10", "S24", "S20", "S13", "S21", "S44" } + .Concat(Enumerable.Range(1, 21).Select(number => $"T{number:00}")), fixtures.LiveRegressionCases.Select(item => item.SourceEvidenceId)); + Assert.Equal( + fixtures.LiveRegressionCases.Count, + fixtures.LiveRegressionCases + .Select(item => (item.SourceEvidenceFile, item.SourceEvidenceId)) + .Distinct() + .Count()); Assert.All(fixtures.LiveRegressionCases, item => { Assert.Contains( item.Classification, - new[] { "ExpectedApproval", "AgentAlignmentDebt", "NetclawPolicyDebt" }); + new[] + { + "ExpectedApproval", + "AgentAlignmentDebt", + "NetclawPolicyDebt", + "ShellSyntaxTreeFactGap" + }); Assert.Contains(item.TargetOutcome, new[] { "Allow", "RequiresApproval" }); Assert.Equal(item.TargetOutcome, item.PolicyCase.Expected.Outcome); - Assert.Equal(sourceEvidence[item.SourceEvidenceId].Classification, item.Classification); + var sourceKey = (item.SourceEvidenceFile, item.SourceEvidenceId); + Assert.True(sourceEvidence.TryGetValue(sourceKey, out var sourceCase)); + Assert.Equal(sourceCase.Classification, item.Classification); }); Assert.Equal(4, fixtures.LiveRegressionCases.Count(item => item.TargetOutcome == "Allow")); Assert.Equal( - 7, + 28, fixtures.LiveRegressionCases.Count(item => item.TargetOutcome == "RequiresApproval")); + var post1952Cases = fixtures.LiveRegressionCases + .Where(item => item.SourceEvidenceFile == Post1952HarvestFile) + .ToList(); + Assert.Equal(21, post1952Cases.Count); + Assert.All(post1952Cases, item => + { + Assert.Equal("RequiresApproval", item.TargetOutcome); + Assert.DoesNotMatch("<[^>]+>", item.PolicyCase.Command); + }); + Assert.Equal(8, post1952Cases.Count(item => item.Classification == "ExpectedApproval")); + Assert.Equal(10, post1952Cases.Count(item => item.Classification == "AgentAlignmentDebt")); + Assert.Equal(3, post1952Cases.Count(item => item.Classification == "ShellSyntaxTreeFactGap")); Assert.Equal( Enumerable.Range(1, 12).Select(number => $"A{number:00}"), fixtures.AdversarialCases.Select(item => item.Id)); @@ -135,13 +160,110 @@ public void Policy_fixtures_load_exact_authority_and_trace_fields() } } + [Fact] + public void Live_regression_evidence_section_matches_locked_digest() + { + var bytes = File.ReadAllBytes(EvidencePath(PolicyFixturesFile)); + + Assert.Equal(LiveRegressionCasesSha256, ComputeLiveRegressionDigest(bytes)); + } + + [Theory] + [InlineData("\"sourceEvidenceId\": \"T01\"", "\"sourceEvidenceId\": \"T99\"")] + [InlineData("\"classification\": \"ExpectedApproval\"", "\"classification\": \"Changed\"")] + [InlineData("mkdir -p /tmp/review-workspace", "mkdir -p /tmp/changed-workspace")] + [InlineData("\"targetOutcome\": \"RequiresApproval\"", "\"targetOutcome\": \"Allow\"")] + [InlineData("\"approvalCandidates\": []", "\"approvalCandidates\": [\"unexpected\"]")] + [InlineData("\"denyReason\": null, \"approvalCandidates\"", "\"denyReason\": null, \"agentCorrection\": \"ChangedCorrection\", \"approvalCandidates\"")] + [InlineData("\"actorCheckCount\": 0", "\"actorCheckCount\": 7")] + public void Live_regression_digest_detects_security_significant_mutation( + string original, + string replacement) + { + var json = File.ReadAllText(EvidencePath(PolicyFixturesFile)); + var mutated = json.Replace(original, replacement, StringComparison.Ordinal); + + Assert.NotEqual(json, mutated); + Assert.NotEqual( + ComputeLiveRegressionDigest(Encoding.UTF8.GetBytes(json)), + ComputeLiveRegressionDigest(Encoding.UTF8.GetBytes(mutated))); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Live_regression_digest_detects_linked_source_mutation(bool mutateCommandShape) + { + var fixtureBytes = File.ReadAllBytes(EvidencePath(PolicyFixturesFile)); + var sourceEvidence = LoadLiveRegressionEvidence().ToDictionary(); + var sourceKey = (File: Post1952HarvestFile, Id: "T01"); + var sourceCase = sourceEvidence[sourceKey]; + sourceEvidence[sourceKey] = mutateCommandShape + ? sourceCase with { CommandShape = "changed source shape" } + : sourceCase with { Classification = "ChangedClassification" }; + + Assert.NotEqual( + ComputeLiveRegressionDigest(fixtureBytes), + ComputeLiveRegressionDigest(fixtureBytes, sourceEvidence)); + } + + private static IReadOnlyDictionary<(string File, string Id), PostMergeApprovalCase> + LoadLiveRegressionEvidence() + { + string[] evidenceFiles = + [ + PostSwapHarvestFile, + ExtendedPostSwapHarvestFile, + Post1952HarvestFile + ]; + + return evidenceFiles + .SelectMany(file => JsonSerializer.Deserialize( + File.ReadAllBytes(EvidencePath(file)), + ShellApprovalEvidenceJsonContext.Default.PostMergeApprovalHarvest)! + .Cases + .Select(item => (Key: (File: file, Id: item.Id), Case: item))) + .ToDictionary(item => item.Key, item => item.Case); + } + + private static string ComputeLiveRegressionDigest(byte[] fixtureBytes) + => ComputeLiveRegressionDigest(fixtureBytes, LoadLiveRegressionEvidence()); + + private static string ComputeLiveRegressionDigest( + byte[] fixtureBytes, + IReadOnlyDictionary<(string File, string Id), PostMergeApprovalCase> sourceEvidence) + { + using var document = JsonDocument.Parse(fixtureBytes); + var liveCasesJson = document.RootElement.GetProperty("liveRegressionCases").GetRawText(); + var fixtures = DeserializeFixtures(fixtureBytes); + var lockedEvidence = new StringBuilder(liveCasesJson); + foreach (var item in fixtures.LiveRegressionCases) + { + var sourceKey = (item.SourceEvidenceFile, item.SourceEvidenceId); + if (!sourceEvidence.TryGetValue(sourceKey, out var sourceCase)) + { + lockedEvidence.Append("\nmissing:").Append(item.SourceEvidenceFile) + .Append(':').Append(item.SourceEvidenceId); + continue; + } + + lockedEvidence.Append('\n').Append(sourceCase.CommandShape.Length).Append(':') + .Append(sourceCase.CommandShape).Append('\n') + .Append(sourceCase.Classification.Length).Append(':') + .Append(sourceCase.Classification); + } + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(lockedEvidence.ToString()))) + .ToLowerInvariant(); + } + [Fact] public void Policy_fixture_schema_rejects_unknown_members() { var json = File.ReadAllText(EvidencePath(PolicyFixturesFile)); var malformed = json.Replace( - "\"schemaVersion\": 2,", - "\"schemaVersion\": 2, \"unexpected\": true,", + "\"schemaVersion\": 3,", + "\"schemaVersion\": 3, \"unexpected\": true,", StringComparison.Ordinal); Assert.Throws(() => JsonSerializer.Deserialize( diff --git a/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs b/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs index 629ecd5c7..8379effd6 100644 --- a/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs +++ b/src/Netclaw.Security.Tests/ShellPolicyEvidenceModels.cs @@ -22,6 +22,8 @@ internal sealed record PolicyFixtureCatalog internal sealed record PolicyLiveRegressionCase { + public required string SourceEvidenceFile { get; init; } + public required string SourceEvidenceId { get; init; } public required string Classification { get; init; } @@ -113,6 +115,8 @@ internal sealed record PolicyAdversarialExpected public string? DenyReason { get; init; } + public string? AgentCorrection { get; init; } + public List? ApprovalCandidates { get; init; } public bool? IsMessy { get; init; }