Skip to content
Merged
80 changes: 78 additions & 2 deletions .github/scripts/CiScanMutation.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,14 @@ BeforeAll {
Find = ' if (Test-MarkerLikeContent -Value $rawBody) {'
Replace = ' if ($false) {'
}
# The hidden/control-content rejection is a distinct trusted-boundary
# layer from the marker check. Disabling it lets a body carrying an HTML
# comment (which the canonical marker also is) or invisible content flow
# to the deeper post-injection backstop instead of stopping at the edge.
'no-hidden-content-rejection' = @{
Find = ' if ($hiddenReason) {'
Replace = ' if ($false) {'
}
# Marker-like match patterns can replay trusted publisher state.
'no-marker-pattern-rejection' = @{
Find = ' if (Test-MarkerLikeContent -Value $matchPattern) {'
Expand Down Expand Up @@ -120,6 +128,15 @@ BeforeAll {
return $Path
}

function Test-FixedManifestHandoff {
param([Parameter(Mandatory = $true)][string]$Source)

return $Source -match 'CI_SCAN_MANIFEST_PATH: \$\{\{ runner\.temp \}\}/gh-aw/safe-jobs/agent/manifest_final\.json' -and
$Source -match 'argument-free `submit_ci_scan`' -and
$Source -notmatch '(?ms)^\s{6}inputs:\s*\r?\n\s{8}(?:manifest|manifest_path):' -and
$Source -notmatch 'one `manifest` argument'
}

function New-ProbeManifest {
param(
[string]$Path,
Expand Down Expand Up @@ -306,20 +323,40 @@ Describe 'CI scanner marker mutation coverage' {

It 'mutation "no-duplicate-rejection": a pre-marked body is rejected downstream' {
$body = "$script:CanonicalMarker`n## Summary`nRecurring sample failure.`n`n## Build Information`n- **Pipeline**: maui-pr`n- **Build ID**: 123456`n`n## Error Message`nAssertion failed"
$result = Invoke-ValidatorProbe -Mutation @('no-duplicate-rejection') -Body $body
# Both edge-layer rejections (marker-like content and hidden/HTML-comment
# content) are disabled so this proves the *post-injection* backstop is
# independently load-bearing against duplicate markers.
$result = Invoke-ValidatorProbe -Mutation @('no-duplicate-rejection', 'no-hidden-content-rejection') -Body $body

$result.ok | Should -BeFalse
$result.error | Should -BeLike '*exactly one canonical fingerprint marker*'
}

It 'mutation "no-duplicate-rejection + no-post-injection-check": duplicate markers would ship' {
$body = "$script:CanonicalMarker`n## Summary`nRecurring sample failure.`n`n## Build Information`n- **Pipeline**: maui-pr`n- **Build ID**: 123456`n`n## Error Message`nAssertion failed"
$result = Invoke-ValidatorProbe -Mutation @('no-duplicate-rejection', 'no-post-injection-check') -Body $body
$result = Invoke-ValidatorProbe -Mutation @('no-duplicate-rejection', 'no-hidden-content-rejection', 'no-post-injection-check') -Body $body

$result.ok | Should -BeTrue
([regex]::Matches($result.body, '<!-- ci-scan-fingerprint:')).Count | Should -Be 2
}

It 'baseline: the real validator rejects a body carrying hidden control content' {
$body = "## Summary`nRecurring sample failure.`n`n## Build Information`n- **Pipeline**: maui-pr`n- **Build ID**: 123456`n`n## Error Message`nAssertion failed$([char]0x1B)[31m"
$result = Invoke-ValidatorProbe -Body $body

$result.ok | Should -BeFalse
$result.error | Should -BeLike '*must not contain*C0 control character*'
}

It 'mutation "no-hidden-content-rejection": a hidden-content body slips past the boundary' {
# A benign HTML comment carries no marker tokens, so only the new
# hidden/control-content layer stands between it and publication.
$body = "## Summary`nRecurring sample failure.`n`n## Build Information`n- **Pipeline**: maui-pr`n- **Build ID**: 123456`n`n## Error Message`nAssertion failed`n<!-- reviewer will not see this -->"
$result = Invoke-ValidatorProbe -Mutation @('no-hidden-content-rejection') -Body $body

$result.error | Should -Not -BeLike '*HTML comment sequence*'
}

It 'baseline: the real validator rejects that same pre-marked body outright' {
$body = "$script:CanonicalMarker`n## Summary`nRecurring sample failure.`n`n## Build Information`n- **Pipeline**: maui-pr`n- **Build ID**: 123456`n`n## Error Message`nAssertion failed"
$result = Invoke-ValidatorProbe -Body $body
Expand Down Expand Up @@ -403,6 +440,45 @@ Describe 'CI scanner twin discovery mutation coverage' {
@(Get-CiScanTwin).Count | Should -Be 2
}

Describe 'CI scanner fixed manifest handoff mutation coverage' {
BeforeAll {
$script:WorkflowSources = @(
Get-Content -LiteralPath (Join-Path $PSScriptRoot '../workflows/ci-status-main.md') -Raw
Get-Content -LiteralPath (Join-Path $PSScriptRoot '../workflows/ci-status-net11.md') -Raw
)
$script:SafeJobStepsNeedle = " steps:`n - name: Require successful agent submission gate"
}

It 'baseline: both twins use the fixed argument-free artifact handoff' {
@($script:WorkflowSources | Where-Object { Test-FixedManifestHandoff -Source $_ }).Count |
Should -Be 2
}

It 'mutation "nested-string-transport": a manifest tool input fails the handoff invariant' {
foreach ($source in $script:WorkflowSources) {
$source.Contains($script:SafeJobStepsNeedle) | Should -BeTrue
$mutated = $source.Replace(
$script:SafeJobStepsNeedle,
" inputs:`n manifest:`n required: true`n type: string`n$($script:SafeJobStepsNeedle)")

$mutated | Should -Not -BeExactly $source
(Test-FixedManifestHandoff -Source $mutated) | Should -BeFalse
}
}

It 'mutation "agent-selected-path": a manifest_path tool input fails the handoff invariant' {
foreach ($source in $script:WorkflowSources) {
$source.Contains($script:SafeJobStepsNeedle) | Should -BeTrue
$mutated = $source.Replace(
$script:SafeJobStepsNeedle,
" inputs:`n manifest_path:`n required: true`n type: string`n$($script:SafeJobStepsNeedle)")

$mutated | Should -Not -BeExactly $source
(Test-FixedManifestHandoff -Source $mutated) | Should -BeFalse
}
}
}

It 'mutation "one-twin-omitted": discovery reports a single twin' {
# Proves the anti-vacuity assertion in Validate-CiScanPublisher.Tests.ps1
# is load-bearing: dropping a twin changes what discovery returns, so the
Expand Down
Loading
Loading