fix(shell): harden PowerShell host probe against cold-start timeouts

.github/workflows/pr_validation.yml

@@ -68,6 +68,19 @@ jobs:
         shell: bash
         run: npx -y @modelcontextprotocol/server-everything --version || true
 
+      # Pre-warm the PowerShell host probe so cold-start (Defender scan, first-run
+      # init) doesn't push the in-test 5s probe timeout on loaded Windows runners.
+      # Warm both pwsh.exe and the powershell.exe fallback — the resolver probes
+      # the fallback when pwsh is missing or its probe fails. Use -Command "exit 0"
+      # (no interpolated variables) so the outer pwsh shell can't mangle the args.
+      - name: "Warm PowerShell host probe"
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          pwsh -NoLogo -NoProfile -NonInteractive -Command "exit 0"
+          & "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" -NoLogo -NoProfile -NonInteractive -Command "exit 0"
+          exit 0
+
       # .NET Framework tests can't run reliably on Linux, so we only do .NET 8
 
       - name: "dotnet test"

src/Netclaw.Daemon.Tests/PowerShellHostProbeTests.cs

@@ -146,22 +146,29 @@ public async Task Oversized_output_fails_closed_after_bounded_capture()
     public async Task Timeout_terminates_process_tree_without_waiting_real_time()
     {
         var timeProvider = new FakeTimeProvider();
-        var process = new ControlledProbeProcess(
-            "7.6.4",
-            string.Empty,
-            waitForKill: true);
-        var probe = CreateProbe(process, timeProvider);
+        var first = new ControlledProbeProcess("7.6.4", string.Empty, waitForKill: true);
+        var second = new ControlledProbeProcess("7.6.4", string.Empty, waitForKill: true);
+        var probe = new PowerShellHostProbe(
+            timeProvider,
+            new FixedExecutableLocator(),
+            new SequenceProcessFactory(first, second));
 
         var pending = probe.ProbeAsync("pwsh.exe", TestContext.Current.CancellationToken);
-        await process.WaitStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+        await first.WaitStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+        timeProvider.Advance(PowerShellHostProbe.ProbeTimeout);
+
+        // ProbeAsync retries once on timeout, so drive the second attempt too.
+        await DrainRetryDelayAsync(second, timeProvider);
         timeProvider.Advance(PowerShellHostProbe.ProbeTimeout);
         var result = await pending;
 
         var failed = Assert.IsType<PowerShellHostProbeResult.Failed>(result);
         Assert.Equal(PowerShellProbeFailure.Timeout, failed.Failure);
-        Assert.True(process.KillTreeCalled);
-        Assert.True(process.WaitedAfterKill);
-        Assert.True(process.Disposed);
+        Assert.True(first.KillTreeCalled);
+        Assert.True(first.WaitedAfterKill);
+        Assert.True(first.Disposed);
+        Assert.True(second.KillTreeCalled);
+        Assert.True(second.Disposed);
     }
 
     [Fact]
@@ -215,6 +222,68 @@ public async Task Termination_wait_is_bounded_by_fake_time()
         Assert.True(process.Disposed);
     }
 
+    [Fact]
+    public async Task Timeout_retries_once_and_recovers_when_second_attempt_succeeds()
+    {
+        var timeProvider = new FakeTimeProvider();
+        var slow = new ControlledProbeProcess("7.6.4", string.Empty, waitForKill: true);
+        var healthy = new ControlledProbeProcess("7.6.4", string.Empty);
+        var probe = new PowerShellHostProbe(
+            timeProvider,
+            new FixedExecutableLocator(),
+            new SequenceProcessFactory(slow, healthy));
+
+        var pending = probe.ProbeAsync("pwsh.exe", TestContext.Current.CancellationToken);
+        await slow.WaitStarted.Task.WaitAsync(TestContext.Current.CancellationToken);
+        timeProvider.Advance(PowerShellHostProbe.ProbeTimeout);
+
+        // Drain the retry delay in small steps so the loop can arm and fire it
+        // regardless of thread-pool scheduling between attempts.
+        await DrainRetryDelayAsync(healthy, timeProvider);
+
+        var result = await pending;
+
+        var found = Assert.IsType<PowerShellHostProbeResult.Found>(result);
+        Assert.Equal(ExecutablePath, found.ExecutablePath);
+        Assert.Equal(new Version(7, 6, 4), found.Version);
+        Assert.True(slow.KillTreeCalled);
+        Assert.True(healthy.Disposed);
+    }
+
+    [Fact]
+    public async Task Non_timeout_failure_does_not_retry()
+    {
+        var factory = new TrackingProcessFactory(
+            new ControlledProbeProcess("not-a-version", string.Empty));
+        var probe = new PowerShellHostProbe(
+            TimeProvider.System,
+            new FixedExecutableLocator(),
+            factory);
+
+        var result = await probe.ProbeAsync(
+            "pwsh.exe",
+            TestContext.Current.CancellationToken);
+
+        var failed = Assert.IsType<PowerShellHostProbeResult.Failed>(result);
+        Assert.Equal(PowerShellProbeFailure.MalformedVersion, failed.Failure);
+        Assert.Equal(1, factory.StartCount);
+    }
+
+    private static async Task DrainRetryDelayAsync(
+        ControlledProbeProcess nextAttempt,
+        FakeTimeProvider timeProvider)
+    {
+        for (var i = 0; i < 50 && !nextAttempt.WaitStarted.Task.IsCompleted; i++)
+        {
+            timeProvider.Advance(TimeSpan.FromMilliseconds(50));
+            await Task.Delay(10, TestContext.Current.CancellationToken);
+        }
+
+        Assert.True(
+            nextAttempt.WaitStarted.Task.IsCompleted,
+            "retry attempt never started within the drain budget");
+    }
+
     private static PowerShellHostProbe CreateProbe(
         IPowerShellProbeProcess process,
         TimeProvider? timeProvider = null) =>
@@ -268,6 +337,31 @@ public IPowerShellProbeProcess Start(string executablePath)
         }
     }
 
+    private sealed class SequenceProcessFactory(params IPowerShellProbeProcess[] processes)
+        : IPowerShellProbeProcessFactory
+    {
+        private readonly Queue<IPowerShellProbeProcess> _processes = new(processes);
+
+        public IPowerShellProbeProcess Start(string executablePath)
+        {
+            Assert.Equal(ExecutablePath, executablePath);
+            return _processes.Dequeue();
+        }
+    }
+
+    private sealed class TrackingProcessFactory(IPowerShellProbeProcess process)
+        : IPowerShellProbeProcessFactory
+    {
+        public int StartCount { get; private set; }
+
+        public IPowerShellProbeProcess Start(string executablePath)
+        {
+            StartCount++;
+            Assert.Equal(ExecutablePath, executablePath);
+            return process;
+        }
+    }
+
     private sealed class ControlledProbeProcess : IPowerShellProbeProcess
     {
         private readonly TaskCompletionSource _exit =

src/Netclaw.Daemon.Tests/ShellExecutionEnvironmentResolverTests.cs

@@ -99,19 +99,42 @@ public async Task Unsupported_preferred_version_selects_Windows_PowerShell51(
     [InlineData((int)PowerShellProbeFailure.StartFailed)]
     [InlineData((int)PowerShellProbeFailure.Timeout)]
     [InlineData((int)PowerShellProbeFailure.MalformedVersion)]
-    public async Task Preferred_operational_failure_stops_without_fallback(
+    public async Task Preferred_probe_failure_selects_Windows_PowerShell51(
         int failureValue)
     {
         var failure = (PowerShellProbeFailure)failureValue;
         var probe = new SequencePowerShellProbe(
-            ("pwsh.exe", new PowerShellHostProbeResult.Failed(failure)));
+            ("pwsh.exe", new PowerShellHostProbeResult.Failed(failure)),
+            ("powershell.exe", Found(
+                "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
+                "5.1")));
+        var resolver = new ShellExecutionEnvironmentResolver(probe);
+
+        var resolution = await resolver.ResolveAsync(
+            ShellPlatform.Windows,
+            TestContext.Current.CancellationToken);
+
+        Assert.Equal(PwshDialect.WindowsPowerShell51, resolution.Environment.PowerShellDialect);
+        Assert.Equal(PowerShellFallbackReason.PreferredHostProbeFailed, resolution.FallbackReason);
+        Assert.Null(resolution.RejectedPreferredVersion);
+        Assert.Equal(["pwsh.exe", "powershell.exe"], probe.Calls);
+    }
+
+    [Fact]
+    public async Task Preferred_and_fallback_probe_failure_stops_startup()
+    {
+        var probe = new SequencePowerShellProbe(
+            ("pwsh.exe", new PowerShellHostProbeResult.Failed(PowerShellProbeFailure.Timeout)),
+            ("powershell.exe", new PowerShellHostProbeResult.Failed(
+                PowerShellProbeFailure.AccessDenied)));
         var resolver = new ShellExecutionEnvironmentResolver(probe);
 
         var exception = await Assert.ThrowsAsync<InvalidOperationException>(() =>
             resolver.ResolveAsync(ShellPlatform.Windows, TestContext.Current.CancellationToken));
 
-        Assert.Contains(failure.ToString(), exception.Message);
-        Assert.Equal(["pwsh.exe"], probe.Calls);
+        Assert.Contains(nameof(PowerShellProbeFailure.Timeout), exception.Message);
+        Assert.Contains(nameof(PowerShellProbeFailure.AccessDenied), exception.Message);
+        Assert.Equal(["pwsh.exe", "powershell.exe"], probe.Calls);
     }
 
     [Fact]

src/Netclaw.Daemon/PowerShellHostProbe.cs

@@ -49,12 +49,33 @@ internal sealed class PowerShellHostProbe(
     IPowerShellProbeProcessFactory processFactory) : IPowerShellHostProbe
 {
     internal static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(5);
+    internal static readonly TimeSpan ProbeRetryDelay = TimeSpan.FromMilliseconds(250);
     internal static readonly TimeSpan TerminationTimeout = TimeSpan.FromSeconds(1);
     private const int MaxOutputChars = 4096;
+    private const int MaxProbeAttempts = 2;
 
     public async Task<PowerShellHostProbeResult> ProbeAsync(
         string executableName,
         CancellationToken cancellationToken)
+    {
+        for (var attempt = 1; ; attempt++)
+        {
+            var result = await ProbeOnceAsync(executableName, cancellationToken).ConfigureAwait(false);
+            if (result is not PowerShellHostProbeResult.Failed { Failure: PowerShellProbeFailure.Timeout }
+                || attempt >= MaxProbeAttempts)
+            {
+                return result;
+            }
+
+            // Cold-start (Defender scan, first-run init) is transient: retry once
+            // before surfacing a Timeout, so a slow-but-healthy host still resolves.
+            await Task.Delay(ProbeRetryDelay, timeProvider, cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    private async Task<PowerShellHostProbeResult> ProbeOnceAsync(
+        string executableName,
+        CancellationToken cancellationToken)
     {
         var lookup = executableLocator.Locate(executableName);
         if (lookup is PowerShellExecutableLookup.NotFound)

src/Netclaw.Daemon/ShellExecutionEnvironmentResolver.cs

@@ -11,7 +11,8 @@ namespace Netclaw.Daemon;
 internal enum PowerShellFallbackReason
 {
     PreferredHostNotFound,
-    PreferredVersionUnsupported
+    PreferredVersionUnsupported,
+    PreferredHostProbeFailed
 }
 
 internal sealed record ShellEnvironmentResolution(
@@ -63,15 +64,14 @@ public async Task<ShellEnvironmentResolution> ResolveAsync(
                     PwshDialect.PowerShell7));
         }
 
-        if (preferred is PowerShellHostProbeResult.Failed preferredFailure)
-            throw OperationalProbeFailure("pwsh.exe", preferredFailure);
-
         var (fallbackReason, rejectedVersion) = preferred switch
         {
             PowerShellHostProbeResult.NotFound =>
                 (PowerShellFallbackReason.PreferredHostNotFound, (Version?)null),
             PowerShellHostProbeResult.Found found =>
                 (PowerShellFallbackReason.PreferredVersionUnsupported, found.Version),
+            PowerShellHostProbeResult.Failed =>
+                (PowerShellFallbackReason.PreferredHostProbeFailed, (Version?)null),
             _ => throw new InvalidOperationException("The preferred PowerShell probe returned an unknown result.")
         };
 
@@ -88,21 +88,22 @@ public async Task<ShellEnvironmentResolution> ResolveAsync(
                 rejectedVersion);
         }
 
-        if (fallback is PowerShellHostProbeResult.Failed fallbackFailure)
-            throw OperationalProbeFailure("powershell.exe", fallbackFailure);
-
         var preferredDescription = preferred switch
         {
             PowerShellHostProbeResult.NotFound => "pwsh.exe was not found",
             PowerShellHostProbeResult.Found found =>
                 $"pwsh.exe reported unsupported version {found.Version}",
+            PowerShellHostProbeResult.Failed failed =>
+                $"pwsh.exe probe failed ({failed.Failure})",
             _ => "pwsh.exe was unavailable"
         };
         var fallbackDescription = fallback switch
         {
             PowerShellHostProbeResult.NotFound => "powershell.exe was not found",
             PowerShellHostProbeResult.Found found =>
                 $"powershell.exe reported unsupported version {found.Version}",
+            PowerShellHostProbeResult.Failed failed =>
+                $"powershell.exe probe failed ({failed.Failure})",
             _ => "powershell.exe was unavailable"
         };
 
@@ -116,11 +117,4 @@ private static bool IsSupportedPowerShell7(Version version) =>
 
     private static bool IsWindowsPowerShell51(Version version) =>
         version.Major == 5 && version.Minor == 1;
-
-    private static InvalidOperationException OperationalProbeFailure(
-        string executableName,
-        PowerShellHostProbeResult.Failed failure) =>
-        new(
-            $"Netclaw could not safely probe {executableName} ({failure.Failure}). "
-            + "Startup stopped so an operational probe error cannot change the selected shell grammar.");
 }