test(cua-driver): isolate intermittent recorder shutdown

.github/workflows/e2e-rust-windows.yml

@@ -83,6 +83,10 @@ jobs:
           retention-days: 14
 
   native:
+    strategy:
+      fail-fast: false
+      matrix:
+        instance: [shim, direct]
     if: inputs.lane == 'all' || inputs.lane == 'native'
     name: "Windows / native WPF + WinUI3 + WebView2"
     needs: source
@@ -97,11 +101,37 @@ jobs:
       - name: Ensure FFmpeg for trajectory video
         shell: pwsh
         run: .\scripts\ci\windows\setup-ffmpeg.ps1
-      - name: Run native Rust harnesses
+      - name: Observe public recording tools across short lifetimes
         shell: pwsh
         env:
-          CUA_E2E_INTERNAL_LANE: native
-        run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui
+          ENCODER_ROUTE: ${{ matrix.instance }}
+        run: |
+          .\scripts\ci\windows\verify-user-session.ps1
+          $root = Join-Path $PWD 'artifacts/recorder-observations'
+          New-Item -ItemType Directory -Force $root | Out-Null
+          if ($env:ENCODER_ROUTE -eq 'direct') {
+            $encoder = @(Get-ChildItem C:/ProgramData/chocolatey/lib/ffmpeg -Filter ffmpeg.exe -Recurse)
+            if ($encoder.Count -ne 1) { throw 'Expected one package-owned FFmpeg executable' }
+            $env:PATH = "$($encoder[0].DirectoryName);$env:PATH"
+          }
+          Get-FileHash (Get-Command ffmpeg.exe).Source | ConvertTo-Json | Set-Content (Join-Path $root 'encoder.json')
+          $env:CUA_RECORDING_DIAGNOSTIC_ROOT = $root
+          Push-Location libs/cua-driver/rust
+          try {
+            & cargo test --locked -p cua-driver-core --test recording_lifetime_diagnostic -- --ignored --exact real_encoder_short_lifetime_observations --nocapture --test-threads=1 2>&1 |
+              Tee-Object (Join-Path $root 'observations.log')
+            if ($LASTEXITCODE -ne 0) { throw 'Public recording lifecycle failure captured' }
+          } finally {
+            Pop-Location
+          }
+      - name: Preserve every diagnostic observation
+        if: always()
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08
+        with:
+          name: recorder-observations-${{ matrix.instance }}
+          path: artifacts/recorder-observations
+          if-no-files-found: error
+          retention-days: 14
       - name: Collect logs
         if: always()
         shell: pwsh
@@ -110,7 +140,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
         with:
-          name: rust-windows-native
+          name: rust-windows-native-${{ matrix.instance }}
           path: artifacts/cua-driver/windows
           if-no-files-found: ignore
           compression-level: 0

libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs

@@ -138,10 +138,12 @@ impl FfmpegVideoBackend {
 impl VideoBackend for FfmpegVideoBackend {
     fn stop(mut self: Box<Self>) -> anyhow::Result<VideoMetadata> {
         let elapsed = self.started_at.elapsed();
-        if let Some(mut stdin) = self.child.stdin.take() {
-            let _ = stdin.write_all(b"q\n");
-            let _ = stdin.flush();
-        }
+        let shutdown_started = Instant::now();
+        let stdin_results = self.child.stdin.take().map(|mut stdin| {
+            let write_result = stdin.write_all(b"q\n");
+            let flush_result = stdin.flush();
+            (write_result, flush_result)
+        });
 
         let shutdown_timeout = Duration::from_millis(3000);
         let deadline = Instant::now() + shutdown_timeout;
@@ -155,8 +157,9 @@ impl VideoBackend for FfmpegVideoBackend {
                     break Err(anyhow::anyhow!("ffmpeg exited with {cause}"));
                 }
                 None if Instant::now() > deadline => {
-                    let _ = self.child.kill();
-                    let _ = self.child.wait();
+                    let kill_result = self.child.kill();
+                    let wait_result = self.child.wait();
+                    tracing::warn!(target: "recording", ?kill_result, ?wait_result, "shutdown diagnostic forced termination");
                     break Err(anyhow::anyhow!(
                         "ffmpeg shutdown timed out after {} ms",
                         shutdown_timeout.as_millis()
@@ -165,6 +168,7 @@ impl VideoBackend for FfmpegVideoBackend {
                 None => std::thread::sleep(Duration::from_millis(80)),
             }
         };
+        tracing::warn!(target: "recording", elapsed_ms = shutdown_started.elapsed().as_millis() as u64, ?stdin_results, ?result, "shutdown diagnostic result");
         if let Some(handle) = self.stderr_thread.take() {
             let stderr = handle.join().unwrap_or_default();
             if let Err(error) = &result {

libs/cua-driver/rust/crates/cua-driver-core/tests/recording_lifetime_diagnostic.rs

@@ -0,0 +1,62 @@
+use std::{
+    path::PathBuf,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use cua_driver_core::{
+    recording::RecordingSession,
+    recording_tools::{GetRecordingStateTool, StartRecordingTool, StopRecordingTool},
+    tool::Tool,
+    video::set_video_backend_factory,
+    video_ffmpeg::FfmpegVideoBackendFactory,
+};
+use serde_json::json;
+
+#[tokio::test]
+#[ignore]
+async fn real_encoder_short_lifetime_observations() {
+    assert!(cfg!(target_os = "windows"));
+    let root = PathBuf::from(std::env::var_os("CUA_RECORDING_DIAGNOSTIC_ROOT").unwrap());
+    std::fs::create_dir_all(&root).unwrap();
+    set_video_backend_factory(Box::new(FfmpegVideoBackendFactory));
+    for sequence in 0..500_u64 {
+        let output = root.join(format!("observation-{sequence:04}"));
+        std::fs::create_dir_all(&output).unwrap();
+        let session = Arc::new(RecordingSession::new());
+        let hold_ms = (sequence * 37) % 201;
+        let started = StartRecordingTool::new(session.clone())
+            .invoke(json!({"output_dir": output, "record_video": true}))
+            .await;
+        std::fs::write(
+            output.join("start.json"),
+            serde_json::to_vec_pretty(&started).unwrap(),
+        )
+        .unwrap();
+        assert_eq!(
+            started.structured_content.as_ref().unwrap()["video_active"],
+            true,
+            "{started:?}"
+        );
+        std::thread::sleep(Duration::from_millis(hold_ms));
+        let clock = Instant::now();
+        let stopped = StopRecordingTool::new(session.clone())
+            .invoke(json!({}))
+            .await;
+        let elapsed_ms = clock.elapsed().as_millis();
+        let state = GetRecordingStateTool::new(session).invoke(json!({})).await;
+        let stopped = serde_json::to_value(stopped).unwrap();
+        let state = state.structured_content.unwrap();
+        let row = json!({"sequence": sequence, "hold_ms": hold_ms, "stop_elapsed_ms": elapsed_ms, "stop": stopped, "state": state});
+        std::fs::write(
+            output.join("observation.json"),
+            serde_json::to_vec_pretty(&row).unwrap(),
+        )
+        .unwrap();
+        eprintln!(
+            "recording observation {sequence} hold_ms={hold_ms} stop_elapsed_ms={elapsed_ms}"
+        );
+        assert_ne!(stopped["isError"], true, "{row}");
+        assert!(state["last_video_path"].is_string(), "{row}");
+    }
+}

scripts/ci/windows/diagnose-recorder-pipe.ps1

@@ -0,0 +1,73 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+Add-Type @'
+using System;
+using System.Runtime.InteropServices;
+public static class EncoderPause {
+    [DllImport("ntdll.dll")] public static extern int NtSuspendProcess(IntPtr handle);
+    [DllImport("ntdll.dll")] public static extern int NtResumeProcess(IntPtr handle);
+}
+'@
+$root = Join-Path $PWD 'artifacts/recorder-observations'
+New-Item -ItemType Directory -Force $root | Out-Null
+$shim = (Get-Command ffmpeg.exe).Source
+$direct = @(Get-ChildItem C:/ProgramData/chocolatey/lib/ffmpeg -Filter ffmpeg.exe -Recurse)
+if ($direct.Count -ne 1) { throw 'Expected one package-owned FFmpeg executable' }
+$results = @()
+foreach ($route in @('shim', 'direct')) {
+    foreach ($close in @($false, $true)) {
+        foreach ($iteration in 1..5) {
+            $label = "$route-close-$close-$iteration"
+            $destination = Join-Path $root $label
+            New-Item -ItemType Directory -Force $destination | Out-Null
+            $info = [Diagnostics.ProcessStartInfo]::new()
+            $info.FileName = if ($route -eq 'shim') { $shim } else { $direct[0].FullName }
+            $info.UseShellExecute = $false
+            $info.RedirectStandardInput = $true
+            $info.RedirectStandardError = $true
+            $info.RedirectStandardOutput = $true
+            foreach ($argument in @('-y','-loglevel','error','-f','gdigrab','-framerate','30','-draw_mouse','1','-i','desktop','-vf','pad=ceil(iw/2)*2:ceil(ih/2)*2','-c:v','libx264','-preset','ultrafast','-pix_fmt','yuv420p','-movflags','+faststart','-g','30',(Join-Path $destination 'recording.mp4'))) {
+                $info.ArgumentList.Add($argument)
+            }
+            $process = [Diagnostics.Process]::Start($info)
+            $stderr = $process.StandardError.ReadToEndAsync()
+            $stdout = $process.StandardOutput.ReadToEndAsync()
+            $encoder = $null
+            $suspended = $false
+            try {
+                Start-Sleep -Milliseconds 1800
+                if ($process.HasExited) { throw "Encoder exited before observation: $($process.ExitCode)" }
+                if ($route -eq 'shim') {
+                    $children = @(Get-CimInstance Win32_Process -Filter "ParentProcessId = $($process.Id)" | Where-Object Name -eq 'ffmpeg.exe')
+                    if ($children.Count -ne 1) { throw 'Expected one FFmpeg child of shim' }
+                    $encoder = [Diagnostics.Process]::GetProcessById($children[0].ProcessId)
+                } else { $encoder = $process }
+                $suspendStatus = [EncoderPause]::NtSuspendProcess($encoder.Handle)
+                if ($suspendStatus -ne 0) { throw "NtSuspendProcess failed: $suspendStatus" }
+                $suspended = $true
+                $process.StandardInput.Write("q`n")
+                $process.StandardInput.Flush()
+                if ($close) { $process.StandardInput.Close() }
+                Start-Sleep -Milliseconds 100
+                $resumeStatus = [EncoderPause]::NtResumeProcess($encoder.Handle)
+                if ($resumeStatus -ne 0) { throw "NtResumeProcess failed: $resumeStatus" }
+                $suspended = $false
+                $watch = [Diagnostics.Stopwatch]::StartNew()
+                $exited = $process.WaitForExit(3000)
+                $row = @{ route = $route; close_stdin = $close; iteration = $iteration; exited = $exited; elapsed_ms = $watch.ElapsedMilliseconds; exit_code = $(if ($exited) { $process.ExitCode } else { $null }); encoder_pid = $encoder.Id; wrapper_pid = $process.Id }
+                if (-not $exited) { $process.Kill($true); $process.WaitForExit() }
+                $stderr.GetAwaiter().GetResult() | Set-Content (Join-Path $destination 'stderr.txt')
+                $stdout.GetAwaiter().GetResult() | Set-Content (Join-Path $destination 'stdout.txt')
+                $row | ConvertTo-Json | Tee-Object -FilePath (Join-Path $destination 'observation.json')
+                $results += $row
+            } finally {
+                if ($suspended) { $null = [EncoderPause]::NtResumeProcess($encoder.Handle) }
+                if (-not $process.HasExited) { $process.Kill($true); $process.WaitForExit() }
+                $process.Dispose()
+            }
+        }
+    }
+}
+$results | ConvertTo-Json | Set-Content (Join-Path $root 'results.json')
+Get-FileHash $shim, $direct[0].FullName | ConvertTo-Json | Set-Content (Join-Path $root 'executable-hashes.json')
+if (@($results | Where-Object { -not $_.exited -or $_.exit_code -ne 0 }).Count -gt 0) { throw 'Encoder shutdown failure captured; see retained observations' }