diff --git a/.github/workflows/e2e-rust-windows.yml b/.github/workflows/e2e-rust-windows.yml index dfa9c445f4..0ab95c5c00 100644 --- a/.github/workflows/e2e-rust-windows.yml +++ b/.github/workflows/e2e-rust-windows.yml @@ -83,6 +83,10 @@ jobs: retention-days: 14 native: + strategy: + fail-fast: false + matrix: + instance: [shim, direct] if: inputs.lane == 'all' || inputs.lane == 'native' name: "Windows / native WPF + WinUI3 + WebView2" needs: source @@ -97,11 +101,37 @@ jobs: - name: Ensure FFmpeg for trajectory video shell: pwsh run: .\scripts\ci\windows\setup-ffmpeg.ps1 - - name: Run native Rust harnesses + - name: Observe public recording tools across short lifetimes shell: pwsh env: - CUA_E2E_INTERNAL_LANE: native - run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui + ENCODER_ROUTE: ${{ matrix.instance }} + run: | + .\scripts\ci\windows\verify-user-session.ps1 + $root = Join-Path $PWD 'artifacts/recorder-observations' + New-Item -ItemType Directory -Force $root | Out-Null + if ($env:ENCODER_ROUTE -eq 'direct') { + $encoder = @(Get-ChildItem C:/ProgramData/chocolatey/lib/ffmpeg -Filter ffmpeg.exe -Recurse) + if ($encoder.Count -ne 1) { throw 'Expected one package-owned FFmpeg executable' } + $env:PATH = "$($encoder[0].DirectoryName);$env:PATH" + } + Get-FileHash (Get-Command ffmpeg.exe).Source | ConvertTo-Json | Set-Content (Join-Path $root 'encoder.json') + $env:CUA_RECORDING_DIAGNOSTIC_ROOT = $root + Push-Location libs/cua-driver/rust + try { + & cargo test --locked -p cua-driver-core --test recording_lifetime_diagnostic -- --ignored --exact real_encoder_short_lifetime_observations --nocapture --test-threads=1 2>&1 | + Tee-Object (Join-Path $root 'observations.log') + if ($LASTEXITCODE -ne 0) { throw 'Public recording lifecycle failure captured' } + } finally { + Pop-Location + } + - name: Preserve every diagnostic observation + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 + with: + name: recorder-observations-${{ matrix.instance }} + path: artifacts/recorder-observations + if-no-files-found: error + retention-days: 14 - name: Collect logs if: always() shell: pwsh @@ -110,7 +140,7 @@ jobs: if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: - name: rust-windows-native + name: rust-windows-native-${{ matrix.instance }} path: artifacts/cua-driver/windows if-no-files-found: ignore compression-level: 0 diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs index 36d108c8c0..d3b9c2c14d 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs @@ -138,10 +138,12 @@ impl FfmpegVideoBackend { impl VideoBackend for FfmpegVideoBackend { fn stop(mut self: Box) -> anyhow::Result { let elapsed = self.started_at.elapsed(); - if let Some(mut stdin) = self.child.stdin.take() { - let _ = stdin.write_all(b"q\n"); - let _ = stdin.flush(); - } + let shutdown_started = Instant::now(); + let stdin_results = self.child.stdin.take().map(|mut stdin| { + let write_result = stdin.write_all(b"q\n"); + let flush_result = stdin.flush(); + (write_result, flush_result) + }); let shutdown_timeout = Duration::from_millis(3000); let deadline = Instant::now() + shutdown_timeout; @@ -155,8 +157,9 @@ impl VideoBackend for FfmpegVideoBackend { break Err(anyhow::anyhow!("ffmpeg exited with {cause}")); } None if Instant::now() > deadline => { - let _ = self.child.kill(); - let _ = self.child.wait(); + let kill_result = self.child.kill(); + let wait_result = self.child.wait(); + tracing::warn!(target: "recording", ?kill_result, ?wait_result, "shutdown diagnostic forced termination"); break Err(anyhow::anyhow!( "ffmpeg shutdown timed out after {} ms", shutdown_timeout.as_millis() @@ -165,6 +168,7 @@ impl VideoBackend for FfmpegVideoBackend { None => std::thread::sleep(Duration::from_millis(80)), } }; + tracing::warn!(target: "recording", elapsed_ms = shutdown_started.elapsed().as_millis() as u64, ?stdin_results, ?result, "shutdown diagnostic result"); if let Some(handle) = self.stderr_thread.take() { let stderr = handle.join().unwrap_or_default(); if let Err(error) = &result { diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_lifetime_diagnostic.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_lifetime_diagnostic.rs new file mode 100644 index 0000000000..5d0d1dbcab --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/recording_lifetime_diagnostic.rs @@ -0,0 +1,62 @@ +use std::{ + path::PathBuf, + sync::Arc, + time::{Duration, Instant}, +}; + +use cua_driver_core::{ + recording::RecordingSession, + recording_tools::{GetRecordingStateTool, StartRecordingTool, StopRecordingTool}, + tool::Tool, + video::set_video_backend_factory, + video_ffmpeg::FfmpegVideoBackendFactory, +}; +use serde_json::json; + +#[tokio::test] +#[ignore] +async fn real_encoder_short_lifetime_observations() { + assert!(cfg!(target_os = "windows")); + let root = PathBuf::from(std::env::var_os("CUA_RECORDING_DIAGNOSTIC_ROOT").unwrap()); + std::fs::create_dir_all(&root).unwrap(); + set_video_backend_factory(Box::new(FfmpegVideoBackendFactory)); + for sequence in 0..500_u64 { + let output = root.join(format!("observation-{sequence:04}")); + std::fs::create_dir_all(&output).unwrap(); + let session = Arc::new(RecordingSession::new()); + let hold_ms = (sequence * 37) % 201; + let started = StartRecordingTool::new(session.clone()) + .invoke(json!({"output_dir": output, "record_video": true})) + .await; + std::fs::write( + output.join("start.json"), + serde_json::to_vec_pretty(&started).unwrap(), + ) + .unwrap(); + assert_eq!( + started.structured_content.as_ref().unwrap()["video_active"], + true, + "{started:?}" + ); + std::thread::sleep(Duration::from_millis(hold_ms)); + let clock = Instant::now(); + let stopped = StopRecordingTool::new(session.clone()) + .invoke(json!({})) + .await; + let elapsed_ms = clock.elapsed().as_millis(); + let state = GetRecordingStateTool::new(session).invoke(json!({})).await; + let stopped = serde_json::to_value(stopped).unwrap(); + let state = state.structured_content.unwrap(); + let row = json!({"sequence": sequence, "hold_ms": hold_ms, "stop_elapsed_ms": elapsed_ms, "stop": stopped, "state": state}); + std::fs::write( + output.join("observation.json"), + serde_json::to_vec_pretty(&row).unwrap(), + ) + .unwrap(); + eprintln!( + "recording observation {sequence} hold_ms={hold_ms} stop_elapsed_ms={elapsed_ms}" + ); + assert_ne!(stopped["isError"], true, "{row}"); + assert!(state["last_video_path"].is_string(), "{row}"); + } +} diff --git a/scripts/ci/windows/diagnose-recorder-pipe.ps1 b/scripts/ci/windows/diagnose-recorder-pipe.ps1 new file mode 100644 index 0000000000..36db89477a --- /dev/null +++ b/scripts/ci/windows/diagnose-recorder-pipe.ps1 @@ -0,0 +1,73 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Add-Type @' +using System; +using System.Runtime.InteropServices; +public static class EncoderPause { + [DllImport("ntdll.dll")] public static extern int NtSuspendProcess(IntPtr handle); + [DllImport("ntdll.dll")] public static extern int NtResumeProcess(IntPtr handle); +} +'@ +$root = Join-Path $PWD 'artifacts/recorder-observations' +New-Item -ItemType Directory -Force $root | Out-Null +$shim = (Get-Command ffmpeg.exe).Source +$direct = @(Get-ChildItem C:/ProgramData/chocolatey/lib/ffmpeg -Filter ffmpeg.exe -Recurse) +if ($direct.Count -ne 1) { throw 'Expected one package-owned FFmpeg executable' } +$results = @() +foreach ($route in @('shim', 'direct')) { + foreach ($close in @($false, $true)) { + foreach ($iteration in 1..5) { + $label = "$route-close-$close-$iteration" + $destination = Join-Path $root $label + New-Item -ItemType Directory -Force $destination | Out-Null + $info = [Diagnostics.ProcessStartInfo]::new() + $info.FileName = if ($route -eq 'shim') { $shim } else { $direct[0].FullName } + $info.UseShellExecute = $false + $info.RedirectStandardInput = $true + $info.RedirectStandardError = $true + $info.RedirectStandardOutput = $true + foreach ($argument in @('-y','-loglevel','error','-f','gdigrab','-framerate','30','-draw_mouse','1','-i','desktop','-vf','pad=ceil(iw/2)*2:ceil(ih/2)*2','-c:v','libx264','-preset','ultrafast','-pix_fmt','yuv420p','-movflags','+faststart','-g','30',(Join-Path $destination 'recording.mp4'))) { + $info.ArgumentList.Add($argument) + } + $process = [Diagnostics.Process]::Start($info) + $stderr = $process.StandardError.ReadToEndAsync() + $stdout = $process.StandardOutput.ReadToEndAsync() + $encoder = $null + $suspended = $false + try { + Start-Sleep -Milliseconds 1800 + if ($process.HasExited) { throw "Encoder exited before observation: $($process.ExitCode)" } + if ($route -eq 'shim') { + $children = @(Get-CimInstance Win32_Process -Filter "ParentProcessId = $($process.Id)" | Where-Object Name -eq 'ffmpeg.exe') + if ($children.Count -ne 1) { throw 'Expected one FFmpeg child of shim' } + $encoder = [Diagnostics.Process]::GetProcessById($children[0].ProcessId) + } else { $encoder = $process } + $suspendStatus = [EncoderPause]::NtSuspendProcess($encoder.Handle) + if ($suspendStatus -ne 0) { throw "NtSuspendProcess failed: $suspendStatus" } + $suspended = $true + $process.StandardInput.Write("q`n") + $process.StandardInput.Flush() + if ($close) { $process.StandardInput.Close() } + Start-Sleep -Milliseconds 100 + $resumeStatus = [EncoderPause]::NtResumeProcess($encoder.Handle) + if ($resumeStatus -ne 0) { throw "NtResumeProcess failed: $resumeStatus" } + $suspended = $false + $watch = [Diagnostics.Stopwatch]::StartNew() + $exited = $process.WaitForExit(3000) + $row = @{ route = $route; close_stdin = $close; iteration = $iteration; exited = $exited; elapsed_ms = $watch.ElapsedMilliseconds; exit_code = $(if ($exited) { $process.ExitCode } else { $null }); encoder_pid = $encoder.Id; wrapper_pid = $process.Id } + if (-not $exited) { $process.Kill($true); $process.WaitForExit() } + $stderr.GetAwaiter().GetResult() | Set-Content (Join-Path $destination 'stderr.txt') + $stdout.GetAwaiter().GetResult() | Set-Content (Join-Path $destination 'stdout.txt') + $row | ConvertTo-Json | Tee-Object -FilePath (Join-Path $destination 'observation.json') + $results += $row + } finally { + if ($suspended) { $null = [EncoderPause]::NtResumeProcess($encoder.Handle) } + if (-not $process.HasExited) { $process.Kill($true); $process.WaitForExit() } + $process.Dispose() + } + } + } +} +$results | ConvertTo-Json | Set-Content (Join-Path $root 'results.json') +Get-FileHash $shim, $direct[0].FullName | ConvertTo-Json | Set-Content (Join-Path $root 'executable-hashes.json') +if (@($results | Where-Object { -not $_.exited -or $_.exit_code -ne 0 }).Count -gt 0) { throw 'Encoder shutdown failure captured; see retained observations' }