diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a0bd4c98b..c76306fbd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -92,7 +92,7 @@ jobs:
       - name: Run unit tests
         run: cargo test --all
 
-  test-benchmarks:
+  test-benchmarks-linux:
     strategy:
       matrix:
         # We split `bench_local` testing into four jobs that run in parallel,
@@ -114,7 +114,7 @@ jobs:
           "Check,Debug,Doc",
           "Opt",
         ]
-    name: Test benchmarks
+    name: Test benchmarks on Linux
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout the source code
@@ -152,6 +152,34 @@ jobs:
           PROFILES: ${{ matrix.PROFILES }}
           SHELL: "/bin/bash"
 
+  # Just a smoke test to ensure that we can fully run a single benchmark
+  test-benchmarks-windows:
+    name: Test benchmarks on Windows
+    runs-on: windows-latest
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Install latest beta toolchain
+        run: |
+          rustup install $RUST_TOOLCHAIN_VERSION
+          rustup default $RUST_TOOLCHAIN_VERSION
+          rustup component add --toolchain $RUST_TOOLCHAIN_VERSION rustfmt clippy
+        shell: bash
+        env:
+          RUST_TOOLCHAIN_VERSION: beta
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Build collector
+        run: cargo build -p collector
+
+      - name: Check eprintln
+        run: cargo run --bin collector profile_local eprintln --exact-match helloworld --profiles All `rustup which rustc`
+        shell: bash
+
   test-backends:
     name: Test codegen backend benchmarks
     runs-on: ubuntu-22.04
@@ -418,7 +446,8 @@ jobs:
     needs:
       - test-linux
       - test-windows
-      - test-benchmarks
+      - test-benchmarks-linux
+      - test-benchmarks-windows
       - test-backends
       - test-runtime-benchmarks
       - test-profiling
diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs
index f54ae246b..8bbab311e 100644
--- a/collector/src/compile/execute/mod.rs
+++ b/collector/src/compile/execute/mod.rs
@@ -370,11 +370,19 @@ impl<'a> CargoProcess<'a> {
 
         loop {
             // Make sure that Cargo.lock isn't changed by the build
-            let _guard = EnsureImmutableFile::new(
-                &self.cwd.join("Cargo.lock"),
-                self.processor_name.0.clone(),
-            )
-            .context("cannot resolve Cargo.lock")?;
+            let _guard = if cfg!(target_os = "linux") {
+                Some(
+                    EnsureImmutableFile::new(
+                        &self.cwd.join("Cargo.lock"),
+                        self.processor_name.0.clone(),
+                    )
+                    .context("cannot resolve Cargo.lock")?,
+                )
+            } else {
+                // We develop on Linux; if we try to build on Windows, the lockfiles would be
+                // re-generated, so we just avoid checking
+                None
+            };
 
             // Get the subcommand. If it's not `rustc` it should be a
             // subcommand that itself invokes `rustc` (so that the `FAKE_RUSTC`
diff --git a/collector/src/toolchain.rs b/collector/src/toolchain.rs
index c3c5986a6..d4c9d6893 100644
--- a/collector/src/toolchain.rs
+++ b/collector/src/toolchain.rs
@@ -591,7 +591,11 @@ pub fn get_local_toolchain(
         )
     } else if profiles.contains(&Profile::Clippy) {
         // We need a `clippy`. Look for one next to `rustc`.
-        if let Ok(clippy) = rustc.with_file_name("clippy-driver").canonicalize() {
+        if let Ok(clippy) = rustc
+            .with_file_name("clippy-driver")
+            .with_extension(rustc.extension().unwrap_or_default())
+            .canonicalize()
+        {
             debug!("found clippy: {:?}", &clippy);
             Some(clippy)
         } else {