[naga xtask] Run validation jobs in parallel, using jobserver.

CHANGELOG.md

@@ -83,6 +83,8 @@ Wgpu now exposes backend feature for the Direct3D 12 (`dx12`) and Metal (`metal`
 
 - Add `--bulk-validate` option to Naga CLI. By @jimblandy in [#4871](https://github.com/gfx-rs/wgpu/pull/4871).
 
+- Naga's `cargo xtask validate` now runs validation jobs in parallel, using the [jobserver](https://crates.io/crates/jobserver) protocol to limit concurrency, and offers a `validate all` subcommand, which runs all available validation types. By @jimblandy in [#4902](https://github.com/gfx-rs/wgpu/pull/4902).
+
 ### Changes
 
 - Arcanization of wgpu core resources: By @gents83 in [#3626](https://github.com/gfx-rs/wgpu/pull/3626) and thanks also to @jimblandy, @nical, @Wumpf, @Elabajaba & @cwfitzgerald

naga/xtask/Cargo.toml

@@ -9,7 +9,10 @@ anyhow = "1"
 env_logger = { version = "0.10.0", default-features = false }
 glob = "0.3.1"
 hlsl-snapshots = { path = "../hlsl-snapshots"}
+indicatif = "0.17"
+jobserver = "0.1"
 log = "0.4.17"
+num_cpus = "1.16"
 pico-args = "0.5.0"
 shell-words = "1.1.0"
 which = "4.4.0"

naga/xtask/src/cli.rs

@@ -77,6 +77,8 @@ impl Subcommand {
     }
 }
 
+// If you add a new validation subcommand, be sure to update the code
+// that processes `All`.
 #[derive(Debug)]
 pub(crate) enum ValidateSubcommand {
     Spirv,
@@ -85,6 +87,7 @@ pub(crate) enum ValidateSubcommand {
     Dot,
     Wgsl,
     Hlsl(ValidateHlslCommand),
+    All,
 }
 
 impl ValidateSubcommand {
@@ -114,12 +117,29 @@ impl ValidateSubcommand {
                 ensure_remaining_args_empty(args)?;
                 Ok(Self::Wgsl)
             }
-            "hlsl" => return Ok(Self::Hlsl(ValidateHlslCommand::parse(args)?)),
+            "hlsl" => Ok(Self::Hlsl(ValidateHlslCommand::parse(args)?)),
+            "all" => {
+                ensure_remaining_args_empty(args)?;
+                Ok(Self::All)
+            }
             other => {
                 bail!("unrecognized `validate` subcommand {other:?}; see `--help` for more details")
             }
         }
     }
+
+    pub(crate) fn all() -> impl Iterator<Item = Self> {
+        [
+            Self::Spirv,
+            Self::Metal,
+            Self::Glsl,
+            Self::Dot,
+            Self::Wgsl,
+            Self::Hlsl(ValidateHlslCommand::Dxc),
+            Self::Hlsl(ValidateHlslCommand::Fxc),
+        ]
+        .into_iter()
+    }
 }
 
 #[derive(Debug)]

naga/xtask/src/glob.rs

@@ -1,37 +1,34 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use anyhow::Context;
 use glob::glob;
 
-use crate::result::{ErrorStatus, LogIfError};
+/// Apply `f` to each file matching `pattern` in `top_dir`.
+///
+/// Pass files as `anyhow::Result` values, to carry errors from
+/// directory iteration and metadata checking.
+pub(crate) fn for_each_file(
+    top_dir: impl AsRef<Path>,
+    pattern: impl AsRef<Path>,
+    mut f: impl FnMut(anyhow::Result<PathBuf>),
+) {
+    fn filter_files(glob: &str, result: glob::GlobResult) -> anyhow::Result<Option<PathBuf>> {
+        let path = result.with_context(|| format!("error while iterating over glob {glob:?}"))?;
+        let metadata = path
+            .metadata()
+            .with_context(|| format!("failed to fetch metadata for {path:?}"))?;
+        Ok(metadata.is_file().then_some(path))
+    }
 
-pub(crate) fn visit_files(
-    path: impl AsRef<Path>,
-    glob_expr: &str,
-    mut f: impl FnMut(&Path) -> anyhow::Result<()>,
-) -> ErrorStatus {
-    let path = path.as_ref();
-    let glob_expr = path.join(glob_expr);
-    let glob_expr = glob_expr.to_str().unwrap();
+    let pattern_in_dir = top_dir.as_ref().join(pattern.as_ref());
+    let pattern_in_dir = pattern_in_dir.to_str().unwrap();
 
-    let mut status = ErrorStatus::NoFailuresFound;
-    glob(glob_expr)
+    glob(pattern_in_dir)
         .context("glob pattern {path:?} is invalid")
         .unwrap()
-        .for_each(|path_res| {
-            if let Some(path) = path_res
-                .with_context(|| format!("error while iterating over glob {path:?}"))
-                .log_if_err_found(&mut status)
-            {
-                if path
-                    .metadata()
-                    .with_context(|| format!("failed to fetch metadata for {path:?}"))
-                    .log_if_err_found(&mut status)
-                    .map_or(false, |m| m.is_file())
-                {
-                    f(&path).log_if_err_found(&mut status);
-                }
+        .for_each(|result| {
+            if let Some(result) = filter_files(pattern_in_dir, result).transpose() {
+                f(result);
             }
         });
-    status
 }

naga/xtask/src/jobserver.rs

@@ -0,0 +1,34 @@
+//! Running jobs in parallel, with a controlled degree of concurrency.
+
+use std::sync::OnceLock;
+
+use jobserver::Client;
+
+static JOB_SERVER: OnceLock<Client> = OnceLock::new();
+
+pub fn init() {
+    JOB_SERVER.get_or_init(|| {
+        // Try to connect to a jobserver inherited from our parent.
+        if let Some(client) = unsafe { Client::from_env() } {
+            log::debug!("connected to inherited jobserver client");
+            client
+        } else {
+            // Otherwise, start our own jobserver.
+            log::debug!("no inherited jobserver client; creating a new jobserver");
+            Client::new(num_cpus::get()).expect("failed to create jobserver")
+        }
+    });
+}
+
+/// Wait until it is okay to start a new job, and then spawn a thread running `body`.
+pub fn start_job_thread<F>(body: F) -> anyhow::Result<()>
+where
+    F: FnOnce() + Send + 'static,
+{
+    let acquired = JOB_SERVER.get().unwrap().acquire()?;
+    std::thread::spawn(move || {
+        body();
+        drop(acquired);
+    });
+    Ok(())
+}