Auto merge of #100260 - ehuss:compiletest-target-cfg, r=Mark-Simulacrum

compiletest: use target cfg instead of hard-coded tables

This changes compiletest to use `rustc --print=cfg` to dynamically determine the properties of a target when matching `ignore` and `only` rules instead of using hard-coded tables or target-triple parsing (which don't always follow the `<arch><sub>-<vendor>-<sys>-<abi>` pattern). The benefit here is that it will more accurately match the target properties, and not require maintaining these extra tables.

This also adds matching the `target_family` in ignore rules. The primary benefit here is so that `ignore-wasm` works as expected (matching all wasm-like targets). There were already several tests that had `ignore-wasm` in them (which previously had no effect), so it is evident that some people expected that to work. This also adds `ignore-unix/only-unix`.

There is some risk that this changes the behavior from before since the tables aren't quite the same as the target properties. However, I did fairly extensive comparisons to see what would be different. https://gist.github.com/ehuss/5bf7ab347605160cefb6f84ba5ea5f6b contains a full list of differences for all targets for all tests. I do not think any of the affected target/test combinations are things that are actually tested in CI. I tested several of the more unusual test images (test-various, dist-various-1, wasm32), and they seem fine.

A summary of most of the reasons behind the differences:

- wasm64-unknown-unknown wasm32-wasi now match "wasm"
- Targets now match "gnu" because they have target_env=gnu
    - aarch64-wrs-vxworks
    - armv7-wrs-vxworks-eabihf
    - i686-wrs-vxworks
    - powerpc-wrs-vxworks
    - powerpc64-wrs-vxworks
    - x86_64-wrs-vxworks
- wasm64-unknown-unknown now matches wasm64
- x86_64-unknown-none-linuxkernel no longer matches "linux", but will match "gnu" and "none"
- Various arm targets now match "aarch64" or "arm":
    - arm64_32-apple-watchos
    - armebv7r-none-eabi
    - armv6-unknown-freebsd
    - armv6-unknown-netbsd-eabihf
    - armv6k-nintendo-3ds
    - armv7-wrs-vxworks-eabihf
    - armv7a-kmc-solid_asp3-eabi
    - armv7a-kmc-solid_asp3-eabihf
    - armv7a-none-eabi
    - armv7a-none-eabihf
    - armv7k-apple-watchos
    - armv7r-none-eabi
    - armv7r-none-eabihf
- Now matches "thumb" and "arm"
    - thumbv4t-none-eabi
    - thumbv6m-none-eabi
    - thumbv7a-pc-windows-msvc
    - thumbv7a-uwp-windows-msvc
    - thumbv7em-none-eabi
    - thumbv7em-none-eabihf
    - thumbv7m-none-eabi
    - thumbv7neon-linux-androideabi
    - thumbv7neon-unknown-linux-gnueabihf
    - thumbv7neon-unknown-linux-musleabihf
    - thumbv8m.base-none-eabi
    - thumbv8m.main-none-eabi
- asmjs-unknown-emscripten now matches "wasm32" because that it is its defined arch
- avr-unknown-gnu-atmega328 now matches "none" (because target_os="none")
- now matches 64bit:
    - bpfeb-unknown-none
    - bpfel-unknown-none
    - sparcv9-sun-solaris
- now matches endian-big:
    - m68k-unknown-linux-gnu
- now matches 16bit:
    - msp430-none-elf
- now matches 32bit:
    - arm64_32-apple-watchos
- now matches riscv32 (and needs-asm-support):
    - riscv32gc-unknown-linux-gnu
    - riscv32gc-unknown-linux-musl
    - riscv32i-unknown-none-elf
    - riscv32im-unknown-none-elf
    - riscv32imac-unknown-none-elf
    - riscv32imac-unknown-xous-elf
    - riscv32imc-esp-espidf
    - riscv32imc-unknown-none-elf
    - riscv64imac-unknown-none-elf

Cargo.lock

@@ -817,6 +817,7 @@ dependencies = [
  "getopts",
  "glob",
  "lazy_static",
+ "lazycell",
  "libc",
  "miow",
  "regex",

src/tools/compiletest/Cargo.toml

@@ -17,6 +17,7 @@ rustfix = "0.6.0"
 lazy_static = "1.0"
 walkdir = "2"
 glob = "0.3.0"
+lazycell = "1.3.0"
 
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"

src/tools/compiletest/src/common.rs

@@ -3,9 +3,11 @@ pub use self::Mode::*;
 use std::ffi::OsString;
 use std::fmt;
 use std::path::{Path, PathBuf};
+use std::process::Command;
 use std::str::FromStr;
 
 use crate::util::PathBufExt;
+use lazycell::LazyCell;
 use test::ColorConfig;
 
 #[derive(Clone, Copy, PartialEq, Debug)]
@@ -371,6 +373,8 @@ pub struct Config {
 
     /// Whether to rerun tests even if the inputs are unchanged.
     pub force_rerun: bool,
+
+    pub target_cfg: LazyCell<TargetCfg>,
 }
 
 impl Config {
@@ -380,6 +384,131 @@ impl Config {
             !self.target.ends_with("-fuchsia")
         })
     }
+
+    fn target_cfg(&self) -> &TargetCfg {
+        self.target_cfg.borrow_with(|| TargetCfg::new(&self.rustc_path, &self.target))
+    }
+
+    pub fn matches_arch(&self, arch: &str) -> bool {
+        self.target_cfg().arch == arch ||
+        // Shorthand for convenience. The arch for
+        // asmjs-unknown-emscripten is actually wasm32.
+        (arch == "asmjs" && self.target.starts_with("asmjs")) ||
+        // Matching all the thumb variants as one can be convenient.
+        // (thumbv6m, thumbv7em, thumbv7m, etc.)
+        (arch == "thumb" && self.target.starts_with("thumb"))
+    }
+
+    pub fn matches_os(&self, os: &str) -> bool {
+        self.target_cfg().os == os
+    }
+
+    pub fn matches_env(&self, env: &str) -> bool {
+        self.target_cfg().env == env
+    }
+
+    pub fn matches_abi(&self, abi: &str) -> bool {
+        self.target_cfg().abi == abi
+    }
+
+    pub fn matches_family(&self, family: &str) -> bool {
+        self.target_cfg().families.iter().any(|f| f == family)
+    }
+
+    pub fn is_big_endian(&self) -> bool {
+        self.target_cfg().endian == Endian::Big
+    }
+
+    pub fn get_pointer_width(&self) -> u32 {
+        *&self.target_cfg().pointer_width
+    }
+
+    pub fn has_asm_support(&self) -> bool {
+        static ASM_SUPPORTED_ARCHS: &[&str] = &[
+            "x86", "x86_64", "arm", "aarch64", "riscv32",
+            "riscv64",
+            // These targets require an additional asm_experimental_arch feature.
+            // "nvptx64", "hexagon", "mips", "mips64", "spirv", "wasm32",
+        ];
+        ASM_SUPPORTED_ARCHS.contains(&self.target_cfg().arch.as_str())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct TargetCfg {
+    arch: String,
+    os: String,
+    env: String,
+    abi: String,
+    families: Vec<String>,
+    pointer_width: u32,
+    endian: Endian,
+}
+
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub enum Endian {
+    Little,
+    Big,
+}
+
+impl TargetCfg {
+    fn new(rustc_path: &Path, target: &str) -> TargetCfg {
+        let output = match Command::new(rustc_path)
+            .arg("--print=cfg")
+            .arg("--target")
+            .arg(target)
+            .output()
+        {
+            Ok(output) => output,
+            Err(e) => panic!("error: failed to get cfg info from {:?}: {e}", rustc_path),
+        };
+        if !output.status.success() {
+            panic!(
+                "error: failed to get cfg info from {:?}\n--- stdout\n{}\n--- stderr\n{}",
+                rustc_path,
+                String::from_utf8(output.stdout).unwrap(),
+                String::from_utf8(output.stderr).unwrap(),
+            );
+        }
+        let print_cfg = String::from_utf8(output.stdout).unwrap();
+        let mut arch = None;
+        let mut os = None;
+        let mut env = None;
+        let mut abi = None;
+        let mut families = Vec::new();
+        let mut pointer_width = None;
+        let mut endian = None;
+        for line in print_cfg.lines() {
+            if let Some((name, value)) = line.split_once('=') {
+                let value = value.trim_matches('"');
+                match name {
+                    "target_arch" => arch = Some(value),
+                    "target_os" => os = Some(value),
+                    "target_env" => env = Some(value),
+                    "target_abi" => abi = Some(value),
+                    "target_family" => families.push(value.to_string()),
+                    "target_pointer_width" => pointer_width = Some(value.parse().unwrap()),
+                    "target_endian" => {
+                        endian = Some(match value {
+                            "little" => Endian::Little,
+                            "big" => Endian::Big,
+                            s => panic!("unexpected {s}"),
+                        })
+                    }
+                    _ => {}
+                }
+            }
+        }
+        TargetCfg {
+            arch: arch.unwrap().to_string(),
+            os: os.unwrap().to_string(),
+            env: env.unwrap().to_string(),
+            abi: abi.unwrap().to_string(),
+            families,
+            pointer_width: pointer_width.unwrap(),
+            endian: endian.unwrap(),
+        }
+    }
 }
 
 #[derive(Debug, Clone)]

src/tools/compiletest/src/header.rs

@@ -661,17 +661,36 @@ impl Config {
 
         let name = line[prefix.len() + 1..].split(&[':', ' '][..]).next().unwrap();
 
+        let matches_pointer_width = || {
+            name.strip_suffix("bit")
+                .and_then(|width| width.parse::<u32>().ok())
+                .map(|width| self.get_pointer_width() == width)
+                .unwrap_or(false)
+        };
+
+        // If something is ignored for emscripten, it likely also needs to be
+        // ignored for wasm32-unknown-unknown.
+        // `wasm32-bare` is an alias to refer to just wasm32-unknown-unknown
+        // (in contrast to `wasm32` which also matches non-bare targets like
+        // asmjs-unknown-emscripten).
+        let matches_wasm32_alias = || {
+            self.target == "wasm32-unknown-unknown" && matches!(name, "emscripten" | "wasm32-bare")
+        };
+
         let is_match = name == "test" ||
             self.target == name ||                              // triple
-            util::matches_os(&self.target, name) ||             // target
-            util::matches_env(&self.target, name) ||            // env
+            self.matches_os(name) ||
+            self.matches_env(name) ||
+            self.matches_abi(name) ||
+            self.matches_family(name) ||
             self.target.ends_with(name) ||                      // target and env
-            name == util::get_arch(&self.target) ||             // architecture
-            name == util::get_pointer_width(&self.target) ||    // pointer width
+            self.matches_arch(name) ||
+            matches_wasm32_alias() ||
+            matches_pointer_width() ||
             name == self.stage_id.split('-').next().unwrap() || // stage
             name == self.channel ||                             // channel
             (self.target != self.host && name == "cross-compile") ||
-            (name == "endian-big" && util::is_big_endian(&self.target)) ||
+            (name == "endian-big" && self.is_big_endian()) ||
             (self.remote_test_client.is_some() && name == "remote") ||
             match self.compare_mode {
                 Some(CompareMode::Polonius) => name == "compare-mode-polonius",
@@ -869,7 +888,7 @@ pub fn make_test_description<R: Read>(
 
     let rustc_has_profiler_support = env::var_os("RUSTC_PROFILER_SUPPORT").is_some();
     let rustc_has_sanitizer_support = env::var_os("RUSTC_SANITIZER_SUPPORT").is_some();
-    let has_asm_support = util::has_asm_support(&config.target);
+    let has_asm_support = config.has_asm_support();
     let has_asan = util::ASAN_SUPPORTED_TARGETS.contains(&&*config.target);
     let has_cfi = util::CFI_SUPPORTED_TARGETS.contains(&&*config.target);
     let has_lsan = util::LSAN_SUPPORTED_TARGETS.contains(&&*config.target);