diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/cranelift-release-branch.yml b/compiler/rustc_codegen_cranelift/.github/workflows/cranelift-release-branch.yml new file mode 100644 index 0000000000000..5cfdc3e8f2936 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/.github/workflows/cranelift-release-branch.yml @@ -0,0 +1,64 @@ +name: Test upcoming Cranelift release branch + +on: + schedule: + - cron: "0 3 6 * *" + workflow_dispatch: {} + +permissions: {} + +env: + CARGO_BUILD_INCREMENTAL: false + RUSTFLAGS: "-Dwarnings" + +jobs: + test_upcoming_cranelift_release: + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - uses: actions/checkout@v6 + + - name: Determine latest Wasmtime release branch + id: wasmtime_release_branch + run: | + branches="$( + git ls-remote --heads https://github.com/bytecodealliance/wasmtime.git "refs/heads/release-*" \ + | awk '{print $2}' \ + | sed 's#refs/heads/##' \ + | sort -V + )" + if [[ -z "${branches}" ]]; then + echo "No wasmtime release branches found" + exit 1 + fi + latest="$(echo "${branches}" | tail -n 1)" + echo "Latest release branch: ${latest}" + echo "branch=${latest}" >> "$GITHUB_OUTPUT" + + - name: Patch Cargo.toml to use release branch Cranelift + run: | + cat >>Cargo.toml <", ); diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs index 216c87f095533..5c8b719ad5404 100644 --- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs +++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs @@ -231,6 +231,8 @@ fn build_clif_sysroot_for_triple( // inlining. rustflags.push("-Zinline-mir".to_owned()); + rustflags.push("-Zdisable-incr-comp-backend-caching".to_owned()); + if let Some(prefix) = env::var_os("CG_CLIF_STDLIB_REMAP_PATH_PREFIX") { rustflags.push("--remap-path-prefix".to_owned()); rustflags.push(format!("library/={}/library", prefix.to_str().unwrap())); diff --git a/compiler/rustc_codegen_cranelift/build_system/main.rs b/compiler/rustc_codegen_cranelift/build_system/main.rs index 0720d72c6d7cb..852fda950d88b 100644 --- a/compiler/rustc_codegen_cranelift/build_system/main.rs +++ b/compiler/rustc_codegen_cranelift/build_system/main.rs @@ -59,7 +59,6 @@ fn main() { if env::var_os("RUST_BACKTRACE").is_none() { env::set_var("RUST_BACKTRACE", "1"); } - env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); let mut args = env::args().skip(1); let command = match args.next().as_deref() { diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs index ba5cc9a29f599..1bc56e311ec7d 100644 --- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs +++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs @@ -11,11 +11,13 @@ pub(crate) fn prepare(dirs: &Dirs) { std::fs::create_dir_all(&dirs.download_dir).unwrap(); crate::tests::RAND_REPO.fetch(dirs); crate::tests::REGEX_REPO.fetch(dirs); + crate::tests::GRAVIOLA_REPO.fetch(dirs); } pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, + submodules: &'static [&'static str], content_hash: &'static str, patch_name: &'static str, } @@ -71,10 +73,17 @@ impl GitRepo { user: &'static str, repo: &'static str, rev: &'static str, + submodules: &'static [&'static str], content_hash: &'static str, patch_name: &'static str, ) -> GitRepo { - GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + GitRepo { + url: GitRepoUrl::Github { user, repo }, + rev, + submodules, + content_hash, + patch_name, + } } fn download_dir(&self, dirs: &Dirs) -> PathBuf { @@ -132,6 +141,7 @@ impl GitRepo { &download_dir, &format!("https://github.com/{}/{}.git", user, repo), self.rev, + self.submodules, ); } } @@ -160,7 +170,7 @@ impl GitRepo { } } -fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { +fn clone_repo(download_dir: &Path, repo: &str, rev: &str, submodules: &[&str]) { eprintln!("[CLONE] {}", repo); match fs::remove_dir_all(download_dir) { @@ -180,6 +190,13 @@ fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { checkout_cmd.arg("-q").arg(rev); spawn_and_wait(checkout_cmd); + if !submodules.is_empty() { + let mut submodule_cmd = git_command(download_dir, "submodule"); + submodule_cmd.arg("update").arg("--init"); + submodule_cmd.args(submodules); + spawn_and_wait(submodule_cmd); + } + std::fs::remove_dir_all(download_dir.join(".git")).unwrap(); } diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs index 3b6a2e7a055cb..685bf8ce9a891 100644 --- a/compiler/rustc_codegen_cranelift/build_system/tests.rs +++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs @@ -125,6 +125,7 @@ pub(crate) static RAND_REPO: GitRepo = GitRepo::github( "rust-random", "rand", "1f4507a8e1cf8050e4ceef95eeda8f64645b6719", + &[], "981f8bf489338978", "rand", ); @@ -135,12 +136,24 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( "rust-lang", "regex", "061ee815ef2c44101dba7b0b124600fcb03c1912", + &[], "dc26aefbeeac03ca", "regex", ); static REGEX: CargoProject = CargoProject::new(REGEX_REPO.source_dir(), "regex_target"); +pub(crate) static GRAVIOLA_REPO: GitRepo = GitRepo::github( + "ctz", + "graviola", + "c779b83cfd7114c4802293700c92cfb5e05cb4b7", + &["thirdparty/cavp", "thirdparty/wycheproof"], + "e0925ceb21a56101", + "graviola", +); + +static GRAVIOLA: CargoProject = CargoProject::new(GRAVIOLA_REPO.source_dir(), "graviola_target"); + static PORTABLE_SIMD_SRC: RelPath = RelPath::build("portable-simd"); static PORTABLE_SIMD: CargoProject = CargoProject::new(PORTABLE_SIMD_SRC, "portable-simd_target"); @@ -199,6 +212,43 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ spawn_and_wait(build_cmd); } }), + TestCase::custom("test.graviola", &|runner| { + let (arch, _) = runner.target_compiler.triple.split_once('-').unwrap(); + + if !["aarch64", "x86_64"].contains(&arch) { + eprintln!("Skipping `graviola` tests: unsupported target"); + return; + } + + GRAVIOLA_REPO.patch(&runner.dirs); + GRAVIOLA.clean(&runner.dirs); + + if runner.is_native { + let mut test_cmd = GRAVIOLA.test(&runner.target_compiler, &runner.dirs); + + // FIXME: Disable AVX-512 until intrinsics are supported. + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512f", "1"); + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512bw", "1"); + test_cmd.env("GRAVIOLA_CPU_DISABLE_avx512vl", "1"); + + test_cmd.args([ + "-p", + "graviola", + "--lib", + "--", + "-q", + // FIXME: Disable AVX-512 until intrinsics are supported. + "--skip", + "check_counter512", + ]); + spawn_and_wait(test_cmd); + } else { + eprintln!("Cross-Compiling: Not running tests"); + let mut build_cmd = GRAVIOLA.build(&runner.target_compiler, &runner.dirs); + build_cmd.args(["-p", "graviola", "--lib"]); + spawn_and_wait(build_cmd); + } + }), TestCase::custom("test.portable-simd", &|runner| { apply_patches( &runner.dirs, diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt index 72631355733c4..7c516e2164b40 100644 --- a/compiler/rustc_codegen_cranelift/config.txt +++ b/compiler/rustc_codegen_cranelift/config.txt @@ -20,7 +20,7 @@ aot.mini_core_hello_world testsuite.base_sysroot aot.arbitrary_self_types_pointers_and_wrappers -#jit.std_example # FIXME(#1619) broken for some reason +jit.std_example aot.std_example aot.dst_field_align aot.subslice-patterns-const-eval @@ -36,4 +36,5 @@ test.sysroot testsuite.extended_sysroot test.rust-random/rand test.regex +test.graviola test.portable-simd diff --git a/compiler/rustc_codegen_cranelift/example/neon.rs b/compiler/rustc_codegen_cranelift/example/neon.rs index 98a2a7af38f6b..1aec5badcbc26 100644 --- a/compiler/rustc_codegen_cranelift/example/neon.rs +++ b/compiler/rustc_codegen_cranelift/example/neon.rs @@ -9,6 +9,25 @@ use std::mem::transmute; #[cfg(target_arch = "aarch64")] use std::simd::*; +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "crc")] +unsafe fn test_crc32() { + assert!(std::arch::is_aarch64_feature_detected!("crc")); + + let a: u32 = 42; + let b: u64 = 0xdeadbeef; + + assert_eq!(__crc32b(a, b as u8), 0xEB0E363F); + assert_eq!(__crc32h(a, b as u16), 0x9A54BD80); + assert_eq!(__crc32w(a, b as u32), 0xF491F059); + assert_eq!(__crc32d(a, b as u64), 0xD14BBEA6); + + assert_eq!(__crc32cb(a, b as u8), 0xF67C32D8); + assert_eq!(__crc32ch(a, b as u16), 0x479108B8); + assert_eq!(__crc32cw(a, b as u32), 0x979F49F8); + assert_eq!(__crc32cd(a, b as u64), 0x0E6BE593); +} + #[cfg(target_arch = "aarch64")] unsafe fn test_vpmin_s8() { let a = i8x8::from([1, -2, 3, -4, 5, 6, 7, 8]); @@ -240,6 +259,272 @@ unsafe fn test_vrndnq_f32() { assert_eq!(r, e); } +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaeseq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aese + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let e = u8x16::from([ + 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, 0xca, + 0xca, + ]); + let r: u8x16 = unsafe { transmute(vaeseq_u8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesdq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesd + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); + let e = u8x16::from([ + 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, 0x7c, + 0x7c, + ]); + let r: u8x16 = unsafe { transmute(vaesdq_u8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesmcq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesmc + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let e = u8x16::from([2, 7, 0, 5, 6, 3, 4, 1, 10, 15, 8, 13, 14, 11, 12, 9]); + let r: u8x16 = unsafe { transmute(vaesmcq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +unsafe fn test_vaesimcq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.aesimc + let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + let e = u8x16::from([10, 15, 8, 13, 14, 11, 12, 9, 2, 7, 0, 5, 6, 3, 4, 1]); + let r: u8x16 = unsafe { transmute(vaesimcq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256hq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x27bb4ae0, 0xd8f61f7c, 0xb7c1ecdc, 0x10800215]); + let r: u32x4 = unsafe { transmute(vsha256hq_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256h2q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256h2 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x6989ee0d, 0x4b055920, 0x52800a12, 0x00000014]); + let r: u32x4 = unsafe { transmute(vsha256h2q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256su0q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su0 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let e = u32x4::from([0x02004000, 0x04008001, 0x0600c002, 0x08010003]); + let r: u32x4 = unsafe { transmute(vsha256su0q_u32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "sha2")] +unsafe fn test_vsha256su1q_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.crypto.sha256su1 + let a = u32x4::from([0, 1, 2, 3]); + let b = u32x4::from([4, 5, 6, 7]); + let c = u32x4::from([8, 9, 10, 11]); + let e = u32x4::from([0x00044005, 0x0004e007, 0xa802211b, 0xec036145]); + let r: u32x4 = unsafe { transmute(vsha256su1q_u32(transmute(a), transmute(b), transmute(c))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "aes")] +fn test_vmull_p64() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull64 + let a: u64 = 3; + let b: u64 = 6; + let e: u128 = 10; + let r: u128 = vmull_p64(a, b); + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vmull_p8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull.v8i16 + let a = u8x8::from([0, 1, 2, 3, 4, 5, 6, 7]); + let b = u8x8::from([8, 9, 10, 11, 12, 13, 14, 15]); + let e = u16x8::from([0x0000, 0x0009, 0x0014, 0x001d, 0x0030, 0x0039, 0x0024, 0x002d]); + let r: u16x8 = unsafe { transmute(vmull_p8(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulh_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i16 + let a = i16x4::from([1, 2, 4, 8]); + let b = i16x4::from([16384, 16384, 16384, 16384]); + let e = i16x4::from([0, 1, 2, 4]); + let r: i16x4 = unsafe { transmute(vqdmulh_s16(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulh_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v2i32 + let a = i32x2::from([1, 2]); + let b = i32x2::from([1073741824, 1073741824]); + let e = i32x2::from([0, 1]); + let r: i32x2 = unsafe { transmute(vqdmulh_s32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulhq_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v8i16 + let a = i16x8::from([1, 2, 4, 8, 16, 32, 64, 128]); + let b = i16x8::from([16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384]); + let e = i16x8::from([0, 1, 2, 4, 8, 16, 32, 64]); + let r: i16x8 = unsafe { transmute(vqdmulhq_s16(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vqdmulhq_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.sqdmulh.v4i32 + let a = i32x4::from([1, 2, 4, 8]); + let b = i32x4::from([1073741824, 1073741824, 1073741824, 1073741824]); + let e = i32x4::from([0, 1, 2, 4]); + let r: i32x4 = unsafe { transmute(vqdmulhq_s32(transmute(a), transmute(b))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i16.v8i8 + let a = i8x8::from([1, 2, 3, 4, -5, -6, -7, -8]); + let e = i16x4::from([3, 7, -11, -15]); + let r: i16x4 = unsafe { transmute(vpaddl_s8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i32.v4i16 + let a = i16x4::from([1, 2, -3, -4]); + let e = i32x2::from([3, -7]); + let r: i32x2 = unsafe { transmute(vpaddl_s16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v1i64.v2i32 + let a = i32x2::from([1, -2]); + let e = i64x1::from([-1]); + let r: i64x1 = unsafe { transmute(vpaddl_s32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v8i16.v16i8 + let a = i8x16::from([1, 2, 3, 4, 5, 6, 7, 8, -9, -10, -11, -12, -13, -14, -15, -16]); + let e = i16x8::from([3, 7, 11, 15, -19, -23, -27, -31]); + let r: i16x8 = unsafe { transmute(vpaddlq_s8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v4i32.v8i16 + let a = i16x8::from([1, 2, 3, 4, -5, -6, -7, -8]); + let e = i32x4::from([3, 7, -11, -15]); + let r: i32x4 = unsafe { transmute(vpaddlq_s16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_s32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.saddlp.v2i64.v4i32 + let a = i32x4::from([1, 2, -3, -4]); + let e = i64x2::from([3, -7]); + let r: i64x2 = unsafe { transmute(vpaddlq_s32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i16.v8i8 + let a = u8x8::from([255, 254, 253, 252, 251, 250, 249, 248]); + let e = u16x4::from([509, 505, 501, 497]); + let r: u16x4 = unsafe { transmute(vpaddl_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i32.v4i16 + let a = u16x4::from([65535, 65534, 65533, 65532]); + let e = u32x2::from([131069, 131065]); + let r: u32x2 = unsafe { transmute(vpaddl_u16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddl_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v1i64.v2i32 + let a = u32x2::from([4294967295, 4294967294]); + let e = u64x1::from([8589934589]); + let r: u64x1 = unsafe { transmute(vpaddl_u32(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u8() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v8i16.v16i8 + let a = u8x16::from([ + 255, 254, 253, 252, 251, 250, 249, 248, 247, 246, 245, 244, 243, 242, 241, 240, + ]); + let e = u16x8::from([509, 505, 501, 497, 493, 489, 485, 481]); + let r: u16x8 = unsafe { transmute(vpaddlq_u8(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u16() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v4i32.v8i16 + let a = u16x8::from([65535, 65534, 65533, 65532, 65531, 65530, 65529, 65528]); + let e = u32x4::from([131069, 131065, 131061, 131057]); + let r: u32x4 = unsafe { transmute(vpaddlq_u16(transmute(a))) }; + assert_eq!(r, e); +} + +#[cfg(target_arch = "aarch64")] +unsafe fn test_vpaddlq_u32() { + // AArch64 llvm intrinsic: llvm.aarch64.neon.uaddlp.v2i64.v4i32 + let a = u32x4::from([4294967295, 4294967294, 4294967293, 4294967292]); + let e = u64x2::from([8589934589, 8589934585]); + let r: u64x2 = unsafe { transmute(vpaddlq_u32(transmute(a))) }; + assert_eq!(r, e); +} + #[cfg(target_arch = "aarch64")] fn main() { unsafe { @@ -272,6 +557,40 @@ fn main() { test_vminq_f32(); test_vaddvq_f32(); test_vrndnq_f32(); + + test_crc32(); + + test_vaeseq_u8(); + test_vaesdq_u8(); + test_vaesmcq_u8(); + test_vaesimcq_u8(); + + test_vsha256hq_u32(); + test_vsha256h2q_u32(); + test_vsha256su0q_u32(); + test_vsha256su1q_u32(); + + test_vmull_p64(); + test_vmull_p8(); + + test_vqdmulh_s16(); + test_vqdmulh_s32(); + test_vqdmulhq_s16(); + test_vqdmulhq_s32(); + + test_vpaddl_s8(); + test_vpaddl_s16(); + test_vpaddl_s32(); + test_vpaddlq_s8(); + test_vpaddlq_s16(); + test_vpaddlq_s32(); + + test_vpaddl_u8(); + test_vpaddl_u16(); + test_vpaddl_u32(); + test_vpaddlq_u8(); + test_vpaddlq_u16(); + test_vpaddlq_u32(); } } diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index f0e38ae0610c9..252344b378e8a 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -1,8 +1,6 @@ #![feature(core_intrinsics, coroutines, coroutine_trait, repr_simd, tuple_trait, unboxed_closures)] #![allow(internal_features)] -#[cfg(target_arch = "x86_64")] -use std::arch::asm; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; use std::hint::black_box; @@ -591,7 +589,7 @@ unsafe fn test_xmm_roundtrip() { let input = [1u8; 16]; let mut output = [0u8; 16]; - asm!( + std::arch::asm!( "movups {xmm}, [{input}]", "movups [{output}], {xmm}", input = in(reg) input.as_ptr(), @@ -611,7 +609,7 @@ unsafe fn test_ymm_roundtrip() { let input = [1u8; 32]; let mut output = [0u8; 32]; - asm!( + std::arch::asm!( "vmovups {ymm}, [{input}]", "vmovups [{output}], {ymm}", input = in(reg) input.as_ptr(), @@ -631,7 +629,7 @@ unsafe fn test_zmm_roundtrip() { let input = [1u8; 64]; let mut output = [0u8; 64]; - asm!( + std::arch::asm!( "vmovups {zmm}, [{input}]", "vmovups [{output}], {zmm}", input = in(reg) input.as_ptr(), diff --git a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch index 029e493227cd1..a2fcd97349e2b 100644 --- a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch +++ b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-f16-usage-in-portable-simd.patch @@ -154,6 +154,21 @@ index 510f4c9..175cbce 100644 -impl_trait! { f16 { bits: u16, mask: i16 }, f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } +impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } } +diff --git a/crates/core_simd/src/simd/prelude.rs b/crates/core_simd/src/simd/prelude.rs +index 51b8def..6e93f16 100644 +--- a/crates/core_simd/src/simd/prelude.rs ++++ b/crates/core_simd/src/simd/prelude.rs +@@ -14,10 +14,6 @@ pub use super::{ + simd_swizzle, + }; + +-#[rustfmt::skip] +-#[doc(no_inline)] +-pub use super::{f16x1, f16x2, f16x4, f16x8, f16x16, f16x32, f16x64}; +- + #[rustfmt::skip] + #[doc(no_inline)] + pub use super::{f32x1, f32x2, f32x4, f32x8, f32x16, f32x32, f32x64}; diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index fbef69f..c8e0b8c 100644 --- a/crates/core_simd/src/vector.rs diff --git a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch index b7276e43153bc..717495cbcdf33 100644 --- a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch +++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch @@ -34,17 +34,17 @@ index a60f0799c0e..af056fbf41f 100644 #[cfg(target_has_atomic_load_store = "8")] #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")] diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs -index bf2b6d59f88..d5ccce03bbf 100644 +index 8a9a0b5..92ed9a6 100644 --- a/library/core/src/sync/atomic.rs +++ b/library/core/src/sync/atomic.rs -@@ -3585,42 +3585,6 @@ pub const fn as_ptr(&self) -> *mut $int_type { +@@ -3762,42 +3757,6 @@ atomic_int! { 8, u64 AtomicU64 } -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { - cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_equal_alignment = "128"), +- cfg(target_has_atomic_primitive_alignment = "128"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), @@ -62,7 +62,7 @@ index bf2b6d59f88..d5ccce03bbf 100644 -#[cfg(target_has_atomic_load_store = "128")] -atomic_int! { - cfg(target_has_atomic = "128"), -- cfg(target_has_atomic_equal_alignment = "128"), +- cfg(target_has_atomic_primitive_alignment = "128"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), - unstable(feature = "integer_atomics", issue = "99069"), diff --git a/compiler/rustc_codegen_cranelift/patches/0029-sysroot_tests-disable-f16-math.patch b/compiler/rustc_codegen_cranelift/patches/0029-sysroot_tests-disable-f16-math.patch index 2aeb4a8a1874c..8d21359aa0043 100644 --- a/compiler/rustc_codegen_cranelift/patches/0029-sysroot_tests-disable-f16-math.patch +++ b/compiler/rustc_codegen_cranelift/patches/0029-sysroot_tests-disable-f16-math.patch @@ -7,11 +7,389 @@ Subject: [PATCH] Disable f16 math tests for cranelift coretests/tests/num/floats.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) -diff --git a/coretests/tests/floats/mod.rs b/coretests/tests/floats/mod.rs -index c61961f8584..d7b4fa20322 100644 +diff --git a/coretests/tests/num/floats.rs b/coretests/tests/num/floats.rs +index 1d7956b..01e4caa 100644 --- a/coretests/tests/num/floats.rs +++ b/coretests/tests/num/floats.rs -@@ -1534,7 +1534,7 @@ fn s_nan() -> Float { +@@ -444,7 +444,7 @@ pub(crate) use float_test; + float_test! { + name: num, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -463,7 +463,7 @@ float_test! { + name: num_rem, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -476,7 +476,7 @@ float_test! { + float_test! { + name: nan, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -496,7 +496,7 @@ float_test! { + float_test! { + name: infinity, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -514,7 +514,7 @@ float_test! { + float_test! { + name: neg_infinity, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -532,7 +532,7 @@ float_test! { + float_test! { + name: zero, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -550,7 +550,7 @@ float_test! { + float_test! { + name: neg_zero, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -570,7 +570,7 @@ float_test! { + float_test! { + name: one, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -588,7 +588,7 @@ float_test! { + float_test! { + name: is_nan, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -609,7 +609,7 @@ float_test! { + float_test! { + name: is_infinite, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -630,7 +630,7 @@ float_test! { + float_test! { + name: is_finite, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -651,7 +651,7 @@ float_test! { + float_test! { + name: is_normal, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -673,7 +673,7 @@ float_test! { + float_test! { + name: classify, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + }, + test { + let nan: Float = Float::NAN; +@@ -695,7 +695,7 @@ float_test! { + name: min, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -737,7 +737,7 @@ float_test! { + name: max, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -780,7 +780,7 @@ float_test! { + name: minimum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -812,7 +812,7 @@ float_test! { + name: maximum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -845,7 +845,7 @@ float_test! { + name: midpoint, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -898,7 +898,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Needs powi +- f16: #[cfg(target_has_reliable_f16_math)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128_math)], + }, + test { +@@ -929,7 +929,7 @@ float_test! { + name: abs, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -948,7 +948,7 @@ float_test! { + name: copysign, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -964,7 +964,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -982,7 +982,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -998,7 +998,7 @@ float_test! { + name: floor, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1028,7 +1028,7 @@ float_test! { + name: ceil, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1058,7 +1058,7 @@ float_test! { + name: round, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1089,7 +1089,7 @@ float_test! { + name: round_ties_even, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1120,7 +1120,7 @@ float_test! { + name: trunc, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1150,7 +1150,7 @@ float_test! { + name: fract, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1182,7 +1182,7 @@ float_test! { + name: signum, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1200,7 +1200,7 @@ float_test! { + float_test! { + name: is_sign_positive, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1219,7 +1219,7 @@ float_test! { + float_test! { + name: is_sign_negative, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1238,7 +1238,7 @@ float_test! { + float_test! { + name: next_up, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1269,7 +1269,7 @@ float_test! { + float_test! { + name: next_down, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1303,7 +1303,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1321,7 +1321,7 @@ float_test! { + name: clamp_min_greater_than_max, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1335,7 +1335,7 @@ float_test! { + name: clamp_min_is_nan, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1349,7 +1349,7 @@ float_test! { + name: clamp_max_is_nan, + attrs: { + const: #[cfg(false)], +- f16: #[should_panic, cfg(target_has_reliable_f16)], ++ f16: #[should_panic, cfg(false)], + f32: #[should_panic], + f64: #[should_panic], + f128: #[should_panic, cfg(target_has_reliable_f128)], +@@ -1363,7 +1363,7 @@ float_test! { + name: total_cmp, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1469,7 +1469,7 @@ float_test! { + attrs: { + const: #[cfg(false)], + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1526,7 +1526,7 @@ float_test! { + name: recip, + attrs: { + // Miri only uses softfloats here, so that always works +- f16: #[cfg(any(miri, target_has_reliable_f16_math))], ++ f16: #[cfg(false)], + f128: #[cfg(any(miri, target_has_reliable_f128_math))], + }, + test { +@@ -1549,7 +1549,7 @@ float_test! { + name: powi, + attrs: { + const: #[cfg(false)], +- f16: #[cfg(target_has_reliable_f16_math)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128_math)], + }, + test { +@@ -1570,7 +1570,7 @@ float_test! { name: powf, attrs: { const: #[cfg(false)], @@ -20,7 +398,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1557,7 +1557,7 @@ fn s_nan() -> Float { +@@ -1593,7 +1593,7 @@ float_test! { name: exp, attrs: { const: #[cfg(false)], @@ -29,7 +407,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1578,7 +1578,7 @@ fn s_nan() -> Float { +@@ -1614,7 +1614,7 @@ float_test! { name: exp2, attrs: { const: #[cfg(false)], @@ -38,7 +416,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1598,7 +1598,7 @@ fn s_nan() -> Float { +@@ -1634,7 +1634,7 @@ float_test! { name: ln, attrs: { const: #[cfg(false)], @@ -47,7 +425,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1620,7 +1620,7 @@ fn s_nan() -> Float { +@@ -1656,7 +1656,7 @@ float_test! { name: log, attrs: { const: #[cfg(false)], @@ -56,7 +434,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1645,7 +1645,7 @@ fn s_nan() -> Float { +@@ -1681,7 +1681,7 @@ float_test! { name: log2, attrs: { const: #[cfg(false)], @@ -65,7 +443,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1668,7 +1668,7 @@ fn s_nan() -> Float { +@@ -1704,7 +1704,7 @@ float_test! { name: log10, attrs: { const: #[cfg(false)], @@ -74,7 +452,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1692,7 +1692,7 @@ fn s_nan() -> Float { +@@ -1728,7 +1728,7 @@ float_test! { name: asinh, attrs: { const: #[cfg(false)], @@ -83,7 +461,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1725,7 +1725,7 @@ fn s_nan() -> Float { +@@ -1764,7 +1764,7 @@ float_test! { name: acosh, attrs: { const: #[cfg(false)], @@ -92,7 +470,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1753,7 +1753,7 @@ fn s_nan() -> Float { +@@ -1795,7 +1795,7 @@ float_test! { name: atanh, attrs: { const: #[cfg(false)], @@ -101,7 +479,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1779,7 +1779,7 @@ fn s_nan() -> Float { +@@ -1821,7 +1821,7 @@ float_test! { name: gamma, attrs: { const: #[cfg(false)], @@ -110,7 +488,7 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -1814,7 +1814,7 @@ fn s_nan() -> Float { +@@ -1856,7 +1856,7 @@ float_test! { name: ln_gamma, attrs: { const: #[cfg(false)], @@ -119,7 +497,79 @@ index c61961f8584..d7b4fa20322 100644 f128: #[cfg(target_has_reliable_f128_math)], }, test { -@@ -2027,7 +2027,7 @@ fn s_nan() -> Float { +@@ -1874,7 +1874,7 @@ float_test! { + float_test! { + name: to_degrees, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1895,7 +1895,7 @@ float_test! { + float_test! { + name: to_radians, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1916,7 +1916,7 @@ float_test! { + float_test! { + name: to_algebraic, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1940,7 +1940,7 @@ float_test! { + float_test! { + name: to_bits_conv, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -1967,7 +1967,7 @@ float_test! { + float_test! { + name: mul_add, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + // FIXME(#140515): mingw has an incorrect fma https://sourceforge.net/p/mingw-w64/bugs/848/ + f32: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], + f64: #[cfg_attr(all(target_os = "windows", target_env = "gnu", not(target_abi = "llvm")), ignore)], +@@ -1992,7 +1992,7 @@ float_test! { + float_test! { + name: from, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2049,7 +2049,7 @@ float_test! { + float_test! { + name: max_exact_integer_constant, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2091,7 +2091,7 @@ float_test! { + float_test! { + name: min_exact_integer_constant, + attrs: { +- f16: #[cfg(target_has_reliable_f16)], ++ f16: #[cfg(false)], + f128: #[cfg(target_has_reliable_f128)], + }, + test { +@@ -2156,7 +2156,7 @@ float_test! { attrs: { // FIXME(f16_f128): add math tests when available const: #[cfg(false)], diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain.toml b/compiler/rustc_codegen_cranelift/rust-toolchain.toml index 486078185db84..faadf08929557 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain.toml +++ b/compiler/rustc_codegen_cranelift/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2026-04-28" +channel = "nightly-2026-06-06" components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] profile = "minimal" diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh index 7ce54a45e2ce6..ab31c43fb1b12 100644 --- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh +++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh @@ -54,7 +54,7 @@ index 2e16f2cf27..3ac3df99a8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1147,6 +1147,8 @@ class RustBuild(object): - args += ["-Zwarnings"] + if deny_warnings: env["CARGO_BUILD_WARNINGS"] = "deny" + env["RUSTFLAGS"] += " -Zbinary-dep-depinfo" diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index 4a663c48c0aff..683adeb49edd1 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -88,6 +88,8 @@ rm -r tests/run-make/reproducible-build-2 rm -r tests/run-make/no-builtins-lto rm -r tests/run-make/reachable-extern-fn-available-lto rm -r tests/run-make/no-builtins-linker-plugin-lto +rm -r tests/run-make/fat-then-thin-lto +rm -r tests/run-make/cross-lang-lto-upstream-rlibs # coverage instrumentation rm tests/ui/consts/precise-drop-with-coverage.rs @@ -146,6 +148,7 @@ rm tests/ui/consts/const-mut-refs-crate.rs # same rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift rm -r tests/run-make/short-ice # ICE backtrace begin/end marker mismatch rm -r tests/run-make/naked-dead-code-elimination # function not eliminated +rm tests/ui/codegen/huge-stacks.rs # Cranelift doesn't allow stack frames to exceed 4GB # doesn't work due to the way the rustc test suite is invoked. # should work when using ./x.py test the way it is intended diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index a8b179f169bf6..2ace6afd13e88 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -31,6 +31,11 @@ use crate::base::codegen_unwind_terminate; use crate::debuginfo::EXCEPTION_HANDLER_CLEANUP; use crate::prelude::*; +struct ArgValue<'tcx> { + value: CValue<'tcx>, + is_underaligned_pointee: bool, +} + fn clif_sig_from_fn_abi<'tcx>( tcx: TyCtxt<'tcx>, default_call_conv: CallConv, @@ -245,8 +250,8 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ // None means pass_mode == NoPass enum ArgKind<'tcx> { - Normal(Option>), - Spread(Vec>>), + Normal(Option>), + Spread(Vec>>), } // FIXME implement variadics in cranelift @@ -299,8 +304,12 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ if fx.instance.def.requires_caller_location(fx.tcx) { // Store caller location for `#[track_caller]`. let arg_abi = arg_abis_iter.next().unwrap(); - fx.caller_location = - Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); + let param = cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap(); + assert!( + !param.is_underaligned_pointee, + "caller location argument should not be underaligned", + ); + fx.caller_location = Some(param.value); } assert_eq!(arg_abis_iter.next(), None, "ArgAbi left behind for {:?}", fx.fn_abi); @@ -311,23 +320,24 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ for (local, arg_kind, ty) in func_params { // While this is normally an optimization to prevent an unnecessary copy when an argument is // not mutated by the current function, this is necessary to support unsized arguments. - if let ArgKind::Normal(Some(val)) = arg_kind { - if let Some((addr, meta)) = val.try_to_ptr() { - // Ownership of the value at the backing storage for an argument is passed to the - // callee per the ABI, so it is fine to borrow the backing storage of this argument - // to prevent a copy. - - let place = if let Some(meta) = meta { - CPlace::for_ptr_with_extra(addr, meta, val.layout()) - } else { - CPlace::for_ptr(addr, val.layout()) - }; + if let ArgKind::Normal(Some(ArgValue { value: val, is_underaligned_pointee: false })) = + arg_kind + && let Some((addr, meta)) = val.try_to_ptr() + { + // Ownership of the value at the backing storage for an argument is passed to the + // callee per the ABI, so it is fine to borrow the backing storage of this argument + // to prevent a copy. + + let place = if let Some(meta) = meta { + CPlace::for_ptr_with_extra(addr, meta, val.layout()) + } else { + CPlace::for_ptr(addr, val.layout()) + }; - self::comments::add_local_place_comments(fx, place, local); + self::comments::add_local_place_comments(fx, place, local); - assert_eq!(fx.local_map.push(place), local); - continue; - } + assert_eq!(fx.local_map.push(place), local); + continue; } let layout = fx.layout_of(ty); @@ -338,13 +348,22 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ match arg_kind { ArgKind::Normal(param) => { if let Some(param) = param { - place.write_cvalue(fx, param); + if param.is_underaligned_pointee { + place.write_cvalue_transmute(fx, param.value); + } else { + place.write_cvalue(fx, param.value); + } } } ArgKind::Spread(params) => { for (i, param) in params.into_iter().enumerate() { if let Some(param) = param { - place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param); + let field_place = place.place_field(fx, FieldIdx::new(i)); + if param.is_underaligned_pointee { + field_place.write_cvalue_transmute(fx, param.value); + } else { + field_place.write_cvalue(fx, param.value); + } } } } diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index edbee60471830..612f89e6a4217 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -7,6 +7,7 @@ use rustc_target::callconv::{ }; use smallvec::{SmallVec, smallvec}; +use super::ArgValue; use crate::prelude::*; use crate::value_and_place::assert_assignable; @@ -285,7 +286,7 @@ pub(super) fn cvalue_for_param<'tcx>( local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, block_params_iter: &mut impl Iterator, -) -> Option> { +) -> Option> { let block_params = arg_abi .get_abi_param(fx.tcx) .into_iter() @@ -306,30 +307,42 @@ pub(super) fn cvalue_for_param<'tcx>( arg_abi.layout, ); - match arg_abi.mode { - PassMode::Ignore => None, + let value = match arg_abi.mode { + PassMode::Ignore => return None, PassMode::Direct(_) => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_val(block_params[0], arg_abi.layout)) + CValue::by_val(block_params[0], arg_abi.layout) } PassMode::Pair(_, _) => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) + CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout) } PassMode::Cast { ref cast, .. } => { - Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + from_casted_value(fx, &block_params, arg_abi.layout, cast) } - PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); - Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) + if let Some(pointee_align) = attrs.pointee_align + && pointee_align < arg_abi.layout.align.abi + && arg_abi.layout.is_sized() + && arg_abi.layout.size != Size::ZERO + { + // Underaligned pointer: treat as `[u8; size]` and transmute-copy into the real type. + let bytes_ty = Ty::new_array(fx.tcx, fx.tcx.types.u8, arg_abi.layout.size.bytes()); + let bytes_layout = fx.layout_of(bytes_ty); + return Some(ArgValue { + value: CValue::by_ref(Pointer::new(block_params[0]), bytes_layout), + is_underaligned_pointee: true, + }); + } else { + CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout) + } } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); - Some(CValue::by_ref_unsized( - Pointer::new(block_params[0]), - block_params[1], - arg_abi.layout, - )) + CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout) } - } + }; + + Some(ArgValue { value, is_underaligned_pointee: false }) } diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index 4a9b0c0952ff3..3c18748ee24b4 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -5,20 +5,11 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use rustc_ast::expand::allocator::{ AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; -use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; use rustc_symbol_mangling::mangle_internal_symbol; use crate::prelude::*; -/// Returns whether an allocator shim was created -pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { - let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; - let methods = allocator_shim_contents(tcx, kind); - codegen_inner(tcx, module, &methods); - true -} - -fn codegen_inner(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[AllocatorMethod]) { +pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module, methods: &[AllocatorMethod]) { let usize_ty = module.target_config().pointer_type(); for method in methods { diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs index 72380f50385a1..c4a31cabcf385 100644 --- a/compiler/rustc_codegen_cranelift/src/analyze.rs +++ b/compiler/rustc_codegen_cranelift/src/analyze.rs @@ -23,14 +23,10 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec { for bb in fx.mir.basic_blocks.iter() { for stmt in bb.statements.iter() { - match &stmt.kind { - Assign(place_and_rval) => match &place_and_rval.1 { - Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => { - flag_map[place.local] = SsaKind::NotSsa; - } - _ => {} - }, - _ => {} + if let Assign(place_and_rval) = &stmt.kind + && let Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) = &place_and_rval.1 + { + flag_map[place.local] = SsaKind::NotSsa; } } } diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 2edbdb560f52b..467eceea221c8 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -147,7 +147,7 @@ pub(crate) fn codegen_fn<'tcx>( } pub(crate) fn compile_fn( - profiler: &SelfProfilerRef, + prof: &SelfProfilerRef, output_filenames: &OutputFilenames, should_write_ir: bool, cached_context: &mut Context, @@ -156,8 +156,7 @@ pub(crate) fn compile_fn( global_asm: &mut String, codegened_func: CodegenedFunction, ) { - let _timer = - profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); + let _timer = prof.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); let clif_comments = codegened_func.clif_comments; global_asm.push_str(&codegened_func.inline_asm); @@ -196,7 +195,7 @@ pub(crate) fn compile_fn( }; // Define function - profiler.generic_activity("define function").run(|| { + prof.generic_activity("define function").run(|| { context.want_disasm = should_write_ir; match module.define_function(codegened_func.func_id, context) { Ok(()) => {} @@ -248,7 +247,7 @@ pub(crate) fn compile_fn( } // Define debuginfo for function - profiler.generic_activity("generate debug info").run(|| { + prof.generic_activity("generate debug info").run(|| { if let Some(debug_context) = debug_context { codegened_func.func_debug_cx.unwrap().finalize( debug_context, @@ -1052,7 +1051,7 @@ pub(crate) fn codegen_operand<'tcx>( Operand::RuntimeChecks(checks) => { let val = checks.value(fx.tcx.sess); let layout = fx.layout_of(fx.tcx.types.bool); - return CValue::const_val(fx, layout, val.into()); + CValue::const_val(fx, layout, val.into()) } } } diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs deleted file mode 100644 index b5a81fc11d57b..0000000000000 --- a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::sync::{Arc, Condvar, Mutex}; - -use rustc_data_structures::jobserver::{self, HelperThread}; -use rustc_errors::DiagCtxtHandle; - -// FIXME don't panic when a worker thread panics - -pub(super) struct ConcurrencyLimiter { - helper_thread: Option>, - state: Arc>, - available_token_condvar: Arc, - finished: bool, -} - -impl ConcurrencyLimiter { - pub(super) fn new(pending_jobs: usize) -> Self { - let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs))); - let available_token_condvar = Arc::new(Condvar::new()); - - let state_helper = state.clone(); - let available_token_condvar_helper = available_token_condvar.clone(); - let helper_thread = jobserver::client() - .clone() - .into_helper_thread(move |token| { - let mut state = state_helper.lock().unwrap(); - match token { - Ok(token) => { - state.add_new_token(token); - available_token_condvar_helper.notify_one(); - } - Err(err) => { - state.poison(format!("failed to acquire jobserver token: {}", err)); - // Notify all threads waiting for a token to give them a chance to - // gracefully exit. - available_token_condvar_helper.notify_all(); - } - } - }) - .unwrap(); - ConcurrencyLimiter { - helper_thread: Some(Mutex::new(helper_thread)), - state, - available_token_condvar, - finished: false, - } - } - - pub(super) fn acquire(&self, dcx: DiagCtxtHandle<'_>) -> ConcurrencyLimiterToken { - let mut state = self.state.lock().unwrap(); - loop { - state.assert_invariants(); - - match state.try_start_job() { - Ok(true) => { - return ConcurrencyLimiterToken { - state: self.state.clone(), - available_token_condvar: self.available_token_condvar.clone(), - }; - } - Ok(false) => {} - Err(err) => { - // An error happened when acquiring the token. Raise it as fatal error. - // Make sure to drop the mutex guard first to prevent poisoning the mutex. - drop(state); - if let Some(err) = err { - dcx.fatal(err); - } else { - // The error was already emitted, but compilation continued. Raise a silent - // fatal error. - rustc_errors::FatalError.raise(); - } - } - } - - self.helper_thread.as_ref().unwrap().lock().unwrap().request_token(); - state = self.available_token_condvar.wait(state).unwrap(); - } - } - - pub(crate) fn finished(mut self) { - self.helper_thread.take(); - - // Assert that all jobs have finished - let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap(); - state.assert_done(); - - self.finished = true; - } -} - -impl Drop for ConcurrencyLimiter { - fn drop(&mut self) { - if !self.finished && !std::thread::panicking() { - panic!("Forgot to call finished() on ConcurrencyLimiter"); - } - } -} - -#[derive(Debug)] -pub(super) struct ConcurrencyLimiterToken { - state: Arc>, - available_token_condvar: Arc, -} - -impl Drop for ConcurrencyLimiterToken { - fn drop(&mut self) { - let mut state = self.state.lock().unwrap(); - state.job_finished(); - self.available_token_condvar.notify_one(); - } -} - -mod state { - use rustc_data_structures::jobserver::Acquired; - - #[derive(Debug)] - pub(super) struct ConcurrencyLimiterState { - pending_jobs: usize, - active_jobs: usize, - - poisoned: bool, - stored_error: Option, - - // None is used to represent the implicit token, Some to represent explicit tokens - tokens: Vec>, - } - - impl ConcurrencyLimiterState { - pub(super) fn new(pending_jobs: usize) -> Self { - ConcurrencyLimiterState { - pending_jobs, - active_jobs: 0, - poisoned: false, - stored_error: None, - tokens: vec![None], - } - } - - pub(super) fn assert_invariants(&self) { - // There must be no excess active jobs - assert!(self.active_jobs <= self.pending_jobs); - - // There may not be more active jobs than there are tokens - assert!(self.active_jobs <= self.tokens.len()); - } - - pub(super) fn assert_done(&self) { - assert_eq!(self.pending_jobs, 0); - assert_eq!(self.active_jobs, 0); - } - - pub(super) fn add_new_token(&mut self, token: Acquired) { - self.tokens.push(Some(token)); - self.drop_excess_capacity(); - } - - pub(super) fn try_start_job(&mut self) -> Result> { - if self.poisoned { - return Err(self.stored_error.take()); - } - - if self.active_jobs < self.tokens.len() { - // Using existing token - self.job_started(); - return Ok(true); - } - - Ok(false) - } - - pub(super) fn job_started(&mut self) { - self.assert_invariants(); - self.active_jobs += 1; - self.drop_excess_capacity(); - self.assert_invariants(); - } - - pub(super) fn job_finished(&mut self) { - self.assert_invariants(); - self.pending_jobs -= 1; - self.active_jobs -= 1; - self.assert_invariants(); - self.drop_excess_capacity(); - self.assert_invariants(); - } - - pub(super) fn poison(&mut self, error: String) { - self.poisoned = true; - self.stored_error = Some(error); - } - - fn drop_excess_capacity(&mut self) { - self.assert_invariants(); - - // Drop all tokens that can never be used anymore - self.tokens.truncate(std::cmp::max(self.pending_jobs, 1)); - - // Keep some excess tokens to satisfy requests faster - const MAX_EXTRA_CAPACITY: usize = 2; - self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1)); - - self.assert_invariants(); - } - } -} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index f85d21db11fb2..c986666f9c46e 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -455,7 +455,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } else { ("", section_name.as_str()) }; - data.set_segment_section(segment_name, section_name); + // FIXME pass correct section flags on Mach-O + data.set_segment_section(segment_name, section_name, 0); } let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs index 1ce424332db20..4b0260a8abc74 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs @@ -204,7 +204,7 @@ impl UnwindContext { let mut data = DataDescription::new(); data.define(gcc_except_table.writer.into_vec().into_boxed_slice()); - data.set_segment_section("", ".gcc_except_table"); + data.set_segment_section("", ".gcc_except_table", 0); for reloc in &gcc_except_table.relocs { match reloc.name { diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 323fec06bcc58..fcaf80d968a49 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -1,184 +1,103 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. -use std::env; +use std::convert::Infallible; use std::fs::File; use std::io::BufWriter; use std::path::PathBuf; use std::sync::Arc; -use std::thread::JoinHandle; +use std::time::Instant; use cranelift_object::{ObjectBuilder, ObjectModule}; -use rustc_codegen_ssa::assert_module_sources::CguReuse; -use rustc_codegen_ssa::back::write::produce_final_output_artifacts; -use rustc_codegen_ssa::base::determine_cgu_reuse; -use rustc_codegen_ssa::{CompiledModule, CompiledModules, ModuleKind}; +use rustc_ast::expand::allocator::AllocatorMethod; +use rustc_codegen_ssa::back::lto::ThinModule; +use rustc_codegen_ssa::back::write::{ + CodegenContext, FatLtoInput, ModuleConfig, SharedEmitter, TargetMachineFactoryFn, ThinLtoInput, +}; +use rustc_codegen_ssa::traits::{ExtraBackendMethods, WriteBackendMethods}; +use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::stable_hash::{StableHash, StableHashCtxt, StableHasher}; -use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; +use rustc_errors::DiagCtxt; use rustc_hir::attrs::Linkage as RLinkage; -use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::dep_graph::WorkProduct; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mono::{CodegenUnit, MonoItem, MonoItemData, Visibility}; +use rustc_middle::mono::{MonoItem, MonoItemData, Visibility}; use rustc_session::Session; -use rustc_session::config::{OutputFilenames, OutputType}; +use rustc_session::config::{OptLevel, OutputFilenames, OutputType}; +use rustc_span::Symbol; use crate::base::CodegenedFunction; -use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; use crate::debuginfo::TypeDebugContext; use crate::global_asm::{GlobalAsmConfig, GlobalAsmContext}; use crate::prelude::*; use crate::unwind_module::UnwindModule; -fn disable_incr_cache() -> bool { - env::var("CG_CLIF_DISABLE_INCR_CACHE").as_deref() == Ok("1") -} - -struct ModuleCodegenResult { - module: CompiledModule, - existing_work_product: Option<(WorkProductId, WorkProduct)>, -} - -enum OngoingModuleCodegen { - Sync(Result), - Async(JoinHandle>), -} - -impl StableHash for OngoingModuleCodegen { - fn stable_hash(&self, _: &mut Hcx, _: &mut StableHasher) { - // do nothing - } -} - -pub(crate) struct OngoingCodegen { - modules: Vec, - allocator_module: Option, - concurrency_limiter: ConcurrencyLimiter, -} - -impl OngoingCodegen { - pub(crate) fn join( - self, - sess: &Session, - outputs: &OutputFilenames, - ) -> (CompiledModules, FxIndexMap) { - let mut work_products = FxIndexMap::default(); - let mut modules = vec![]; - let disable_incr_cache = disable_incr_cache(); - - for module_codegen in self.modules { - let module_codegen_result = match module_codegen { - OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result, - OngoingModuleCodegen::Async(join_handle) => match join_handle.join() { - Ok(module_codegen_result) => module_codegen_result, - Err(panic) => std::panic::resume_unwind(panic), - }, - }; - - let module_codegen_result = match module_codegen_result { - Ok(module_codegen_result) => module_codegen_result, - Err(err) => sess.dcx().fatal(err), - }; - let ModuleCodegenResult { module, existing_work_product } = module_codegen_result; - - if let Some((work_product_id, work_product)) = existing_work_product { - work_products.insert(work_product_id, work_product); - } else { - let work_product = if disable_incr_cache { - None - } else if let Some(global_asm_object) = &module.global_asm_object { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - sess, - &module.name, - &[("o", module.object.as_ref().unwrap()), ("asm.o", global_asm_object)], - &[], - ) - } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - sess, - &module.name, - &[("o", module.object.as_ref().unwrap())], - &[], - ) - }; - if let Some((work_product_id, work_product)) = work_product { - work_products.insert(work_product_id, work_product); - } - } - - modules.push(module); - } - - self.concurrency_limiter.finished(); - - sess.dcx().abort_if_errors(); - - let compiled_modules = CompiledModules { modules, allocator_module: self.allocator_module }; - - produce_final_output_artifacts(sess, &compiled_modules, outputs); - - (compiled_modules, work_products) - } +pub(crate) struct AotModule { + producer: String, + global_asm_config: GlobalAsmConfig, + module: UnwindModule, + debug_context: Option, + codegened_functions: Vec, + global_asm: String, } -fn make_module(sess: &Session, name: String) -> UnwindModule { - let isa = crate::build_isa(sess, false); +fn make_module(tcx: TyCtxt<'_>, cgu_name: &str) -> AotModule { + let isa = crate::build_isa(tcx.sess, false); - let mut builder = - ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); + let mut builder = ObjectBuilder::new( + isa, + cgu_name.to_owned() + ".o", + cranelift_module::default_libcall_names(), + ) + .unwrap(); // Disable function sections by default on MSVC as it causes significant slowdowns with link.exe. // Maybe link.exe has exponential behavior when there are many sections with the same name? Also // explicitly disable it on MinGW as rustc already disables it by default on MinGW and as such // isn't tested. If rustc enables it in the future on MinGW, we can re-enable it too once it has // been on MinGW. - let default_function_sections = sess.target.function_sections && !sess.target.is_like_windows; + let default_function_sections = + tcx.sess.target.function_sections && !tcx.sess.target.is_like_windows; builder.per_function_section( - sess.opts.unstable_opts.function_sections.unwrap_or(default_function_sections), + tcx.sess.opts.unstable_opts.function_sections.unwrap_or(default_function_sections), ); - UnwindModule::new(ObjectModule::new(builder), true) -} + let module = UnwindModule::new(ObjectModule::new(builder), true); -fn emit_cgu( - output_filenames: &OutputFilenames, - prof: &SelfProfilerRef, - name: String, - module: UnwindModule, - debug: Option, - global_asm_object_file: Option, - producer: &str, -) -> Result { - let mut product = module.finish(); - - if let Some(mut debug) = debug { - debug.emit(&mut product); - } + let producer = crate::debuginfo::producer(tcx.sess); + let global_asm_config = GlobalAsmConfig::new(tcx.sess); + let debug_context = DebugContext::new(tcx, module.isa(), false, cgu_name); + let codegened_functions = vec![]; + let global_asm = String::new(); - let module = emit_module( - output_filenames, - prof, - product.object, - ModuleKind::Regular, - name.clone(), - global_asm_object_file, + AotModule { producer, - )?; - - Ok(ModuleCodegenResult { module, existing_work_product: None }) + global_asm_config, + module, + debug_context, + codegened_functions, + global_asm, + } } fn emit_module( output_filenames: &OutputFilenames, prof: &SelfProfilerRef, - mut object: cranelift_object::object::write::Object<'_>, + module: UnwindModule, + debug: Option, kind: ModuleKind, name: String, global_asm_object: Option, producer_str: &str, ) -> Result { - if object.format() == cranelift_object::object::BinaryFormat::Elf { - let comment_section = object.add_section( + let mut product = module.finish(); + + if let Some(mut debug) = debug { + debug.emit(&mut product); + } + + if product.object.format() == cranelift_object::object::BinaryFormat::Elf { + let comment_section = product.object.add_section( Vec::new(), b".comment".to_vec(), cranelift_object::object::SectionKind::OtherString, @@ -186,7 +105,7 @@ fn emit_module( let mut producer = vec![0]; producer.extend(producer_str.as_bytes()); producer.push(0); - object.set_section_data(comment_section, producer, 1); + product.object.set_section_data(comment_section, producer, 1); } let tmp_file = output_filenames.temp_path_for_cgu(OutputType::Object, &name); @@ -196,7 +115,7 @@ fn emit_module( }; let mut file = BufWriter::new(file); - if let Err(err) = object.write_stream(&mut file) { + if let Err(err) = product.object.write_stream(&mut file) { return Err(format!("error writing object file: {}", err)); } let file = match file.into_inner() { @@ -225,87 +144,22 @@ fn emit_module( }) } -fn reuse_workproduct_for_cgu( - tcx: TyCtxt<'_>, - cgu: &CodegenUnit<'_>, -) -> Result { - let work_product = cgu.previous_work_product(tcx); - let obj_out_regular = - tcx.output_filenames(()).temp_path_for_cgu(OutputType::Object, cgu.name().as_str()); - let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( - tcx.sess, - work_product.saved_files.get("o").expect("no saved object file in work product"), - ); - - if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { - return Err(format!( - "unable to copy {} to {}: {}", - source_file_regular.display(), - obj_out_regular.display(), - err - )); - } - - let obj_out_global_asm = - tcx.output_filenames(()).temp_path_ext_for_cgu("asm.o", cgu.name().as_str()); - let source_file_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { - let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(tcx.sess, asm_o); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) - { - return Err(format!( - "unable to copy {} to {}: {}", - source_file_global_asm.display(), - obj_out_global_asm.display(), - err - )); - } - Some(source_file_global_asm) - } else { - None - }; - - Ok(ModuleCodegenResult { - module: CompiledModule { - name: cgu.name().to_string(), - kind: ModuleKind::Regular, - object: Some(obj_out_regular), - global_asm_object: source_file_global_asm.as_ref().map(|_| obj_out_global_asm), - dwarf_object: None, - bytecode: None, - assembly: None, - llvm_ir: None, - links_from_incr_cache: if let Some(source_file_global_asm) = source_file_global_asm { - vec![source_file_regular, source_file_global_asm] - } else { - vec![source_file_regular] - }, - }, - existing_work_product: Some((cgu.work_product_id(), work_product)), - }) -} - -fn codegen_cgu_content( - tcx: TyCtxt<'_>, - module: &mut dyn Module, - cgu_name: rustc_span::Symbol, -) -> (Option, Vec, String) { +fn codegen_cgu(tcx: TyCtxt<'_>, cgu_name: Symbol) -> AotModule { let _timer = tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()); let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut debug_context = DebugContext::new(tcx, module.isa(), false, cgu_name.as_str()); - let mut global_asm = String::new(); + let mut module = make_module(tcx, cgu_name.as_str()); let mut type_dbg = TypeDebugContext::default(); - super::predefine_mono_items(tcx, module, &mono_items); - let mut codegened_functions = vec![]; + super::predefine_mono_items(tcx, &mut module.module, &mono_items); for (mono_item, item_data) in mono_items { match mono_item { MonoItem::Fn(instance) => { let flags = tcx.codegen_instance_attrs(instance.def).flags; if flags.contains(CodegenFnAttrFlags::NAKED) { rustc_codegen_ssa::mir::naked_asm::codegen_naked_asm( - &mut GlobalAsmContext { tcx, global_asm: &mut global_asm }, + &mut GlobalAsmContext { tcx, global_asm: &mut module.global_asm }, instance, MonoItemData { linkage: RLinkage::External, @@ -321,186 +175,219 @@ fn codegen_cgu_content( } let codegened_function = crate::base::codegen_fn( tcx, - cgu_name, - debug_context.as_mut(), + cgu.name(), + module.debug_context.as_mut(), &mut type_dbg, Function::new(), - module, + &mut module.module, instance, ); - codegened_functions.push(codegened_function); + module.codegened_functions.push(codegened_function); } MonoItem::Static(def_id) => { - let data_id = crate::constant::codegen_static(tcx, module, def_id); - if let Some(debug_context) = debug_context.as_mut() { + let data_id = crate::constant::codegen_static(tcx, &mut module.module, def_id); + if let Some(debug_context) = module.debug_context.as_mut() { debug_context.define_static(tcx, &mut type_dbg, def_id, data_id); } } MonoItem::GlobalAsm(item_id) => { rustc_codegen_ssa::base::codegen_global_asm( - &mut GlobalAsmContext { tcx, global_asm: &mut global_asm }, + &mut GlobalAsmContext { tcx, global_asm: &mut module.global_asm }, item_id, ); } } } - crate::main_shim::maybe_create_entry_wrapper(tcx, module, false, cgu.is_primary()); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module.module, false, cgu.is_primary()); - (debug_context, codegened_functions, global_asm) + module } -fn module_codegen( - tcx: TyCtxt<'_>, - global_asm_config: Arc, - cgu_name: rustc_span::Symbol, - token: ConcurrencyLimiterToken, -) -> OngoingModuleCodegen { - let mut module = make_module(tcx.sess, cgu_name.as_str().to_string()); +fn compile_cgu( + prof: &SelfProfilerRef, + output_filenames: &OutputFilenames, + should_write_ir: bool, + mut aot_module: AotModule, + cgu_name: String, + kind: ModuleKind, +) -> Result { + prof.generic_activity_with_arg("compile functions", &*cgu_name).run(|| { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + prof.clone(), + ))); + + let mut cached_context = Context::new(); + for codegened_func in aot_module.codegened_functions { + crate::base::compile_fn( + &prof, + &output_filenames, + should_write_ir, + &mut cached_context, + &mut aot_module.module, + aot_module.debug_context.as_mut(), + &mut aot_module.global_asm, + codegened_func, + ); + } + }); - let (mut debug_context, codegened_functions, mut global_asm) = - codegen_cgu_content(tcx, &mut module, cgu_name); + let global_asm_object_file = + prof.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { + if aot_module.global_asm.is_empty() { + return Ok::<_, String>(None); + } - let cgu_name = cgu_name.as_str().to_owned(); + let global_asm_object_file = output_filenames.temp_path_ext_for_cgu("asm.o", &cgu_name); + crate::global_asm::compile_global_asm( + &aot_module.global_asm_config, + aot_module.global_asm, + &global_asm_object_file, + )?; + + Ok(Some(global_asm_object_file)) + })?; + + prof.generic_activity_with_arg("write object file", &*cgu_name).run(|| { + emit_module( + output_filenames, + prof, + aot_module.module, + aot_module.debug_context, + kind, + cgu_name.clone(), + global_asm_object_file, + &aot_module.producer, + ) + }) +} - let producer = crate::debuginfo::producer(tcx.sess); +#[derive(Copy, Clone)] +pub(crate) struct AotDriver; + +impl ExtraBackendMethods for AotDriver { + type Module = AotModule; + + fn codegen_allocator<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + module_name: &str, + methods: &[AllocatorMethod], + ) -> Self::Module { + let mut allocator_module = make_module(tcx, module_name); + crate::allocator::codegen(tcx, &mut allocator_module.module, methods); + allocator_module + } - let profiler = tcx.prof.clone(); - let output_filenames = tcx.output_filenames(()).clone(); - let should_write_ir = crate::pretty_clif::should_write_ir(tcx.sess); - - OngoingModuleCodegen::Async(std::thread::spawn(move || { - profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { - cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( - profiler.clone(), - ))); - - let mut cached_context = Context::new(); - for codegened_func in codegened_functions { - crate::base::compile_fn( - &profiler, - &output_filenames, - should_write_ir, - &mut cached_context, - &mut module, - debug_context.as_mut(), - &mut global_asm, - codegened_func, - ); - } - }); - - let global_asm_object_file = - profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { - crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, global_asm) - })?; - - let codegen_result = - profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { - emit_cgu( - &global_asm_config.output_filenames, - &profiler, - cgu_name, - module, - debug_context, - global_asm_object_file, - &producer, - ) - }); - std::mem::drop(token); - codegen_result - })) -} + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: Symbol, + ) -> (ModuleCodegen, u64) { + let start_time = Instant::now(); + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( + dep_node, + tcx, + || { + let aot_module = codegen_cgu(tcx, cgu_name); + ModuleCodegen::new_regular(cgu_name.as_str().to_owned(), aot_module) + }, + Some(rustc_middle::dep_graph::hash_result), + ); -fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option { - let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string()); - let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module); - - if created_alloc_shim { - let product = allocator_module.finish(); - - match emit_module( - tcx.output_filenames(()), - &tcx.sess.prof, - product.object, - ModuleKind::Allocator, - "allocator_shim".to_owned(), - None, - &crate::debuginfo::producer(tcx.sess), - ) { - Ok(allocator_module) => Some(allocator_module), - Err(err) => tcx.dcx().fatal(err), - } - } else { - None + let time_to_codegen = start_time.elapsed(); + + // We assume that the cost to run LLVM on a CGU is proportional to + // the time we needed for codegenning it. + let cost = time_to_codegen.as_nanos() as u64; + + (module, cost) } } -pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box { - let cgus = tcx.collect_and_partition_mono_items(()).codegen_units; +impl WriteBackendMethods for AotDriver { + type Module = AotModule; - if tcx.dep_graph.is_fully_enabled() { - for cgu in cgus { - tcx.ensure_ok().codegen_unit(cgu.name()); - } + type TargetMachine = (); + + type ModuleBuffer = Infallible; + + type ThinData = Infallible; + + fn target_machine_factory( + &self, + _sess: &Session, + _opt_level: OptLevel, + _target_features: &[String], + ) -> TargetMachineFactoryFn { + Arc::new(|_, _| ()) } - // Calculate the CGU reuse - let cgu_reuse = tcx.sess.time("find_cgu_reuse", || { - cgus.iter().map(|cgu| determine_cgu_reuse(tcx, cgu)).collect::>() - }); + fn optimize_and_codegen_fat_lto( + _sess: &Session, + _cgcx: &CodegenContext, + _shared_emitter: &SharedEmitter, + _tm_factory: TargetMachineFactoryFn, + _exported_symbols_for_lto: &[String], + _each_linked_rlib_for_lto: &[PathBuf], + _modules: Vec>, + ) -> CompiledModule { + unreachable!() + } - rustc_codegen_ssa::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| { - for (i, cgu) in cgus.iter().enumerate() { - let cgu_reuse = cgu_reuse[i]; - cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); - } - }); + fn run_thin_lto( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _dcx: rustc_errors::DiagCtxtHandle<'_>, + _exported_symbols_for_lto: &[String], + _each_linked_rlib_for_lto: &[PathBuf], + _modules: Vec>, + ) -> (Vec>, Vec) { + unreachable!() + } - let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + fn optimize( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _shared_emitter: &SharedEmitter, + _module: &mut ModuleCodegen, + _config: &ModuleConfig, + ) { + } - let disable_incr_cache = disable_incr_cache(); - let (todo_cgus, done_cgus) = - cgus.iter().enumerate().partition::, _>(|&(i, _)| match cgu_reuse[i] { - _ if disable_incr_cache => true, - CguReuse::No => true, - CguReuse::PreLto | CguReuse::PostLto => false, - }); + fn optimize_and_codegen_thin( + _cgcx: &CodegenContext, + _prof: &SelfProfilerRef, + _shared_emitter: &SharedEmitter, + _tm_factory: TargetMachineFactoryFn, + _thin: ThinModule, + ) -> CompiledModule { + unreachable!() + } - let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(todo_cgus.len())); + fn codegen( + cgcx: &CodegenContext, + prof: &SelfProfilerRef, + shared_emitter: &SharedEmitter, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> CompiledModule { + compile_cgu( + prof, + &cgcx.output_filenames, + config.emit_ir, + module.module_llvm, + module.name, + module.kind, + ) + .unwrap_or_else(|err| { + let dcx = DiagCtxt::new(Box::new(shared_emitter.clone())); + dcx.handle().fatal(err) + }) + } - let modules: Vec<_> = - tcx.sess.time("codegen mono items", || { - let modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| { - let dep_node = cgu.codegen_dep_node(tcx); - let (module, _) = tcx.dep_graph.with_task( - dep_node, - tcx, - || { - module_codegen( - tcx, - global_asm_config.clone(), - cgu.name(), - concurrency_limiter.acquire(tcx.dcx()), - ) - }, - Some(rustc_middle::dep_graph::hash_result), - ); - IntoDynSyncSend(module) - }); - modules - .into_iter() - .map(|module| module.0) - .chain(done_cgus.into_iter().map(|(_, cgu)| { - OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu)) - })) - .collect() - }); - - let allocator_module = emit_allocator_module(tcx); - - Box::new(OngoingCodegen { - modules, - allocator_module, - concurrency_limiter: concurrency_limiter.0, - }) + fn serialize_module(_module: Self::Module, _is_thin: bool) -> Self::ModuleBuffer { + unreachable!() + } } diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 33b88d70d6f8d..7d6ece02e4a62 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -6,6 +6,7 @@ use std::os::raw::{c_char, c_int}; use cranelift_jit::{JITBuilder, JITModule}; use rustc_codegen_ssa::CrateInfo; +use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mono::MonoItem; use rustc_session::Session; @@ -28,7 +29,9 @@ fn create_jit_module( let cx = DebugContext::new(tcx, jit_module.isa(), false, "dummy_cgu_name"); - crate::allocator::codegen(tcx, &mut jit_module); + if let Some(kind) = allocator_kind_for_codegen(tcx) { + crate::allocator::codegen(tcx, &mut jit_module, &allocator_shim_contents(tcx, kind)); + } (jit_module, cx) } diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index ee7e6732e6a77..769c008c13f75 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -2,9 +2,8 @@ //! standalone executable. use std::io::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::Arc; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::traits::{AsmCodegenMethods, GlobalAsmOperandRef}; @@ -12,7 +11,7 @@ use rustc_middle::ty::TyCtxt; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers, }; -use rustc_session::config::OutputFilenames; +use rustc_session::Session; use rustc_target::asm::InlineAsmArch; use crate::prelude::*; @@ -163,32 +162,28 @@ fn codegen_global_asm_inner<'tcx>( pub(crate) struct GlobalAsmConfig { assembler: PathBuf, target: String, - pub(crate) output_filenames: Arc, } impl GlobalAsmConfig { - pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + pub(crate) fn new(sess: &Session) -> Self { GlobalAsmConfig { - assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), - target: match &tcx.sess.opts.target_triple { + assembler: crate::toolchain::get_toolchain_binary(sess, "as"), + target: match &sess.opts.target_triple { rustc_target::spec::TargetTuple::TargetTuple(triple) => triple.clone(), rustc_target::spec::TargetTuple::TargetJson { path_for_rustdoc, .. } => { path_for_rustdoc.to_str().unwrap().to_owned() } }, - output_filenames: tcx.output_filenames(()).clone(), } } } pub(crate) fn compile_global_asm( config: &GlobalAsmConfig, - cgu_name: &str, global_asm: String, -) -> Result, String> { - if global_asm.is_empty() { - return Ok(None); - } + global_asm_object_file: &Path, +) -> Result<(), String> { + assert!(!global_asm.is_empty()); // Remove all LLVM style comments let mut global_asm = global_asm @@ -198,8 +193,6 @@ pub(crate) fn compile_global_asm( .join("\n"); global_asm.push('\n'); - let global_asm_object_file = config.output_filenames.temp_path_ext_for_cgu("asm.o", cgu_name); - // Assemble `global_asm` if option_env!("CG_CLIF_FORCE_GNU_AS").is_some() { let mut child = Command::new(&config.assembler) @@ -266,5 +259,5 @@ pub(crate) fn compile_global_asm( } } - Ok(Some(global_asm_object_file)) + Ok(()) } diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 8100d565b3974..f9fc7002be87b 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -424,13 +424,10 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for inout for (i, operand) in self.operands.iter().enumerate() { - match *operand { - CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => { - let slot = new_slot(reg.reg_class()); - slots_input[i] = Some(slot); - slots_output[i] = Some(slot); - } - _ => (), + if let CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } = *operand { + let slot = new_slot(reg.reg_class()); + slots_input[i] = Some(slot); + slots_output[i] = Some(slot); } } @@ -456,11 +453,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> { // Allocate stack slots for output for (i, operand) in self.operands.iter().enumerate() { - match *operand { - CInlineAsmOperand::Out { reg, place: Some(_), .. } => { - slots_output[i] = Some(new_slot(reg.reg_class())); - } - _ => (), + if let CInlineAsmOperand::Out { reg, place: Some(_), .. } = *operand { + slots_output[i] = Some(new_slot(reg.reg_class())); } } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs index 3cd7ebb88f447..9da87a5774e8f 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs @@ -494,6 +494,385 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( }); } */ + "llvm.aarch64.crc32b" + | "llvm.aarch64.crc32h" + | "llvm.aarch64.crc32w" + | "llvm.aarch64.crc32x" + | "llvm.aarch64.crc32cb" + | "llvm.aarch64.crc32ch" + | "llvm.aarch64.crc32cw" + | "llvm.aarch64.crc32cx" => { + // ARM ARM v8-A: CRC32{,C}{B,H,W,X}. + // Backs core::arch::aarch64::__crc32{,c}{b,h,w,d}. + intrinsic_args!(fx, args => (crc, v); intrinsic); + + let crc = crc.load_scalar(fx); + let v = v.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crc32b" => "crc32b w0, w0, w1", + "llvm.aarch64.crc32h" => "crc32h w0, w0, w1", + "llvm.aarch64.crc32w" => "crc32w w0, w0, w1", + "llvm.aarch64.crc32x" => "crc32x w0, w0, x1", + "llvm.aarch64.crc32cb" => "crc32cb w0, w0, w1", + "llvm.aarch64.crc32ch" => "crc32ch w0, w0, w1", + "llvm.aarch64.crc32cw" => "crc32cw w0, w0, w1", + "llvm.aarch64.crc32cx" => "crc32cx w0, w0, x1", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + _late: true, + in_value: crc, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: v, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.aese" | "llvm.aarch64.crypto.aesd" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.aese" => "aese v0.16b, v1.16b", + "llvm.aarch64.crypto.aesd" => "aesd v0.16b, v1.16b", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.aesmc" | "llvm.aarch64.crypto.aesimc" => { + intrinsic_args!(fx, args => (a); intrinsic); + + let a = a.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.aesmc" => "aesmc v0.16b, v0.16b", + "llvm.aarch64.crypto.aesimc" => "aesimc v0.16b, v0.16b", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256h" | "llvm.aarch64.crypto.sha256h2" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + let asm = match intrinsic { + "llvm.aarch64.crypto.sha256h" => "sha256h q0, q1, v2.4s", + "llvm.aarch64.crypto.sha256h2" => "sha256h2 q0, q1, v2.4s", + _ => unreachable!(), + }; + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(asm.into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256su0" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256su0 v0.4s, v1.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.crypto.sha256su1" => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let c = c.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("sha256su1 v0.4s, v1.4s, v2.4s".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v2, + )), + value: c, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.pmull64" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String( + "fmov d0, x0 + fmov d1, x1 + pmull v0.1q, v0.1d, v1.1d" + .into(), + )], + &[ + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + late: true, + place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x0, + )), + value: a, + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::x1, + )), + value: b, + }, + CInlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + late: true, + place: None, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.pmull.v8i16" => { + intrinsic_args!(fx, args => (a, b); intrinsic); + + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String("pmull v0.8h, v0.8b, v1.8b".into())], + &[ + CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v0, + )), + _late: true, + in_value: a, + out_place: Some(ret), + }, + CInlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64( + AArch64InlineAsmReg::v1, + )), + value: b, + }, + ], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + + "llvm.aarch64.neon.sqdmulh.v2i32" + | "llvm.aarch64.neon.sqdmulh.v4i16" + | "llvm.aarch64.neon.sqdmulh.v4i32" + | "llvm.aarch64.neon.sqdmulh.v8i16" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/SQDMULH--vector---Signed-saturating-doubling-multiply-returning-high-half- + intrinsic_args!(fx, args => (a, b); intrinsic); + + // Simplify the "double and shift by esize" into "shift by esize - 1". + // https://github.com/qemu/qemu/blob/81cc5f39aa3042e9c0b2ea772b42a2c8b1488e76/target/arm/tcg/mve_helper.c#L1267-L1283 + let (result_ty, product_ty, shift, max) = match intrinsic { + "llvm.aarch64.neon.sqdmulh.v4i16" | "llvm.aarch64.neon.sqdmulh.v8i16" => { + (types::I16, types::I32, 15, i64::from(i16::MAX)) + } + "llvm.aarch64.neon.sqdmulh.v2i32" | "llvm.aarch64.neon.sqdmulh.v4i32" => { + (types::I32, types::I64, 31, i64::from(i32::MAX)) + } + _ => unreachable!(), + }; + + simd_pair_for_each_lane( + fx, + a, + b, + ret, + &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| { + let a_lane = fx.bcx.ins().sextend(product_ty, a_lane); + let b_lane = fx.bcx.ins().sextend(product_ty, b_lane); + let product = fx.bcx.ins().imul(a_lane, b_lane); + let product = fx.bcx.ins().sshr_imm(product, shift); + let max = fx.bcx.ins().iconst(product_ty, max); + let result = fx.bcx.ins().smin(product, max); + fx.bcx.ins().ireduce(result_ty, result) + }, + ); + } + + "llvm.aarch64.neon.saddlp.v1i64.v2i32" + | "llvm.aarch64.neon.saddlp.v2i32.v4i16" + | "llvm.aarch64.neon.saddlp.v2i64.v4i32" + | "llvm.aarch64.neon.saddlp.v4i16.v8i8" + | "llvm.aarch64.neon.saddlp.v4i32.v8i16" + | "llvm.aarch64.neon.saddlp.v8i16.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/SADDLP--Signed-add-long-pairwise- + intrinsic_args!(fx, args => (a); intrinsic); + + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let wide_ty = fx.clif_type(ret_lane_ty).unwrap(); + + for lane_idx in 0..ret_lane_count { + let base = lane_idx * 2; + let a_lane0 = a.value_lane(fx, base).load_scalar(fx); + let a_lane1 = a.value_lane(fx, base + 1).load_scalar(fx); + let a_lane0 = fx.bcx.ins().sextend(wide_ty, a_lane0); + let a_lane1 = fx.bcx.ins().sextend(wide_ty, a_lane1); + let sum = fx.bcx.ins().iadd(a_lane0, a_lane1); + let res_lane = CValue::by_val(sum, ret_lane_layout); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } + } + + "llvm.aarch64.neon.uaddlp.v1i64.v2i32" + | "llvm.aarch64.neon.uaddlp.v2i32.v4i16" + | "llvm.aarch64.neon.uaddlp.v2i64.v4i32" + | "llvm.aarch64.neon.uaddlp.v4i16.v8i8" + | "llvm.aarch64.neon.uaddlp.v4i32.v8i16" + | "llvm.aarch64.neon.uaddlp.v8i16.v16i8" => { + // https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/UADDLP--Unsigned-add-long-pairwise- + intrinsic_args!(fx, args => (a); intrinsic); + + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + let ret_lane_layout = fx.layout_of(ret_lane_ty); + let wide_ty = fx.clif_type(ret_lane_ty).unwrap(); + + for lane_idx in 0..ret_lane_count { + let base = lane_idx * 2; + let a_lane0 = a.value_lane(fx, base).load_scalar(fx); + let a_lane1 = a.value_lane(fx, base + 1).load_scalar(fx); + let a_lane0 = fx.bcx.ins().uextend(wide_ty, a_lane0); + let a_lane1 = fx.bcx.ins().uextend(wide_ty, a_lane1); + let sum = fx.bcx.ins().iadd(a_lane0, a_lane1); + let res_lane = CValue::by_val(sum, ret_lane_layout); + ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane); + } + } + _ => { fx.tcx.dcx().warn(format!( "unsupported AArch64 llvm intrinsic {}; replacing with trap", diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index b8e1886b2d3c1..5d7e457b8e1ba 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -371,7 +371,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } for i in 0..lane_count { - let ret_lane = ret.place_lane(fx, i.into()); + let ret_lane = ret.place_lane(fx, i); ret_lane.write_cvalue(fx, value); } } diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 48858e20381dd..38cb986034859 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -18,9 +18,7 @@ extern crate rustc_codegen_ssa; extern crate rustc_const_eval; extern crate rustc_data_structures; extern crate rustc_errors; -extern crate rustc_fs_util; extern crate rustc_hir; -extern crate rustc_incremental; extern crate rustc_index; extern crate rustc_log; extern crate rustc_session; @@ -40,7 +38,7 @@ use std::sync::Arc; use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::traits::CodegenBackend; -use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig}; +use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig, back}; use rustc_log::tracing::info; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; @@ -60,7 +58,6 @@ mod codegen_f16_f128; mod codegen_i128; mod common; mod compiler_builtins; -mod concurrency_limiter; mod config; mod constant; mod debuginfo; @@ -133,7 +130,7 @@ impl CodegenBackend for CraneliftCodegenBackend { match sess.lto() { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => { - sess.dcx().warn("LTO is not supported. You may get a linker error.") + sess.dcx().fatal("LTO is not supported by rustc_codegen_cranelift"); } } @@ -152,6 +149,10 @@ impl CodegenBackend for CraneliftCodegenBackend { } } + fn thin_lto_supported(&self) -> bool { + false + } + fn target_config(&self, sess: &Session) -> TargetConfig { // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)] let target_features = match sess.target.arch { @@ -224,7 +225,7 @@ impl CodegenBackend for CraneliftCodegenBackend { #[cfg(not(feature = "jit"))] tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift"); } else { - driver::aot::run_aot(tcx) + Box::new(rustc_codegen_ssa::base::codegen_crate(driver::aot::AotDriver, tcx)) } } @@ -232,10 +233,13 @@ impl CodegenBackend for CraneliftCodegenBackend { &self, ongoing_codegen: Box, sess: &Session, - outputs: &OutputFilenames, - _crate_info: &CrateInfo, + _outputs: &OutputFilenames, + crate_info: &CrateInfo, ) -> (CompiledModules, FxIndexMap) { - ongoing_codegen.downcast::().unwrap().join(sess, outputs) + ongoing_codegen + .downcast::>() + .unwrap() + .join(sess, crate_info) } fn fallback_intrinsics(&self) -> Vec { @@ -254,7 +258,9 @@ fn enable_verifier(sess: &Session) -> bool { } fn target_triple(sess: &Session) -> target_lexicon::Triple { - match sess.target.llvm_target.parse() { + // Use versioned target triple to make `OperatingSystem::MacOSX(...)` + // contain a value, which we use when emitting `LC_BUILD_VERSION`. + match back::versioned_llvm_target(sess).parse() { Ok(triple) => triple, Err(err) => sess.dcx().fatal(format!("target not recognized: {}", err)), } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 1d9dcdab4ced9..a2a2cac3faaa8 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -686,17 +686,14 @@ impl<'tcx> CPlace<'tcx> { ) -> CPlace<'tcx> { let layout = self.layout(); - match self.inner { - CPlaceInner::VarPair(local, var1, var2) => { - let layout = layout.field(&*fx, field.index()); + if let CPlaceInner::VarPair(local, var1, var2) = self.inner { + let layout = layout.field(&*fx, field.index()); - match field.as_u32() { - 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, - 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, - _ => unreachable!("field should be 0 or 1"), - } + match field.as_u32() { + 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, + 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, + _ => unreachable!("field should be 0 or 1"), } - _ => {} } let (base, extra) = match self.inner { diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index ef08286122916..89aae46ec8f23 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -597,6 +597,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "log", "mach2", "memchr", + "memmap2", "object", "proc-macro2", "quote",