From 640c3f730a998f7f5ba82ea4a2363d92064e7bc4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 25 Jul 2022 16:07:57 +0200 Subject: [PATCH 1/5] Merge commit 'c19edfd71a1d0ddef86c2c67fdb40718d40a72b4' into sync_cg_clif-2022-07-25 --- .vscode/settings.json | 7 +- Cargo.lock | 139 ++++++++++++++++++------- Cargo.toml | 14 +-- build_sysroot/Cargo.lock | 17 ++-- build_system/build_sysroot.rs | 2 +- example/mini_core.rs | 6 +- example/mini_core_hello_world.rs | 42 ++++++++ example/std_example.rs | 19 ++++ rust-toolchain | 2 +- scripts/setup_rust_fork.sh | 10 +- scripts/test_rustc_tests.sh | 12 ++- src/abi/mod.rs | 34 ++++++- src/abi/pass_mode.rs | 28 ++--- src/archive.rs | 19 +++- src/base.rs | 39 +++++-- src/cast.rs | 13 +-- src/constant.rs | 23 +++-- src/inline_asm.rs | 170 ++++++++++++++++--------------- src/intrinsics/mod.rs | 22 +++- src/lib.rs | 7 +- src/main_shim.rs | 3 +- src/pretty_clif.rs | 3 +- src/unsize.rs | 6 +- src/value_and_place.rs | 57 +++++++++-- 24 files changed, 474 insertions(+), 220 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ecb20f22d8c92..d88309e412ed0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,9 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], - "rust-analyzer.assist.importGranularity": "module", - "rust-analyzer.assist.importEnforceGranularity": true, - "rust-analyzer.assist.importPrefix": "crate", - "rust-analyzer.cargo.runBuildScripts": true, + "rust-analyzer.imports.granularity.enforce": true, + "rust-analyzer.imports.granularity.group": "module", + "rust-analyzer.imports.prefix": "crate", "rust-analyzer.cargo.features": ["unstable-features"], "rust-analyzer.linkedProjects": [ "./Cargo.toml", diff --git a/Cargo.lock b/Cargo.lock index 2f5d1c0432f55..532049c858d4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,17 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "anyhow" version = "1.0.56" @@ -25,6 +36,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cfg-if" version = "1.0.0" @@ -33,56 +50,57 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed44413e7e2fe3260d0ed73e6956ab188b69c10ee92b892e401e0f4f6808c68b" +checksum = "749d0d6022c9038dccf480bdde2a38d435937335bf2bb0f14e815d94517cdce8" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5d83f0f26bf213f971f45589d17e5b65e4861f9ed22392b0cbb6eaa5bd329c" +checksum = "e94370cc7b37bf652ccd8bb8f09bd900997f7ccf97520edfc75554bb5c4abbea" dependencies = [ "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", "cranelift-entity", + "cranelift-isle", "gimli", "log", - "regalloc", + "regalloc2", "smallvec", "target-lexicon", ] [[package]] name = "cranelift-codegen-meta" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800dc386177df6ecc5a32680607ed8ba1fa0d31a2a59c8c61fbf44826b8191d" +checksum = "e0a3cea8fdab90e44018c5b9a1dfd460d8ee265ac354337150222a354628bdb6" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c961f85070985ebc8fcdb81b838a5cf842294d1e6ed4852446161c7e246fd455" +checksum = "5ac72f76f2698598951ab26d8c96eaa854810e693e7dd52523958b5909fde6b2" [[package]] name = "cranelift-entity" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2347b2b8d1d5429213668f2a8e36c85ee3c73984a2f6a79007e365d3e575e7ed" +checksum = "09eaeacfcd2356fe0e66b295e8f9d59fdd1ac3ace53ba50de14d628ec902f72d" [[package]] name = "cranelift-frontend" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbcdbf7bed29e363568b778649b69dabc3d727256d5d25236096ef693757654" +checksum = "dba69c9980d5ffd62c18a2bde927855fcd7c8dc92f29feaf8636052662cbd99c" dependencies = [ "cranelift-codegen", "log", @@ -90,11 +108,17 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "cranelift-isle" +version = "0.85.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2920dc1e05cac40304456ed3301fde2c09bd6a9b0210bcfa2f101398d628d5b" + [[package]] name = "cranelift-jit" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c769d4e0d76f59c8b2a3bf0477d89ee149bb0731b53fbb245ee081d49063095" +checksum = "1c3c5ed067f2c81577e431f3039148a9c187b33cc79e0d1731fede27d801ec56" dependencies = [ "anyhow", "cranelift-codegen", @@ -110,9 +134,9 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab57d399a2401074bb0cc40b3031e420f3d66d46ec0cf21eeae53ac04bd73e2" +checksum = "eee6784303bf9af235237a4885f7417e09a35df896d38ea969a0081064b3ede4" dependencies = [ "anyhow", "cranelift-codegen", @@ -120,9 +144,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4cdf93552e5ceb2e3c042829ebb4de4378492705f769eadc6a7c6c5251624c" +checksum = "f04dfa45f9b2a6f587c564d6b63388e00cd6589d2df6ea2758cf79e1a13285e6" dependencies = [ "cranelift-codegen", "libc", @@ -131,9 +155,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.83.0" +version = "0.85.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8e65f4839c26e6237fc0744911d79b0a2ac5e76b4e4eebd14db2b8d849fd31" +checksum = "0bf38b2c505db749276793116c0cb30bd096206c7810e471677a453134881881" dependencies = [ "anyhow", "cranelift-codegen", @@ -152,6 +176,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "gimli" version = "0.26.1" @@ -161,6 +205,15 @@ dependencies = [ "indexmap", ] +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -174,14 +227,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", ] [[package]] name = "libc" -version = "0.2.119" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "libloading" @@ -219,11 +272,12 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "object" -version = "0.27.1" +version = "0.28.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "crc32fast", + "hashbrown 0.11.2", "indexmap", "memchr", ] @@ -235,13 +289,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] -name = "regalloc" -version = "0.0.34" +name = "regalloc2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62446b1d3ebf980bdc68837700af1d77b37bc430e524bf95319c6eada2a4cc02" +checksum = "4a8d23b35d7177df3b9d31ed8a9ab4bf625c668be77a319d4f5efd4a5257701c" dependencies = [ + "fxhash", "log", - "rustc-hash", + "slice-group-by", "smallvec", ] @@ -257,12 +312,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc_codegen_cranelift" version = "0.1.0" @@ -283,6 +332,12 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "slice-group-by" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" + [[package]] name = "smallvec" version = "1.8.1" @@ -295,6 +350,18 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7fa7e55043acb85fca6b3c01485a2eeb6b69c5d21002e273c79e465f43b7ac1" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index ff71d7a209e4d..61e977e3e69bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,15 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.83.0", features = ["unwind", "all-arch"] } -cranelift-frontend = "0.83.0" -cranelift-module = "0.83.0" -cranelift-native = "0.83.0" -cranelift-jit = { version = "0.83.0", optional = true } -cranelift-object = "0.83.0" +cranelift-codegen = { version = "0.85.3", features = ["unwind", "all-arch"] } +cranelift-frontend = "0.85.3" +cranelift-module = "0.85.3" +cranelift-native = "0.85.3" +cranelift-jit = { version = "0.85.3", optional = true } +cranelift-object = "0.85.3" target-lexicon = "0.12.0" gimli = { version = "0.26.0", default-features = false, features = ["write"]} -object = { version = "0.27.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } +object = { version = "0.28.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.9.1" diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index efee6ef3f3780..7b2cdd273366f 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -56,9 +56,9 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.72" +version = "0.1.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afdbb35d279238cf77f0c9e8d90ad50d6c7bff476ab342baafa29440f0f10bff" +checksum = "c6e3183e88f659a862835db8f4b67dbeed3d93e44dd4927eef78edb1c149d784" dependencies = [ "rustc-std-workspace-core", ] @@ -112,9 +112,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -123,20 +123,21 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.0" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab7905ea95c6d9af62940f9d7dd9596d54c334ae2c15300c482051292d5637f" +checksum = "7668753748e445859e4e373c3d41117235d9feed578392f5a3a73efdc751ca4a" dependencies = [ "compiler_builtins", "libc", + "rustc-std-workspace-alloc", "rustc-std-workspace-core", ] [[package]] name = "libc" -version = "0.2.125" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" dependencies = [ "rustc-std-workspace-core", ] diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 8682204f4fd30..16cce83dd9c85 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -205,7 +205,7 @@ fn build_clif_sysroot_for_triple( { let entry = entry.unwrap(); if let Some(ext) = entry.path().extension() { - if ext == "rmeta" || ext == "d" || ext == "dSYM" { + if ext == "rmeta" || ext == "d" || ext == "dSYM" || ext == "clif" { continue; } } else { diff --git a/example/mini_core.rs b/example/mini_core.rs index 489259d1a6bc6..8b6042a3d6638 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -458,7 +458,7 @@ pub trait FnMut: FnOnce { #[lang = "panic"] #[track_caller] -pub fn panic(_msg: &str) -> ! { +pub fn panic(_msg: &'static str) -> ! { unsafe { libc::puts("Panicking\n\0" as *const str as *const i8); intrinsics::abort(); @@ -497,7 +497,7 @@ pub trait Deref { #[repr(transparent)] #[rustc_layout_scalar_valid_range_start(1)] #[rustc_nonnull_optimization_guaranteed] -pub struct NonNull(pub *mut T); +pub struct NonNull(pub *const T); impl CoerceUnsized> for NonNull where T: Unsize {} impl DispatchFromDyn> for NonNull where T: Unsize {} @@ -521,7 +521,7 @@ impl Drop for Box { } } -impl Deref for Box { +impl Deref for Box { type Target = T; fn deref(&self) -> &Self::Target { diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 0f1245c2758ed..aa1f239bae23e 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -124,6 +124,23 @@ fn call_return_u128_pair() { return_u128_pair(); } +#[repr(C)] +pub struct bool_11 { + field0: bool, + field1: bool, + field2: bool, + field3: bool, + field4: bool, + field5: bool, + field6: bool, + field7: bool, + field8: bool, + field9: bool, + field10: bool, +} + +extern "C" fn bool_struct_in_11(arg0: bool_11) {} + #[allow(unreachable_code)] // FIXME false positive fn main() { take_unique(Unique { @@ -134,6 +151,20 @@ fn main() { call_return_u128_pair(); + bool_struct_in_11(bool_11 { + field0: true, + field1: true, + field2: true, + field3: true, + field4: true, + field5: true, + field6: true, + field7: true, + field8: true, + field9: true, + field10: true, + }); + let slice = &[0, 1] as &[i32]; let slice_ptr = slice as *const [i32] as *const i32; @@ -299,6 +330,17 @@ fn main() { static REF1: &u8 = &42; static REF2: &u8 = REF1; assert_eq!(*REF1, *REF2); + + extern "C" { + type A; + } + + fn main() { + let x: &A = unsafe { &*(1usize as *const A) }; + + assert_eq!(unsafe { intrinsics::size_of_val(x) }, 0); + assert_eq!(unsafe { intrinsics::min_align_of_val(x) }, 1); +} } #[cfg(all(not(jit), target_arch = "x86_64", target_os = "linux"))] diff --git a/example/std_example.rs b/example/std_example.rs index 0a2bce2621d96..0b5b6cd55d720 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -128,6 +128,25 @@ fn main() { 0 => loop {}, v => panic(v), }; + + if black_box(false) { + // Based on https://github.com/rust-lang/rust/blob/2f320a224e827b400be25966755a621779f797cc/src/test/ui/debuginfo/debuginfo_with_uninhabitable_field_and_unsized.rs + let _ = Foo::::new(); + + #[allow(dead_code)] + struct Foo { + base: Never, + value: T, + } + + impl Foo { + pub fn new() -> Box> { + todo!() + } + } + + enum Never {} + } } fn panic(_: u128) { diff --git a/rust-toolchain b/rust-toolchain index e98e92e468e93..3ab395d89d50e 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2022-05-15" +channel = "nightly-2022-07-25" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index 4d0dfa16c5ecf..091bfa1e9926f 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -29,14 +29,15 @@ diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/ru index 8431aa7b818..a3ff7e68ce5 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs -@@ -3489,11 +3489,7 @@ fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> S - .join("library"); - normalize_path(&src_dir, "$(echo '$SRC_DIR')"); +@@ -3489,12 +3489,7 @@ fn normalize_output(&self, output: &str, custom_rules: &[(String, String)]) -> S + let compiler_src_dir = base_dir.join("compiler"); + normalize_path(&compiler_src_dir, "$(echo '$COMPILER_DIR')"); - if let Some(virtual_rust_source_base_dir) = - option_env!("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR").map(PathBuf::from) - { - normalize_path(&virtual_rust_source_base_dir.join("library"), "$(echo '$SRC_DIR')"); +- normalize_path(&virtual_rust_source_base_dir.join("compiler"), "$(echo '$COMPILER_DIR')"); - } + normalize_path(&Path::new("$(cd ../build_sysroot/sysroot_src/library; pwd)"), "$(echo '$SRC_DIR')"); @@ -62,3 +63,6 @@ deny-warnings = false verbose-tests = false EOF popd + +# FIXME remove once inline asm is fully supported +export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc" diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 9bdb9f22c549a..944787612d8bc 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -33,6 +33,7 @@ rm src/test/ui/test-attrs/test-fn-signature-verification-for-explicit-return-typ rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same rm src/test/ui/generator/size-moved-locals.rs # same +rm -r src/test/ui/macros/rfc-2011-nicer-assert-messages/ # vendor intrinsics rm src/test/ui/sse2.rs # cpuid not supported, so sse2 not detected @@ -65,11 +66,13 @@ rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and n rm src/test/ui/target-feature/missing-plusminus.rs # error not implemented rm src/test/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment rm -r src/test/run-make/emit-named-files # requires full --emit support +rm src/test/ui/abi/stack-probes.rs # stack probes not yet implemented # optimization tests # ================== -rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations +rm src/test/ui/codegen/issue-28950.rs # depends on stack size optimizations rm src/test/ui/codegen/init-large-type.rs # same +rm src/test/ui/issues/issue-40883.rs # same rm -r src/test/run-make/fmt-write-bloat/ # tests an optimization # backend specific tests @@ -89,14 +92,13 @@ rm src/test/ui/consts/issue-33537.rs # same rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ rm -r src/test/run-make/unstable-flag-required # same rm -r src/test/run-make/rustdoc-* # same +rm -r src/test/run-make/issue-88756-default-output # same +rm -r src/test/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump # genuine bugs # ============ rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition -rm -r src/test/ui/polymorphization/ # polymorphization not yet supported -rm src/test/codegen-units/polymorphization/unused_type_parameters.rs # same - rm src/test/incremental/spike-neg1.rs # errors out for some reason rm src/test/incremental/spike-neg2.rs # same rm src/test/ui/issues/issue-74564-if-expr-stack-overflow.rs # gives a stackoverflow before the backend runs @@ -111,6 +113,8 @@ rm src/test/ui/backtrace.rs # TODO warning rm src/test/ui/empty_global_asm.rs # TODO add needs-asm-support rm src/test/ui/simple_global_asm.rs # TODO add needs-asm-support rm src/test/ui/test-attrs/test-type.rs # TODO panic message on stderr. correct stdout +# not sure if this is actually a bug in the test suite, but the symbol list shows the function without leading _ for some reason +rm -r src/test/run-make/native-link-modifier-bundle echo "[TEST] rustc test suite" RUST_TEST_NOCAPTURE=1 COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0 src/test/{codegen-units,run-make,run-pass-valgrind,ui,incremental} diff --git a/src/abi/mod.rs b/src/abi/mod.rs index ffa5d747b1160..815450f689e4a 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -4,6 +4,7 @@ mod comments; mod pass_mode; mod returning; +use cranelift_module::ModuleError; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::layout::FnAbiOf; use rustc_target::abi::call::{Conv, FnAbi}; @@ -69,7 +70,17 @@ pub(crate) fn import_function<'tcx>( ) -> FuncId { let name = tcx.symbol_name(inst).name; let sig = get_function_sig(tcx, module.isa().triple(), inst); - module.declare_function(name, Linkage::Import, &sig).unwrap() + match module.declare_function(name, Linkage::Import, &sig) { + Ok(func_id) => func_id, + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( + "attempt to declare `{name}` as function, but it was already declared as static" + )), + Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.sess.fatal(&format!( + "attempt to declare `{name}` with signature {new_sig:?}, \ + but it was already declared with signature {prev_sig:?}" + )), + Err(err) => Err::<_, _>(err).unwrap(), + } } impl<'tcx> FunctionCx<'_, '_, 'tcx> { @@ -182,6 +193,15 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ } let fn_abi = fx.fn_abi.take().unwrap(); + + // FIXME implement variadics in cranelift + if fn_abi.c_variadic { + fx.tcx.sess.span_fatal( + fx.mir.span, + "Defining variadic functions is not yet supported by Cranelift", + ); + } + let mut arg_abis_iter = fn_abi.args.iter(); let func_params = fx @@ -376,9 +396,15 @@ pub(crate) fn codegen_terminator_call<'tcx>( RevealAllLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_ty.fn_sig(fx.tcx), extra_args) }; - let is_cold = instance - .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)) - .unwrap_or(false); + let is_cold = if fn_sig.abi == Abi::RustCold { + true + } else { + instance + .map(|inst| { + fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) + }) + .unwrap_or(false) + }; if is_cold { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); if let Some(destination_block) = target { diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 9f0bd31e95fcc..6c10baa53d415 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -18,9 +18,9 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { let clif_ty = match (reg.kind, reg.size.bytes()) { (RegKind::Integer, 1) => types::I8, (RegKind::Integer, 2) => types::I16, - (RegKind::Integer, 4) => types::I32, - (RegKind::Integer, 8) => types::I64, - (RegKind::Integer, 16) => types::I128, + (RegKind::Integer, 3..=4) => types::I32, + (RegKind::Integer, 5..=8) => types::I64, + (RegKind::Integer, 9..=16) => types::I128, (RegKind::Float, 4) => types::F32, (RegKind::Float, 8) => types::F64, (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), @@ -48,23 +48,9 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { ) }; - if cast.prefix.iter().all(|x| x.is_none()) { - // Simplify to a single unit when there is no prefix and size <= unit size - if cast.rest.total <= cast.rest.unit.size { - let clif_ty = match (cast.rest.unit.kind, cast.rest.unit.size.bytes()) { - (RegKind::Integer, 1) => types::I8, - (RegKind::Integer, 2) => types::I16, - (RegKind::Integer, 3..=4) => types::I32, - (RegKind::Integer, 5..=8) => types::I64, - (RegKind::Integer, 9..=16) => types::I128, - (RegKind::Float, 4) => types::F32, - (RegKind::Float, 8) => types::F64, - (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), - _ => unreachable!("{:?}", cast.rest.unit), - }; - return smallvec![AbiParam::new(clif_ty)]; - } - } + // Note: Unlike the LLVM equivalent of this code we don't have separate branches for when there + // is no prefix as a single unit, an array and a heterogeneous struct are not represented using + // different types in Cranelift IR. Instead a single array of primitive types is used. // Create list of fields in the main structure let mut args = cast @@ -230,7 +216,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>( arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, is_owned: bool, ) -> SmallVec<[Value; 2]> { - assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); + assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty, 16); match arg_abi.mode { PassMode::Ignore => smallvec![], PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], diff --git a/src/archive.rs b/src/archive.rs index 0812f930b5dea..7b41620c1d36e 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -86,7 +86,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { let mut entries = Vec::new(); - for (entry_name, entry) in self.entries { + for (mut entry_name, entry) in self.entries { // FIXME only read the symbol table of the object files to avoid having to keep all // object files in memory at once, or read them twice. let data = match entry { @@ -109,6 +109,23 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> { }; if !self.no_builtin_ranlib { + if symbol_table.contains_key(&entry_name) { + // The ar crate can't handle creating a symbol table in case of multiple archive + // members with the same name. Work around this by prepending a number until we + // get a unique name. + for i in 1.. { + let new_name = format!("{}_", i) + .into_bytes() + .into_iter() + .chain(entry_name.iter().copied()) + .collect::>(); + if !symbol_table.contains_key(&new_name) { + entry_name = new_name; + break; + } + } + } + match object::File::parse(&*data) { Ok(object) => { symbol_table.insert( diff --git a/src/base.rs b/src/base.rs index 63cd4d6de4c3e..122e103ff62bc 100644 --- a/src/base.rs +++ b/src/base.rs @@ -175,10 +175,37 @@ fn compile_fn<'tcx>( ); }); + #[cfg(any())] // This is never true + let _clif_guard = { + use std::fmt::Write; + + let func_clone = context.func.clone(); + let clif_comments_clone = clif_comments.clone(); + let mut clif = String::new(); + for flag in module.isa().flags().iter() { + writeln!(clif, "set {}", flag).unwrap(); + } + write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap(); + for isa_flag in module.isa().isa_flags().iter() { + write!(clif, " {}", isa_flag).unwrap(); + } + writeln!(clif, "\n").unwrap(); + crate::PrintOnPanic(move || { + let mut clif = clif.clone(); + ::cranelift_codegen::write::decorate_function( + &mut &clif_comments_clone, + &mut clif, + &func_clone, + ) + .unwrap(); + clif + }) + }; + // Define function tcx.sess.time("define function", || { context.want_disasm = crate::pretty_clif::should_write_ir(tcx); - module.define_function(func_id, context).unwrap() + module.define_function(func_id, context).unwrap(); }); // Write optimized function to file for debugging @@ -815,15 +842,7 @@ pub(crate) fn codegen_place<'tcx>( for elem in place.projection { match elem { PlaceElem::Deref => { - if cplace.layout().ty.is_box() { - cplace = cplace - .place_field(fx, Field::new(0)) // Box -> Unique - .place_field(fx, Field::new(0)) // Unique -> NonNull - .place_field(fx, Field::new(0)) // NonNull -> *mut T - .place_deref(fx); - } else { - cplace = cplace.place_deref(fx); - } + cplace = cplace.place_deref(fx); } PlaceElem::Field(field, _ty) => { cplace = cplace.place_field(fx, field); diff --git a/src/cast.rs b/src/cast.rs index b24e49e94c91e..bad5d1f08a9cf 100644 --- a/src/cast.rs +++ b/src/cast.rs @@ -149,17 +149,8 @@ pub(crate) fn clif_int_or_float_cast( } let is_not_nan = fx.bcx.ins().fcmp(FloatCC::Equal, from, from); - if to_ty == types::I128 { - // FIXME(bytecodealliance/wasmtime#3963): select.i128 on fcmp eq miscompiles - let (lsb, msb) = fx.bcx.ins().isplit(val); - let zero = fx.bcx.ins().iconst(types::I64, 0); - let lsb = fx.bcx.ins().select(is_not_nan, lsb, zero); - let msb = fx.bcx.ins().select(is_not_nan, msb, zero); - fx.bcx.ins().iconcat(lsb, msb) - } else { - let zero = fx.bcx.ins().iconst(to_ty, 0); - fx.bcx.ins().select(is_not_nan, val, zero) - } + let zero = fx.bcx.ins().iconst(to_ty, 0); + fx.bcx.ins().select(is_not_nan, val, zero) } else if from_ty.is_float() && to_ty.is_float() { // float -> float match (from_ty, to_ty) { diff --git a/src/constant.rs b/src/constant.rs index 94a2fb2fbddc2..7f7fd0e9c579d 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -328,14 +328,18 @@ fn data_id_for_static( let attrs = tcx.codegen_fn_attrs(def_id); - let data_id = module - .declare_data( - &*symbol_name, - linkage, - is_mutable, - attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), - ) - .unwrap(); + let data_id = match module.declare_data( + &*symbol_name, + linkage, + is_mutable, + attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), + ) { + Ok(data_id) => data_id, + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( + "attempt to declare `{symbol_name}` as static, but it was already declared as function" + )), + Err(err) => Err::<_, _>(err).unwrap(), + }; if rlinkage.is_some() { // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141 @@ -441,7 +445,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant let data_id = match reloc_target_alloc { GlobalAlloc::Function(instance) => { assert_eq!(addend, 0); - let func_id = crate::abi::import_function(tcx, module, instance); + let func_id = + crate::abi::import_function(tcx, module, instance.polymorphize(tcx)); let local_func_id = module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr(offset.bytes() as u32, local_func_id); continue; diff --git a/src/inline_asm.rs b/src/inline_asm.rs index deac5dfd3ec1a..241de5e36530c 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -18,86 +18,96 @@ pub(crate) fn codegen_inline_asm<'tcx>( ) { // FIXME add .eh_frame unwind info directives - if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) { - let true_ = fx.bcx.ins().iconst(types::I32, 1); - fx.bcx.ins().trapnz(true_, TrapCode::User(1)); - return; - } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) - && matches!( - template[1], - InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } - ) - && template[2] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string()) - && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) - && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string()) - && matches!( - template[6], - InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: Some('r'), span: _ } - ) - { - assert_eq!(operands.len(), 4); - let (leaf, eax_place) = match operands[1] { - InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)) - ); - ( - crate::base::codegen_operand(fx, in_value).load_scalar(fx), - crate::base::codegen_place(fx, out_place.unwrap()), - ) - } - _ => unreachable!(), - }; - let ebx_place = match operands[0] { - InlineAsmOperand::Out { reg, late: true, place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( - X86InlineAsmRegClass::reg - )) - ); - crate::base::codegen_place(fx, place.unwrap()) - } - _ => unreachable!(), - }; - let (sub_leaf, ecx_place) = match operands[2] { - InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)) - ); - ( - crate::base::codegen_operand(fx, in_value).load_scalar(fx), - crate::base::codegen_place(fx, out_place.unwrap()), - ) - } - _ => unreachable!(), - }; - let edx_place = match operands[3] { - InlineAsmOperand::Out { reg, late: true, place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)) - ); - crate::base::codegen_place(fx, place.unwrap()) - } - _ => unreachable!(), - }; - - let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); - - eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); - ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); - ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); - edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); - return; - } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { - // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); - } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { - crate::trap::trap_unimplemented(fx, "Alloca is not supported"); + if !template.is_empty() { + if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) { + let true_ = fx.bcx.ins().iconst(types::I32, 1); + fx.bcx.ins().trapnz(true_, TrapCode::User(1)); + return; + } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) + && matches!( + template[1], + InlineAsmTemplatePiece::Placeholder { + operand_idx: 0, + modifier: Some('r'), + span: _ + } + ) + && template[2] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[3] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string()) + && matches!( + template[6], + InlineAsmTemplatePiece::Placeholder { + operand_idx: 0, + modifier: Some('r'), + span: _ + } + ) + { + assert_eq!(operands.len(), 4); + let (leaf, eax_place) = match operands[1] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + assert_eq!( + reg, + InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)) + ); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let ebx_place = match operands[0] { + InlineAsmOperand::Out { reg, late: true, place } => { + assert_eq!( + reg, + InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( + X86InlineAsmRegClass::reg + )) + ); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + let (sub_leaf, ecx_place) = match operands[2] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + assert_eq!( + reg, + InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)) + ); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let edx_place = match operands[3] { + InlineAsmOperand::Out { reg, late: true, place } => { + assert_eq!( + reg, + InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)) + ); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + + let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); + + eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + return; + } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); + } } let mut inputs = Vec::new(); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index d5a79e254a891..8d8db1da58183 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -404,7 +404,9 @@ fn codegen_regular_intrinsic_call<'tcx>( }; size_of_val, (c ptr) { let layout = fx.layout_of(substs.type_at(0)); - let size = if layout.is_unsized() { + // Note: Can't use is_unsized here as truly unsized types need to take the fixed size + // branch + let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi { let (_ptr, info) = ptr.load_scalar_pair(fx); let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info); size @@ -418,7 +420,9 @@ fn codegen_regular_intrinsic_call<'tcx>( }; min_align_of_val, (c ptr) { let layout = fx.layout_of(substs.type_at(0)); - let align = if layout.is_unsized() { + // Note: Can't use is_unsized here as truly unsized types need to take the fixed size + // branch + let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi { let (_ptr, info) = ptr.load_scalar_pair(fx); let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info); align @@ -1145,6 +1149,20 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME implement black_box semantics ret.write_cvalue(fx, a); }; + + // FIXME implement variadics in cranelift + va_copy, (o _dest, o _src) { + fx.tcx.sess.span_fatal( + source_info.span, + "Defining variadic functions is not yet supported by Cranelift", + ); + }; + va_arg | va_end, (o _valist) { + fx.tcx.sess.span_fatal( + source_info.span, + "Defining variadic functions is not yet supported by Cranelift", + ); + }; } let ret_block = fx.get_block(destination.unwrap()); diff --git a/src/lib.rs b/src/lib.rs index 3ed3453c6c7b3..568bb20a3f4a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -141,7 +141,11 @@ impl<'tcx> CodegenCx<'tcx> { let unwind_context = UnwindContext::new(isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); - let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; + let debug_context = if debug_info && !tcx.sess.target.options.is_like_windows { + Some(DebugContext::new(tcx, isa)) + } else { + None + }; CodegenCx { tcx, global_asm: String::new(), @@ -243,6 +247,7 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box "elf_gd", diff --git a/src/main_shim.rs b/src/main_shim.rs index 2f71a70a44946..c67b6e98b32c7 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -109,7 +109,8 @@ pub(crate) fn maybe_create_entry_wrapper( tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()), ) .unwrap() - .unwrap(); + .unwrap() + .polymorphize(tcx); let report_name = tcx.symbol_name(report).name; let report_sig = get_function_sig(tcx, m.isa().triple(), report); diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index ca7116b887d5a..1d1ec21680e30 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -66,7 +66,7 @@ use rustc_session::config::OutputType; use crate::prelude::*; -#[derive(Debug)] +#[derive(Clone, Debug)] pub(crate) struct CommentWriter { enabled: bool, global_comments: Vec, @@ -237,6 +237,7 @@ pub(crate) fn write_clif_file<'tcx>( func: &cranelift_codegen::ir::Function, mut clif_comments: &CommentWriter, ) { + // FIXME work around filename too long errors write_ir_file( tcx, || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), diff --git a/src/unsize.rs b/src/unsize.rs index fd63c3ecddbdf..052ca0a082b3c 100644 --- a/src/unsize.rs +++ b/src/unsize.rs @@ -153,11 +153,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>( layout: TyAndLayout<'tcx>, info: Value, ) -> (Value, Value) { - if !layout.is_unsized() { - let size = fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64); - let align = fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64); - return (size, align); - } + assert!(layout.is_unsized() || layout.abi == Abi::Uninhabited); match layout.ty.kind() { ty::Dynamic(..) => { // load size/align from vtable diff --git a/src/value_and_place.rs b/src/value_and_place.rs index a68225de58b32..45ae2bd8f07cb 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -324,6 +324,12 @@ impl<'tcx> CPlace<'tcx> { }; } + if layout.size.bytes() >= u64::from(u32::MAX - 16) { + fx.tcx + .sess + .fatal(&format!("values of type {} are too big to store on the stack", layout.ty)); + } + let stack_slot = fx.bcx.create_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to @@ -420,7 +426,7 @@ impl<'tcx> CPlace<'tcx> { } pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) { - assert_assignable(fx, from.layout().ty, self.layout().ty); + assert_assignable(fx, from.layout().ty, self.layout().ty, 16); self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); } @@ -774,18 +780,25 @@ pub(crate) fn assert_assignable<'tcx>( fx: &FunctionCx<'_, '_, 'tcx>, from_ty: Ty<'tcx>, to_ty: Ty<'tcx>, + limit: usize, ) { + if limit == 0 { + // assert_assignable exists solely to catch bugs in cg_clif. it isn't necessary for + // soundness. don't attempt to check deep types to avoid exponential behavior in certain + // cases. + return; + } match (from_ty.kind(), to_ty.kind()) { (ty::Ref(_, a, _), ty::Ref(_, b, _)) | ( ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), ) => { - assert_assignable(fx, *a, *b); + assert_assignable(fx, *a, *b, limit - 1); } (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ })) | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => { - assert_assignable(fx, *a, *b); + assert_assignable(fx, *a, *b, limit - 1); } (ty::FnPtr(_), ty::FnPtr(_)) => { let from_sig = fx.tcx.normalize_erasing_late_bound_regions( @@ -815,6 +828,17 @@ pub(crate) fn assert_assignable<'tcx>( } // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed } + (&ty::Tuple(types_a), &ty::Tuple(types_b)) => { + let mut types_a = types_a.iter(); + let mut types_b = types_b.iter(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b)) if adt_def_a.did() == adt_def_b.did() => { @@ -822,18 +846,37 @@ pub(crate) fn assert_assignable<'tcx>( let mut types_b = substs_b.types(); loop { match (types_a.next(), types_b.next()) { - (Some(a), Some(b)) => assert_assignable(fx, a, b), + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), (None, None) => return, (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), } } } - (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b), + (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b, limit - 1), + (&ty::Closure(def_id_a, substs_a), &ty::Closure(def_id_b, substs_b)) + if def_id_a == def_id_b => + { + let mut types_a = substs_a.types(); + let mut types_b = substs_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } + (ty::Param(_), _) | (_, ty::Param(_)) if fx.tcx.sess.opts.unstable_opts.polymorphize => { + // No way to check if it is correct or not with polymorphization enabled + } _ => { assert_eq!( - from_ty, to_ty, + from_ty, + to_ty, "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", - from_ty, to_ty, fx, + from_ty.kind(), + to_ty.kind(), + fx, ); } } From e151964a70d64b5f58f436f49cae82c22222ee64 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:45:00 +0000 Subject: [PATCH 2/5] Start moving away from the intrinsic_match macro It isn't clear most people other than me and it blocks formatting by rustfmt. --- src/intrinsics/llvm.rs | 137 ++++++++------ src/intrinsics/mod.rs | 14 +- src/intrinsics/simd.rs | 399 ++++++++++++++++++++++++++--------------- 3 files changed, 346 insertions(+), 204 deletions(-) diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 77ac46540a9ba..a987e20b11a77 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -13,15 +13,11 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( ret: CPlace<'tcx>, target: Option, ) { - intrinsic_match! { - fx, intrinsic, args, - _ => { - fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); - crate::trap::trap_unimplemented(fx, intrinsic); - }; - + match intrinsic { // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` - "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) { + "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => { + intrinsic_args!(fx, args => (a); intrinsic); + let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); @@ -29,7 +25,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let mut res = fx.bcx.ins().iconst(types::I32, 0); for lane in (0..lane_count).rev() { - let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx); + let a_lane = + a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx); // cast float to int let a_lane = match lane_ty { @@ -49,18 +46,33 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); - }; - "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) { - let kind = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); - let flt_cc = match kind.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) { + } + "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => { + let (x, y, kind) = match args { + [x, y, kind] => (x, y, kind), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let x = codegen_operand(fx, x); + let y = codegen_operand(fx, y); + let kind = crate::constant::mir_operand_get_const_val(fx, kind) + .expect("llvm.x86.sse2.cmp.* kind not const"); + + let flt_cc = match kind + .try_to_bits(Size::from_bytes(1)) + .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) + { 0 => FloatCC::Equal, 1 => FloatCC::LessThan, 2 => FloatCC::LessThanOrEqual, 7 => { - unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`."); + unimplemented!( + "Compares corresponding elements in `a` and `b` to see if neither is `NaN`." + ); } 3 => { - unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`."); + unimplemented!( + "Compares corresponding elements in `a` and `b` to see if either is `NaN`." + ); } 4 => FloatCC::NotEqual, 5 => { @@ -79,50 +91,67 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) }); - }; - "llvm.x86.sse2.psrli.d", (c a, o imm8) { - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { - match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { - imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - } + } + "llvm.x86.sse2.psrli.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrli.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), }); - }; - "llvm.x86.sse2.pslli.d", (c a, o imm8) { - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| { - match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { - imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - } + } + "llvm.x86.sse2.pslli.d" => { + let (a, imm8) = match args { + [a, imm8] => (a, imm8), + _ => bug!("wrong number of args for intrinsic {intrinsic}"), + }; + let a = codegen_operand(fx, a); + let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) + .expect("llvm.x86.sse2.psrli.d imm8 not const"); + + simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 + .try_to_bits(Size::from_bytes(4)) + .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) + { + imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), + _ => fx.bcx.ins().iconst(types::I32, 0), }); - }; - "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) { + } + "llvm.x86.sse2.storeu.dq" => { + intrinsic_args!(fx, args => (mem_addr, a); intrinsic); + let mem_addr = mem_addr.load_scalar(fx); + // FIXME correctly handle the unalignment let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); - }; - "llvm.x86.addcarry.64", (v c_in, c a, c b) { - llvm_add_sub( - fx, - BinOp::Add, - ret, - c_in, - a, - b - ); - }; - "llvm.x86.subborrow.64", (v b_in, c a, c b) { - llvm_add_sub( - fx, - BinOp::Sub, - ret, - b_in, - a, - b - ); - }; + } + "llvm.x86.addcarry.64" => { + intrinsic_args!(fx, args => (c_in, a, b); intrinsic); + let c_in = c_in.load_scalar(fx); + + llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b); + } + "llvm.x86.subborrow.64" => { + intrinsic_args!(fx, args => (b_in, a, b); intrinsic); + let b_in = b_in.load_scalar(fx); + + llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b); + } + _ => { + fx.tcx + .sess + .warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); + crate::trap::trap_unimplemented(fx, intrinsic); + } } let dest = target.expect("all llvm intrinsics used by stdlib should return"); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 8d8db1da58183..de7695041d360 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -11,9 +11,6 @@ macro_rules! intrinsic_pat { (kw.$name:ident) => { kw::$name }; - ($name:literal) => { - $name - }; } macro_rules! intrinsic_arg { @@ -26,6 +23,17 @@ macro_rules! intrinsic_arg { }; } +macro_rules! intrinsic_args { + ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => { + #[allow(unused_parens)] + let ($($arg),*) = if let [$($arg),*] = $args { + ($(codegen_operand($fx, $arg)),*) + } else { + bug!("wrong number of args for intrinsic {}", $intrinsic); + }; + } +} + macro_rules! intrinsic_match { ($fx:expr, $intrinsic:expr, $args:expr, _ => $unknown:block; diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index d1ca9edf2e0f1..30e3d112594a6 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -25,13 +25,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ret: CPlace<'tcx>, span: Span, ) { - intrinsic_match! { - fx, intrinsic, args, - _ => { - fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); - }; + match intrinsic { + sym::simd_cast => { + intrinsic_args!(fx, args => (a); intrinsic); - simd_cast, (c a) { if !a.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); return; @@ -45,9 +42,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed) }); - }; + } + + sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => { + intrinsic_args!(fx, args => (x, y); intrinsic); - simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) { if !x.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); return; @@ -57,7 +56,9 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| { let res_lane = match (lane_ty.kind(), intrinsic) { (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane), - (ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane), + (ty::Uint(_), sym::simd_ne) => { + fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane) + } (ty::Uint(_), sym::simd_lt) => { fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane) } @@ -72,8 +73,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane), - (ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane), - (ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane), + (ty::Int(_), sym::simd_ne) => { + fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane) + } + (ty::Int(_), sym::simd_lt) => { + fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane) + } (ty::Int(_), sym::simd_le) => { fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane) } @@ -84,13 +89,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane) } - (ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane), - (ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane), - (ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane), + (ty::Float(_), sym::simd_eq) => { + fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane) + } + (ty::Float(_), sym::simd_ne) => { + fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane) + } + (ty::Float(_), sym::simd_lt) => { + fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane) + } (ty::Float(_), sym::simd_le) => { fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane) } - (ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane), + (ty::Float(_), sym::simd_gt) => { + fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane) + } (ty::Float(_), sym::simd_ge) => { fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane) } @@ -103,10 +116,19 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let res_lane = fx.bcx.ins().bint(ty, res_lane); fx.bcx.ins().ineg(res_lane) }); - }; + } // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U - _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { + _ if intrinsic.as_str().starts_with("simd_shuffle") => { + let (x, y, idx) = match args { + [x, y, idx] => (x, y, idx), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let x = codegen_operand(fx, x); + let y = codegen_operand(fx, y); + if !x.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); return; @@ -119,11 +141,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( // version of this intrinsic. let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx)); match idx_ty.kind() { - ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { - len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len + .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()) + .unwrap_or_else(|| { span_bug!(span, "could not evaluate shuffle index array length") - }).try_into().unwrap() - } + }) + .try_into() + .unwrap(), _ => { fx.tcx.sess.span_err( span, @@ -154,24 +178,30 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let indexes = { use rustc_middle::mir::interpret::*; - let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const"); + let idx_const = crate::constant::mir_operand_get_const_val(fx, idx) + .expect("simd_shuffle* idx not const"); let idx_bytes = match idx_const { ConstValue::ByRef { alloc, offset } => { - let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */); + let size = Size::from_bytes( + 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */ + ); alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap() } _ => unreachable!("{:?}", idx_const), }; - (0..ret_lane_count).map(|i| { - let i = usize::try_from(i).unwrap(); - let idx = rustc_middle::mir::interpret::read_target_uint( - fx.tcx.data_layout.endian, - &idx_bytes[4*i.. 4*i + 4], - ).expect("read_target_uint"); - u16::try_from(idx).expect("try_from u32") - }).collect::>() + (0..ret_lane_count) + .map(|i| { + let i = usize::try_from(i).unwrap(); + let idx = rustc_middle::mir::interpret::read_target_uint( + fx.tcx.data_layout.endian, + &idx_bytes[4 * i..4 * i + 4], + ) + .expect("read_target_uint"); + u16::try_from(idx).expect("try_from u32") + }) + .collect::>() }; for &idx in &indexes { @@ -187,43 +217,63 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap()); out_lane.write_cvalue(fx, in_lane); } - }; + } + + sym::simd_insert => { + let (base, idx, val) = match args { + [base, idx, val] => (base, idx, val), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let base = codegen_operand(fx, base); + let val = codegen_operand(fx, val); - simd_insert, (c base, o idx, c val) { // FIXME validate - let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { + let idx_const = if let Some(idx_const) = + crate::constant::mir_operand_get_const_val(fx, idx) + { idx_const } else { - fx.tcx.sess.span_fatal( - span, - "Index argument for `simd_insert` is not a constant", - ); + fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant"); }; - let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); + let idx = idx_const + .try_to_bits(Size::from_bytes(4 /* u32*/)) + .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { - fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); + fx.tcx.sess.span_fatal( + fx.mir.span, + &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count), + ); } ret.write_cvalue(fx, base); let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap())); ret_lane.write_cvalue(fx, val); - }; + } + + sym::simd_extract => { + let (v, idx) = match args { + [v, idx] => (v, idx), + _ => { + bug!("wrong number of args for intrinsic {intrinsic}"); + } + }; + let v = codegen_operand(fx, v); - simd_extract, (c v, o idx) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } - let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { + let idx_const = if let Some(idx_const) = + crate::constant::mir_operand_get_const_val(fx, idx) + { idx_const } else { - fx.tcx.sess.span_warn( - span, - "Index argument for `simd_extract` is not a constant", - ); + fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant"); let res = crate::trap::trap_unimplemented_ret_value( fx, ret.layout(), @@ -233,89 +283,105 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( return; }; - let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); + let idx = idx_const + .try_to_bits(Size::from_bytes(4 /* u32*/)) + .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { - fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); + fx.tcx.sess.span_fatal( + fx.mir.span, + &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count), + ); } let ret_lane = v.value_lane(fx, idx.try_into().unwrap()); ret.write_cvalue(fx, ret_lane); - }; + } + + sym::simd_neg => { + intrinsic_args!(fx, args => (a); intrinsic); - simd_neg, (c a) { if !a.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); return; } - simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| { - match lane_ty.kind() { + simd_for_each_lane( + fx, + a, + ret, + &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() { ty::Int(_) => fx.bcx.ins().ineg(lane), ty::Float(_) => fx.bcx.ins().fneg(lane), _ => unreachable!(), - } - }); - }; - - simd_add | simd_sub | simd_mul | simd_div | simd_rem - | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) { - if !x.layout().ty.is_simd() { - report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); - return; - } + }, + ); + } + + sym::simd_add + | sym::simd_sub + | sym::simd_mul + | sym::simd_div + | sym::simd_rem + | sym::simd_shl + | sym::simd_shr + | sym::simd_and + | sym::simd_or + | sym::simd_xor => { + intrinsic_args!(fx, args => (x, y); intrinsic); // FIXME use vector instructions when possible - simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match ( - lane_ty.kind(), - intrinsic, - ) { - (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), - (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), - (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), - (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane), - (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane), - - (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), - (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), - (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), - (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane), - (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane), - - (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane), - (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane), - (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane), - (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane), - (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call( - "fmodf", - vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], - vec![AbiParam::new(types::F32)], - &[x_lane, y_lane], - )[0], - (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call( - "fmod", - vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], - vec![AbiParam::new(types::F64)], - &[x_lane, y_lane], - )[0], - - (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), - (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane), - (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), - (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), - (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), - - (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), - (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane), - (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), - (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), - (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), - - _ => unreachable!(), + simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| { + match (lane_ty.kind(), intrinsic) { + (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), + (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), + (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), + (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane), + (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane), + + (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane), + (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane), + (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane), + (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane), + (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane), + + (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane), + (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane), + (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane), + (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane), + (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call( + "fmodf", + vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], + vec![AbiParam::new(types::F32)], + &[x_lane, y_lane], + )[0], + (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call( + "fmod", + vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], + vec![AbiParam::new(types::F64)], + &[x_lane, y_lane], + )[0], + + (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), + (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane), + (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), + (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), + (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), + + (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane), + (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane), + (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane), + (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane), + (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane), + + _ => unreachable!(), + } }); - }; + } + + sym::simd_fma => { + intrinsic_args!(fx, args => (a, b, c); intrinsic); - simd_fma, (c a, c b, c c) { if !a.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); return; @@ -333,16 +399,22 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let c_lane = c.value_lane(fx, lane); let res_lane = match lane_ty.kind() { - ty::Float(FloatTy::F32) => fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty), - ty::Float(FloatTy::F64) => fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty), + ty::Float(FloatTy::F32) => { + fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty) + } + ty::Float(FloatTy::F64) => { + fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty) + } _ => unreachable!(), }; ret.place_lane(fx, lane).write_cvalue(fx, res_lane); } - }; + } + + sym::simd_fmin | sym::simd_fmax => { + intrinsic_args!(fx, args => (x, y); intrinsic); - simd_fmin | simd_fmax, (c x, c y) { if !x.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); return; @@ -351,7 +423,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( // FIXME use vector instructions when possible simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| { match lane_ty.kind() { - ty::Float(_) => {}, + ty::Float(_) => {} _ => unreachable!("{:?}", lane_ty), } match intrinsic { @@ -360,16 +432,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( _ => unreachable!(), } }); - }; + } + + sym::simd_round => { + intrinsic_args!(fx, args => (a); intrinsic); - simd_round, (c a) { if !a.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); return; } - simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| { - match lane_ty.kind() { + simd_for_each_lane( + fx, + a, + ret, + &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() { ty::Float(FloatTy::F32) => fx.lib_call( "roundf", vec![AbiParam::new(types::F32)], @@ -383,11 +460,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( &[lane], )[0], _ => unreachable!("{:?}", lane_ty), - } - }); - }; + }, + ); + } + + sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => { + intrinsic_args!(fx, args => (a); intrinsic); - simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) { if !a.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty); return; @@ -395,7 +474,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| { match lane_ty.kind() { - ty::Float(_) => {}, + ty::Float(_) => {} _ => unreachable!("{:?}", lane_ty), } match intrinsic { @@ -407,9 +486,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( _ => unreachable!(), } }); - }; + } + + sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => { + intrinsic_args!(fx, args => (v, acc); intrinsic); + let acc = acc.load_scalar(fx); - simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) { // FIXME there must be no acc param for integer vectors if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); @@ -423,9 +505,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.ins().iadd(a, b) } }); - }; + } + + sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => { + intrinsic_args!(fx, args => (v, acc); intrinsic); + let acc = acc.load_scalar(fx); - simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) { // FIXME there must be no acc param for integer vectors if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); @@ -439,54 +524,66 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.ins().imul(a, b) } }); - }; + } + + sym::simd_reduce_all => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_all, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b)); - }; + } + + sym::simd_reduce_any => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_any, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b)); - }; + } + + sym::simd_reduce_and => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_and, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b)); - }; + } + + sym::simd_reduce_or => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_or, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b)); - }; + } + + sym::simd_reduce_xor => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_xor, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; } simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b)); - }; + } + + sym::simd_reduce_min => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_min, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; @@ -501,9 +598,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; fx.bcx.ins().select(lt, a, b) }); - }; + } + + sym::simd_reduce_max => { + intrinsic_args!(fx, args => (v); intrinsic); - simd_reduce_max, (c v) { if !v.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty); return; @@ -518,9 +617,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; fx.bcx.ins().select(gt, a, b) }); - }; + } + + sym::simd_select => { + intrinsic_args!(fx, args => (m, a, b); intrinsic); - simd_select, (c m, c a, c b) { if !m.layout().ty.is_simd() { report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty); return; @@ -540,15 +641,19 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let b_lane = b.value_lane(fx, lane).load_scalar(fx); let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); - let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); + let res_lane = + CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); ret.place_lane(fx, lane).write_cvalue(fx, res_lane); } - }; + } // simd_saturating_* // simd_bitmask // simd_scatter // simd_gather + _ => { + fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); + } } } From 54eb0d992a824da4579f7e8db252cba18f7f6f0d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 26 Jul 2022 15:14:22 +0000 Subject: [PATCH 3/5] Fully remove the intrinsic_match macro --- src/intrinsics/mod.rs | 610 +++++++++++++++++++++++++----------------- 1 file changed, 360 insertions(+), 250 deletions(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index de7695041d360..49c0fa3434081 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -1,28 +1,6 @@ //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"` //! and LLVM intrinsics that have symbol names starting with `llvm.`. -macro_rules! intrinsic_pat { - (_) => { - _ - }; - ($name:ident) => { - sym::$name - }; - (kw.$name:ident) => { - kw::$name - }; -} - -macro_rules! intrinsic_arg { - (o $fx:expr, $arg:ident) => {}; - (c $fx:expr, $arg:ident) => { - let $arg = codegen_operand($fx, $arg); - }; - (v $fx:expr, $arg:ident) => { - let $arg = codegen_operand($fx, $arg).load_scalar($fx); - }; -} - macro_rules! intrinsic_args { ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => { #[allow(unused_parens)] @@ -34,28 +12,6 @@ macro_rules! intrinsic_args { } } -macro_rules! intrinsic_match { - ($fx:expr, $intrinsic:expr, $args:expr, - _ => $unknown:block; - $( - $($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block; - )*) => { - match $intrinsic { - $( - $(intrinsic_pat!($($name).*))|* $(if $cond)? => { - if let [$($arg),*] = $args { - $(intrinsic_arg!($a $fx, $arg);)* - $content - } else { - bug!("wrong number of args for intrinsic {:?}", $intrinsic); - } - } - )* - _ => $unknown, - } - } -} - mod cpuid; mod llvm; mod simd; @@ -359,28 +315,31 @@ fn codegen_regular_intrinsic_call<'tcx>( ) { let usize_layout = fx.layout_of(fx.tcx.types.usize); - intrinsic_match! { - fx, intrinsic, args, - _ => { - fx.tcx.sess.span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic)); - }; + match intrinsic { + sym::assume => { + intrinsic_args!(fx, args => (_a); intrinsic); + } + sym::likely | sym::unlikely => { + intrinsic_args!(fx, args => (a); intrinsic); - assume, (c _a) {}; - likely | unlikely, (c a) { ret.write_cvalue(fx, a); - }; - breakpoint, () { + } + sym::breakpoint => { + intrinsic_args!(fx, args => (); intrinsic); + fx.bcx.ins().debugtrap(); - }; - copy | copy_nonoverlapping, (v src, v dst, v count) { + } + sym::copy | sym::copy_nonoverlapping => { + intrinsic_args!(fx, args => (src, dst, count); intrinsic); + let src = src.load_scalar(fx); + let dst = dst.load_scalar(fx); + let count = count.load_scalar(fx); + let elem_ty = substs.type_at(0); let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); assert_eq!(args.len(), 3); - let byte_amount = if elem_size != 1 { - fx.bcx.ins().imul_imm(count, elem_size as i64) - } else { - count - }; + let byte_amount = + if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy @@ -389,17 +348,19 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME emit_small_memmove fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount); } - }; - // NOTE: the volatile variants have src and dst swapped - volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) { + } + sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => { + // NOTE: the volatile variants have src and dst swapped + intrinsic_args!(fx, args => (dst, src, count); intrinsic); + let dst = dst.load_scalar(fx); + let src = src.load_scalar(fx); + let count = count.load_scalar(fx); + let elem_ty = substs.type_at(0); let elem_size: u64 = fx.layout_of(elem_ty).size.bytes(); assert_eq!(args.len(), 3); - let byte_amount = if elem_size != 1 { - fx.bcx.ins().imul_imm(count, elem_size as i64) - } else { - count - }; + let byte_amount = + if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} if intrinsic == sym::volatile_copy_nonoverlapping_memory { @@ -409,8 +370,10 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME emit_small_memmove fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount); } - }; - size_of_val, (c ptr) { + } + sym::size_of_val => { + intrinsic_args!(fx, args => (ptr); intrinsic); + let layout = fx.layout_of(substs.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch @@ -419,14 +382,13 @@ fn codegen_regular_intrinsic_call<'tcx>( let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info); size } else { - fx - .bcx - .ins() - .iconst(fx.pointer_type, layout.size.bytes() as i64) + fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64) }; ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); - }; - min_align_of_val, (c ptr) { + } + sym::min_align_of_val => { + intrinsic_args!(fx, args => (ptr); intrinsic); + let layout = fx.layout_of(substs.type_at(0)); // Note: Can't use is_unsized here as truly unsized types need to take the fixed size // branch @@ -435,26 +397,37 @@ fn codegen_regular_intrinsic_call<'tcx>( let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info); align } else { - fx - .bcx - .ins() - .iconst(fx.pointer_type, layout.align.abi.bytes() as i64) + fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64) }; ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); - }; + } + + sym::vtable_size => { + intrinsic_args!(fx, args => (vtable); intrinsic); + let vtable = vtable.load_scalar(fx); - vtable_size, (v vtable) { let size = crate::vtable::size_of_obj(fx, vtable); ret.write_cvalue(fx, CValue::by_val(size, usize_layout)); - }; + } + + sym::vtable_align => { + intrinsic_args!(fx, args => (vtable); intrinsic); + let vtable = vtable.load_scalar(fx); - vtable_align, (v vtable) { let align = crate::vtable::min_align_of_obj(fx, vtable); ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); - }; + } + + sym::unchecked_add + | sym::unchecked_sub + | sym::unchecked_mul + | sym::unchecked_div + | sym::exact_div + | sym::unchecked_rem + | sym::unchecked_shl + | sym::unchecked_shr => { + intrinsic_args!(fx, args => (x, y); intrinsic); - unchecked_add | unchecked_sub | unchecked_mul | unchecked_div | exact_div | unchecked_rem - | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { sym::unchecked_add => BinOp::Add, @@ -468,8 +441,10 @@ fn codegen_regular_intrinsic_call<'tcx>( }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); - }; - add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { + } + sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { + intrinsic_args!(fx, args => (x, y); intrinsic); + assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { sym::add_with_overflow => BinOp::Add, @@ -478,15 +453,12 @@ fn codegen_regular_intrinsic_call<'tcx>( _ => unreachable!(), }; - let res = crate::num::codegen_checked_int_binop( - fx, - bin_op, - x, - y, - ); + let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); - }; - saturating_add | saturating_sub, (c lhs, c rhs) { + } + sym::saturating_add | sym::saturating_sub => { + intrinsic_args!(fx, args => (lhs, rhs); intrinsic); + assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { sym::saturating_add => BinOp::Add, @@ -496,12 +468,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let signed = type_sign(lhs.layout().ty); - let checked_res = crate::num::codegen_checked_int_binop( - fx, - bin_op, - lhs, - rhs, - ); + let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); let (val, has_overflow) = checked_res.load_scalar_pair(fx); let clif_ty = fx.clif_type(lhs.layout().ty).unwrap(); @@ -513,13 +480,15 @@ fn codegen_regular_intrinsic_call<'tcx>( (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let rhs_ge_zero = + fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); - let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); + let rhs_ge_zero = + fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); fx.bcx.ins().select(has_overflow, sat_val, val) } @@ -529,23 +498,32 @@ fn codegen_regular_intrinsic_call<'tcx>( let res = CValue::by_val(val, lhs.layout()); ret.write_cvalue(fx, res); - }; - rotate_left, (c x, v y) { + } + sym::rotate_left => { + intrinsic_args!(fx, args => (x, y); intrinsic); + let y = y.load_scalar(fx); + let layout = x.layout(); let x = x.load_scalar(fx); let res = fx.bcx.ins().rotl(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); - }; - rotate_right, (c x, v y) { + } + sym::rotate_right => { + intrinsic_args!(fx, args => (x, y); intrinsic); + let y = y.load_scalar(fx); + let layout = x.layout(); let x = x.load_scalar(fx); let res = fx.bcx.ins().rotr(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); - }; + } // The only difference between offset and arith_offset is regarding UB. Because Cranelift // doesn't have UB both are codegen'ed the same way - offset | arith_offset, (c base, v offset) { + sym::offset | sym::arith_offset => { + intrinsic_args!(fx, args => (base, offset); intrinsic); + let offset = offset.load_scalar(fx); + let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty; let pointee_size = fx.layout_of(pointee_ty).size.bytes(); let ptr_diff = if pointee_size != 1 { @@ -556,12 +534,18 @@ fn codegen_regular_intrinsic_call<'tcx>( let base_val = base.load_scalar(fx); let res = fx.bcx.ins().iadd(base_val, ptr_diff); ret.write_cvalue(fx, CValue::by_val(res, base.layout())); - }; + } + + sym::transmute => { + intrinsic_args!(fx, args => (from); intrinsic); - transmute, (c from) { ret.write_cvalue_transmute(fx, from); - }; - write_bytes | volatile_set_memory, (c dst, v val, v count) { + } + sym::write_bytes | sym::volatile_set_memory => { + intrinsic_args!(fx, args => (dst, val, count); intrinsic); + let val = val.load_scalar(fx); + let count = count.load_scalar(fx); + let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty; let pointee_size = fx.layout_of(pointee_ty).size.bytes(); let count = if pointee_size != 1 { @@ -573,34 +557,42 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset fx.bcx.call_memset(fx.target_config, dst_ptr, val, count); - }; - ctlz | ctlz_nonzero, (c arg) { + } + sym::ctlz | sym::ctlz_nonzero => { + intrinsic_args!(fx, args => (arg); intrinsic); let val = arg.load_scalar(fx); + // FIXME trap on `ctlz_nonzero` with zero arg. let res = fx.bcx.ins().clz(val); let res = CValue::by_val(res, arg.layout()); ret.write_cvalue(fx, res); - }; - cttz | cttz_nonzero, (c arg) { + } + sym::cttz | sym::cttz_nonzero => { + intrinsic_args!(fx, args => (arg); intrinsic); let val = arg.load_scalar(fx); + // FIXME trap on `cttz_nonzero` with zero arg. let res = fx.bcx.ins().ctz(val); let res = CValue::by_val(res, arg.layout()); ret.write_cvalue(fx, res); - }; - ctpop, (c arg) { + } + sym::ctpop => { + intrinsic_args!(fx, args => (arg); intrinsic); let val = arg.load_scalar(fx); + let res = fx.bcx.ins().popcnt(val); let res = CValue::by_val(res, arg.layout()); ret.write_cvalue(fx, res); - }; - bitreverse, (c arg) { + } + sym::bitreverse => { + intrinsic_args!(fx, args => (arg); intrinsic); let val = arg.load_scalar(fx); + let res = fx.bcx.ins().bitrev(val); let res = CValue::by_val(res, arg.layout()); ret.write_cvalue(fx, res); - }; - bswap, (c arg) { + } + sym::bswap => { // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value { match bcx.func.dfg.value_type(v) { @@ -676,11 +668,15 @@ fn codegen_regular_intrinsic_call<'tcx>( ty => unreachable!("bswap {}", ty), } } + intrinsic_args!(fx, args => (arg); intrinsic); let val = arg.load_scalar(fx); + let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout()); ret.write_cvalue(fx, res); - }; - assert_inhabited | assert_zero_valid | assert_uninit_valid, () { + } + sym::assert_inhabited | sym::assert_zero_valid | sym::assert_uninit_valid => { + intrinsic_args!(fx, args => (); intrinsic); + let layout = fx.layout_of(substs.type_at(0)); if layout.abi.is_uninhabited() { with_no_trimmed_paths!({ @@ -697,7 +693,10 @@ fn codegen_regular_intrinsic_call<'tcx>( with_no_trimmed_paths!({ crate::base::codegen_panic( fx, - &format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty), + &format!( + "attempted to zero-initialize type `{}`, which is invalid", + layout.ty + ), source_info, ); }); @@ -708,41 +707,53 @@ fn codegen_regular_intrinsic_call<'tcx>( with_no_trimmed_paths!({ crate::base::codegen_panic( fx, - &format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty), + &format!( + "attempted to leave type `{}` uninitialized, which is invalid", + layout.ty + ), source_info, ) }); return; } - }; + } + + sym::volatile_load | sym::unaligned_volatile_load => { + intrinsic_args!(fx, args => (ptr); intrinsic); - volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default // FIXME correctly handle unaligned_volatile_load - let inner_layout = - fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); + let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); ret.write_cvalue(fx, val); - }; - volatile_store | unaligned_volatile_store, (v ptr, c val) { + } + sym::volatile_store | sym::unaligned_volatile_store => { + intrinsic_args!(fx, args => (ptr, val); intrinsic); + let ptr = ptr.load_scalar(fx); + // Cranelift treats stores as volatile by default // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); - }; + } + + sym::pref_align_of + | sym::needs_drop + | sym::type_id + | sym::type_name + | sym::variant_count => { + intrinsic_args!(fx, args => (); intrinsic); - pref_align_of | needs_drop | type_id | type_name | variant_count, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); - let val = crate::constant::codegen_const_value( - fx, - const_val, - ret.layout().ty, - ); + let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty); ret.write_cvalue(fx, val); - }; + } - ptr_offset_from | ptr_offset_from_unsigned, (v ptr, v base) { + sym::ptr_offset_from | sym::ptr_offset_from_unsigned => { + intrinsic_args!(fx, args => (ptr, base); intrinsic); + let ptr = ptr.load_scalar(fx); + let base = base.load_scalar(fx); let ty = substs.type_at(0); let pointee_size: u64 = fx.layout_of(ty).size.bytes(); @@ -758,31 +769,44 @@ fn codegen_regular_intrinsic_call<'tcx>( CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout) }; ret.write_cvalue(fx, val); - }; + } + + sym::ptr_guaranteed_eq => { + intrinsic_args!(fx, args => (a, b); intrinsic); - ptr_guaranteed_eq, (c a, c b) { let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b); ret.write_cvalue(fx, val); - }; + } + + sym::ptr_guaranteed_ne => { + intrinsic_args!(fx, args => (a, b); intrinsic); - ptr_guaranteed_ne, (c a, c b) { let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b); ret.write_cvalue(fx, val); - }; + } + + sym::caller_location => { + intrinsic_args!(fx, args => (); intrinsic); - caller_location, () { let caller_location = fx.get_caller_location(source_info); ret.write_cvalue(fx, caller_location); - }; + } + + _ if intrinsic.as_str().starts_with("atomic_fence") => { + intrinsic_args!(fx, args => (); intrinsic); - _ if intrinsic.as_str().starts_with("atomic_fence"), () { fx.bcx.ins().fence(); - }; - _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { + } + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => { + intrinsic_args!(fx, args => (); intrinsic); + // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); - }; - _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) { + } + _ if intrinsic.as_str().starts_with("atomic_load") => { + intrinsic_args!(fx, args => (ptr); intrinsic); + let ptr = ptr.load_scalar(fx); + let ty = substs.type_at(0); match ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { @@ -794,7 +818,9 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().jump(ret_block, &[]); return; } else { - fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported"); + fx.tcx + .sess + .span_fatal(source_info.span, "128bit atomics not yet supported"); } } ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -809,8 +835,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(ty)); ret.write_cvalue(fx, val); - }; - _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { + } + _ if intrinsic.as_str().starts_with("atomic_store") => { + intrinsic_args!(fx, args => (ptr, val); intrinsic); + let ptr = ptr.load_scalar(fx); + let ty = substs.type_at(0); match ty.kind() { ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { @@ -822,7 +851,9 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().jump(ret_block, &[]); return; } else { - fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported"); + fx.tcx + .sess + .span_fatal(source_info.span, "128bit atomics not yet supported"); } } ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -835,8 +866,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = val.load_scalar(fx); fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); - }; - _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { + } + _ if intrinsic.as_str().starts_with("atomic_xchg") => { + intrinsic_args!(fx, args => (ptr, new); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = new.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -853,8 +887,12 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + } + _ if intrinsic.as_str().starts_with("atomic_cxchg") => { + // both atomic_cxchg_* and atomic_cxchgweak_* + intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = new.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -870,11 +908,15 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new); let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old); - let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout()); + let ret_val = + CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout()); ret.write_cvalue(fx, ret_val) - }; + } + + _ if intrinsic.as_str().starts_with("atomic_xadd") => { + intrinsic_args!(fx, args => (ptr, amount); intrinsic); + let ptr = ptr.load_scalar(fx); - _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { let layout = amount.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -887,12 +929,16 @@ fn codegen_regular_intrinsic_call<'tcx>( let amount = amount.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { + } + _ if intrinsic.as_str().starts_with("atomic_xsub") => { + intrinsic_args!(fx, args => (ptr, amount); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = amount.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -905,12 +951,16 @@ fn codegen_regular_intrinsic_call<'tcx>( let amount = amount.load_scalar(fx); - let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); + let old = + fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_and") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -927,8 +977,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_or") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -945,8 +998,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_xor") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -963,8 +1019,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_nand") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -981,8 +1040,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_max") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -999,8 +1061,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_umax") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -1017,8 +1082,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_min") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -1035,8 +1103,11 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; - _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { + } + _ if intrinsic.as_str().starts_with("atomic_umin") => { + intrinsic_args!(fx, args => (ptr, src); intrinsic); + let ptr = ptr.load_scalar(fx); + let layout = src.layout(); match layout.ty.kind() { ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} @@ -1053,30 +1124,51 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); - }; + } + + sym::minnumf32 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); - minnumf32, (v a, v b) { let val = crate::num::codegen_float_min(fx, a, b); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); ret.write_cvalue(fx, val); - }; - minnumf64, (v a, v b) { + } + sym::minnumf64 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let val = crate::num::codegen_float_min(fx, a, b); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); - }; - maxnumf32, (v a, v b) { + } + sym::maxnumf32 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let val = crate::num::codegen_float_max(fx, a, b); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32)); ret.write_cvalue(fx, val); - }; - maxnumf64, (v a, v b) { + } + sym::maxnumf64 => { + intrinsic_args!(fx, args => (a, b); intrinsic); + let a = a.load_scalar(fx); + let b = b.load_scalar(fx); + let val = crate::num::codegen_float_max(fx, a, b); let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64)); ret.write_cvalue(fx, val); - }; + } + + kw::Try => { + intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic); + let f = f.load_scalar(fx); + let data = data.load_scalar(fx); + let _catch_fn = catch_fn.load_scalar(fx); - kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: fx.target_config.default_call_conv, @@ -1089,20 +1181,30 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = ret.layout(); let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size)); ret.write_cvalue(fx, ret_val); - }; + } - fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { - let res = crate::num::codegen_float_binop(fx, match intrinsic { - sym::fadd_fast => BinOp::Add, - sym::fsub_fast => BinOp::Sub, - sym::fmul_fast => BinOp::Mul, - sym::fdiv_fast => BinOp::Div, - sym::frem_fast => BinOp::Rem, - _ => unreachable!(), - }, x, y); + sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + let res = crate::num::codegen_float_binop( + fx, + match intrinsic { + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, + _ => unreachable!(), + }, + x, + y, + ); ret.write_cvalue(fx, res); - }; - float_to_int_unchecked, (v f) { + } + sym::float_to_int_unchecked => { + intrinsic_args!(fx, args => (f); intrinsic); + let f = f.load_scalar(fx); + let res = crate::cast::clif_int_or_float_cast( fx, f, @@ -1111,66 +1213,74 @@ fn codegen_regular_intrinsic_call<'tcx>( type_sign(ret.layout().ty), ); ret.write_cvalue(fx, CValue::by_val(res, ret.layout())); - }; + } + + sym::raw_eq => { + intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic); + let lhs_ref = lhs_ref.load_scalar(fx); + let rhs_ref = rhs_ref.load_scalar(fx); - raw_eq, (v lhs_ref, v rhs_ref) { let size = fx.layout_of(substs.type_at(0)).layout.size(); // FIXME add and use emit_small_memcmp - let is_eq_value = - if size == Size::ZERO { - // No bytes means they're trivially equal - fx.bcx.ins().iconst(types::I8, 1) - } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) { - // Can't use `trusted` for these loads; they could be unaligned. - let mut flags = MemFlags::new(); - flags.set_notrap(); - let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0); - let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0); - let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val); - fx.bcx.ins().bint(types::I8, eq) - } else { - // Just call `memcmp` (like slices do in core) when the - // size is too large or it's not a power-of-two. - let signed_bytes = i64::try_from(size.bytes()).unwrap(); - let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes); - let params = vec![AbiParam::new(fx.pointer_type); 3]; - let returns = vec![AbiParam::new(types::I32)]; - let args = &[lhs_ref, rhs_ref, bytes_val]; - let cmp = fx.lib_call("memcmp", params, returns, args)[0]; - let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0); - fx.bcx.ins().bint(types::I8, eq) - }; + let is_eq_value = if size == Size::ZERO { + // No bytes means they're trivially equal + fx.bcx.ins().iconst(types::I8, 1) + } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) { + // Can't use `trusted` for these loads; they could be unaligned. + let mut flags = MemFlags::new(); + flags.set_notrap(); + let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0); + let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0); + let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val); + fx.bcx.ins().bint(types::I8, eq) + } else { + // Just call `memcmp` (like slices do in core) when the + // size is too large or it's not a power-of-two. + let signed_bytes = i64::try_from(size.bytes()).unwrap(); + let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes); + let params = vec![AbiParam::new(fx.pointer_type); 3]; + let returns = vec![AbiParam::new(types::I32)]; + let args = &[lhs_ref, rhs_ref, bytes_val]; + let cmp = fx.lib_call("memcmp", params, returns, args)[0]; + let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0); + fx.bcx.ins().bint(types::I8, eq) + }; ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout())); - }; + } + + sym::const_allocate => { + intrinsic_args!(fx, args => (_size, _align); intrinsic); - const_allocate, (c _size, c _align) { // returns a null pointer at runtime. let null = fx.bcx.ins().iconst(fx.pointer_type, 0); ret.write_cvalue(fx, CValue::by_val(null, ret.layout())); - }; + } - const_deallocate, (c _ptr, c _size, c _align) { + sym::const_deallocate => { + intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic); // nop at runtime. - }; + } + + sym::black_box => { + intrinsic_args!(fx, args => (a); intrinsic); - black_box, (c a) { // FIXME implement black_box semantics ret.write_cvalue(fx, a); - }; + } // FIXME implement variadics in cranelift - va_copy, (o _dest, o _src) { + sym::va_copy | sym::va_arg | sym::va_end => { fx.tcx.sess.span_fatal( source_info.span, "Defining variadic functions is not yet supported by Cranelift", ); - }; - va_arg | va_end, (o _valist) { - fx.tcx.sess.span_fatal( - source_info.span, - "Defining variadic functions is not yet supported by Cranelift", - ); - }; + } + + _ => { + fx.tcx + .sess + .span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic)); + } } let ret_block = fx.get_block(destination.unwrap()); From d3099a40c77b7782b031d05c60d595a236306dba Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 26 Jul 2022 16:26:35 +0000 Subject: [PATCH 4/5] Outline a bug! invocation This reduces the amount of llvm ir lines for intrinsic related code from 23801 to 20478. --- src/intrinsics/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 49c0fa3434081..b2a83e1d4ebc9 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -7,7 +7,7 @@ macro_rules! intrinsic_args { let ($($arg),*) = if let [$($arg),*] = $args { ($(codegen_operand($fx, $arg)),*) } else { - bug!("wrong number of args for intrinsic {}", $intrinsic); + $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic); }; } } @@ -26,6 +26,10 @@ use rustc_span::symbol::{kw, sym, Symbol}; use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; +fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! { + bug!("wrong number of args for intrinsic {}", intrinsic); +} + fn report_atomic_type_validation_error<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, intrinsic: Symbol, From d3a2366ee877075c59b38bd8ced55f224fc7ef51 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 26 Jul 2022 16:27:14 +0000 Subject: [PATCH 5/5] Implement some more llvm float compare intrinsic options --- src/intrinsics/llvm.rs | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index a987e20b11a77..869670c8cfac7 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -64,23 +64,11 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( 0 => FloatCC::Equal, 1 => FloatCC::LessThan, 2 => FloatCC::LessThanOrEqual, - 7 => { - unimplemented!( - "Compares corresponding elements in `a` and `b` to see if neither is `NaN`." - ); - } - 3 => { - unimplemented!( - "Compares corresponding elements in `a` and `b` to see if either is `NaN`." - ); - } + 7 => FloatCC::Ordered, + 3 => FloatCC::Unordered, 4 => FloatCC::NotEqual, - 5 => { - unimplemented!("not less than"); - } - 6 => { - unimplemented!("not less than or equal"); - } + 5 => FloatCC::UnorderedOrGreaterThanOrEqual, + 6 => FloatCC::UnorderedOrGreaterThan, kind => unreachable!("kind {:?}", kind), };