Introduce InstanceKind::LlvmIntrinsic

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -440,18 +440,6 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             }
         }
 
-        if fx.tcx.symbol_name(instance).name.starts_with("llvm.") {
-            crate::intrinsics::codegen_llvm_intrinsic_call(
-                fx,
-                fx.tcx.symbol_name(instance).name,
-                args,
-                ret_place,
-                target,
-                source_info.span,
-            );
-            return;
-        }
-
         match instance.def {
             InstanceKind::Intrinsic(_) => {
                 match crate::intrinsics::codegen_intrinsic_call(
@@ -466,6 +454,17 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                     Err(instance) => Some(instance),
                 }
             }
+            InstanceKind::LlvmIntrinsic(_) => {
+                crate::intrinsics::codegen_llvm_intrinsic_call(
+                    fx,
+                    fx.tcx.symbol_name(instance).name,
+                    args,
+                    ret_place,
+                    target,
+                    source_info.span,
+                );
+                return;
+            }
             // We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
             // it is `func returning noop future`
             InstanceKind::Shim(ShimKind::DropGlue(_, None)) => {

compiler/rustc_codegen_ssa/src/base.rs

@@ -892,12 +892,8 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
 ) -> bool {
-    fn is_llvm_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-        if let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name {
-            name.as_str().starts_with("llvm.")
-        } else {
-            false
-        }
+    if let ty::InstanceKind::LlvmIntrinsic(_) = instance.def {
+        return false;
     }
 
     fn is_extern_call_to_local_crate<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> bool {
@@ -910,7 +906,6 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>(
     let def_id = instance.def_id();
     !def_id.is_local()
         && tcx.is_compiler_builtins(LOCAL_CRATE)
-        && !is_llvm_intrinsic(tcx, def_id)
         && !tcx.should_codegen_locally(instance)
         && !is_extern_call_to_local_crate(tcx, instance)
 }

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1112,8 +1112,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
 
         if let Some(instance) = instance
+            && let ty::InstanceKind::LlvmIntrinsic(_) = instance.def
             && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name
-            && name.as_str().starts_with("llvm.")
             // This is the only LLVM intrinsic we use that unwinds
             // FIXME either add unwind support to codegen_llvm_intrinsic_call or replace usage of
             // this intrinsic with something else

compiler/rustc_const_eval/src/const_eval/dummy_machine.rs

@@ -105,6 +105,16 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
         unimplemented!()
     }
 
+    fn call_llvm_intrinsic(
+        _ecx: &mut InterpCx<'tcx, Self>,
+        _instance: ty::Instance<'tcx>,
+        _args: &[interpret::OpTy<'tcx, Self::Provenance>],
+        _destination: &interpret::PlaceTy<'tcx, Self::Provenance>,
+        _target: Option<BasicBlock>,
+    ) -> interpret::InterpResult<'tcx> {
+        unimplemented!()
+    }
+
     fn assert_panic(
         _ecx: &mut InterpCx<'tcx, Self>,
         _msg: &rustc_middle::mir::AssertMessage<'tcx>,

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -745,6 +745,18 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
         interp_ok(None)
     }
 
+    fn call_llvm_intrinsic(
+        ecx: &mut InterpCx<'tcx, Self>,
+        instance: ty::Instance<'tcx>,
+        _args: &[OpTy<'tcx>],
+        _dest: &PlaceTy<'tcx, Self::Provenance>,
+        _target: Option<mir::BasicBlock>,
+    ) -> InterpResult<'tcx> {
+        let intrinsic_name = ecx.tcx.codegen_fn_attrs(instance.def_id()).symbol_name.unwrap();
+
+        throw_unsup_format!("LLVM intrinsic `{intrinsic_name}` is not supported at compile-time");
+    }
+
     fn assert_panic(
         ecx: &mut InterpCx<'tcx, Self>,
         msg: &AssertMessage<'tcx>,

compiler/rustc_const_eval/src/interpret/call.rs

@@ -652,6 +652,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                     interp_ok(())
                 }
             }
+            ty::InstanceKind::LlvmIntrinsic(_) => {
+                // FIXME: Should `InPlace` arguments be reset to uninit?
+                M::call_llvm_intrinsic(
+                    self,
+                    instance,
+                    &Self::copy_fn_args(args),
+                    destination,
+                    target,
+                )
+            }
             ty::InstanceKind::Shim(ty::ShimKind::VTable(..))
             | ty::InstanceKind::Shim(ty::ShimKind::Reify(..))
             | ty::InstanceKind::Shim(ty::ShimKind::ClosureOnce { .. })

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -247,6 +247,16 @@ pub trait Machine<'tcx>: Sized {
         unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>;
 
+    /// Directly process an LLVM intrinsic without pushing a stack frame. It is the hook's
+    /// responsibility to advance the instruction pointer as appropriate.
+    fn call_llvm_intrinsic(
+        ecx: &mut InterpCx<'tcx, Self>,
+        instance: ty::Instance<'tcx>,
+        args: &[OpTy<'tcx, Self::Provenance>],
+        destination: &PlaceTy<'tcx, Self::Provenance>,
+        target: Option<mir::BasicBlock>,
+    ) -> InterpResult<'tcx>;
+
     /// Check whether the given function may be executed on the current machine, in terms of the
     /// target features is requires.
     fn check_fn_target_features(

compiler/rustc_middle/src/mir/visit.rs

@@ -347,6 +347,7 @@ macro_rules! make_mir_visitor {
                         ty::InstanceKind::Item(_def_id) => {}
 
                         ty::InstanceKind::Intrinsic(_def_id)
+                        | ty::InstanceKind::LlvmIntrinsic(_def_id)
                         | ty::InstanceKind::Shim(ty::ShimKind::VTable(_def_id))
                         | ty::InstanceKind::Shim(ty::ShimKind::Reify(_def_id, _))
                         | ty::InstanceKind::Virtual(_def_id, _)

compiler/rustc_middle/src/mono.rs

@@ -524,6 +524,7 @@ impl<'tcx> CodegenUnit<'tcx> {
                 MonoItem::Fn(ref instance) => match instance.def {
                     InstanceKind::Item(def) => def.as_local().map(|_| def),
                     InstanceKind::Intrinsic(..)
+                    | InstanceKind::LlvmIntrinsic(..)
                     | InstanceKind::Virtual(..)
                     | InstanceKind::Shim(ShimKind::VTable(..))
                     | InstanceKind::Shim(ShimKind::Reify(..))

compiler/rustc_middle/src/ty/instance.rs

@@ -70,11 +70,18 @@ pub enum InstanceKind<'tcx> {
 
     /// An intrinsic `fn` item (with`#[rustc_intrinsic]`).
     ///
-    /// Alongside `Virtual`, this is the only `InstanceKind` that does not have its own callable MIR.
-    /// Instead, codegen and const eval "magically" evaluate calls to intrinsics purely in the
-    /// caller.
+    /// Alongside `LlvmIntrinsic` and `Virtual`, this is the only `InstanceKind`
+    /// that does not have its own callable MIR. Instead, codegen and const eval
+    /// "magically" evaluate calls to intrinsics purely in the caller.
     Intrinsic(DefId),
 
+    /// An LLVM intrinsic `fn` item (with `extern "unadjusted"`).
+    ///
+    /// Alongside `Intrinsic` and `Virtual`, this is the only `InstanceKind`
+    /// that does not have its own callable MIR. Instead, codegen and const eval
+    /// "magically" evaluate calls to LLVM intrinsics purely in the caller.
+    LlvmIntrinsic(DefId),
+
     /// Dynamic dispatch to `<dyn Trait as Trait>::fn`.
     ///
     /// This `InstanceKind` may have a callable MIR as the default implementation.
@@ -253,7 +260,8 @@ impl<'tcx> InstanceKind<'tcx> {
         match self {
             InstanceKind::Item(def_id)
             | InstanceKind::Virtual(def_id, _)
-            | InstanceKind::Intrinsic(def_id) => def_id,
+            | InstanceKind::Intrinsic(def_id)
+            | InstanceKind::LlvmIntrinsic(def_id) => def_id,
             InstanceKind::Shim(shim) => shim.def_id(),
         }
     }
@@ -262,7 +270,9 @@ impl<'tcx> InstanceKind<'tcx> {
     pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option<DefId> {
         match self {
             InstanceKind::Item(def) => Some(def),
-            InstanceKind::Virtual(..) | InstanceKind::Intrinsic(..) => None,
+            InstanceKind::Virtual(..)
+            | InstanceKind::Intrinsic(..)
+            | InstanceKind::LlvmIntrinsic(..) => None,
             InstanceKind::Shim(shim) => shim.def_id_if_not_guaranteed_local_codegen(),
         }
     }
@@ -280,7 +290,9 @@ impl<'tcx> InstanceKind<'tcx> {
                 DefPathData::Ctor | DefPathData::Closure
             ),
             InstanceKind::Shim(shim) => shim.requires_inline(),
-            InstanceKind::Virtual(..) | InstanceKind::Intrinsic(..) => true,
+            InstanceKind::Virtual(..)
+            | InstanceKind::Intrinsic(..)
+            | InstanceKind::LlvmIntrinsic(..) => true,
         }
     }
 
@@ -308,7 +320,10 @@ impl<'tcx> InstanceKind<'tcx> {
     /// body should perform necessary instantiations.
     pub fn has_polymorphic_mir_body(&self) -> bool {
         match *self {
-            InstanceKind::Item(_) | InstanceKind::Intrinsic(..) | InstanceKind::Virtual(..) => true,
+            InstanceKind::Item(_)
+            | InstanceKind::Intrinsic(..)
+            | InstanceKind::LlvmIntrinsic(..)
+            | InstanceKind::Virtual(..) => true,
             InstanceKind::Shim(shim) => shim.has_polymorphic_mir_body(),
         }
     }

compiler/rustc_middle/src/ty/mod.rs

@@ -1833,7 +1833,9 @@ impl<'tcx> TyCtxt<'tcx> {
                     _ => self.optimized_mir(def),
                 }
             }
-            ty::InstanceKind::Intrinsic(..) => bug!("intrinsics have no instance MIR"),
+            ty::InstanceKind::Intrinsic(..) | ty::InstanceKind::LlvmIntrinsic(..) => {
+                bug!("intrinsics have no instance MIR")
+            }
             ty::InstanceKind::Virtual(..) => bug!("virtual dispatches have no instance MIR"),
             ty::InstanceKind::Shim(shim) => self.mir_shims(shim),
         };

compiler/rustc_middle/src/ty/print/mod.rs

@@ -368,6 +368,7 @@ impl<'tcx, P: Printer<'tcx> + std::fmt::Write> Print<P> for ty::Instance<'tcx> {
         match self.def {
             ty::InstanceKind::Item(_) => {}
             ty::InstanceKind::Intrinsic(_) => cx.write_str(" - intrinsic")?,
+            ty::InstanceKind::LlvmIntrinsic(_) => cx.write_str(" - LLVM intrinsic")?,
             ty::InstanceKind::Virtual(_, num) => cx.write_str(&format!(" - virtual#{num}"))?,
             ty::InstanceKind::Shim(shim) => {
                 cx.write_str(" - ")?;

compiler/rustc_mir_transform/src/inline.rs

@@ -730,7 +730,7 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
             }
         }
         // These have no own callable MIR.
-        InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
+        InstanceKind::Intrinsic(_) | InstanceKind::LlvmIntrinsic(_) | InstanceKind::Virtual(..) => {
             debug!("instance without MIR (intrinsic / virtual)");
             return Err("implementation limitation -- cannot inline intrinsic");
         }

compiler/rustc_mir_transform/src/inline/cycle.rs

@@ -21,7 +21,9 @@ fn should_recurse<'tcx>(tcx: TyCtxt<'tcx>, callee: ty::Instance<'tcx>) -> bool {
         }
 
         // These have no own callable MIR.
-        InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => return false,
+        InstanceKind::Intrinsic(_) | InstanceKind::LlvmIntrinsic(_) | InstanceKind::Virtual(..) => {
+            return false;
+        }
 
         // These have MIR and if that MIR is inlined, instantiated and then inlining is run
         // again, a function item can end up getting inlined. Thus we'll be able to cause

compiler/rustc_monomorphize/src/collector.rs

@@ -1018,7 +1018,9 @@ fn visit_instance_use<'tcx>(
     }
 
     match instance.def {
-        ty::InstanceKind::Virtual(..) | ty::InstanceKind::Intrinsic(_) => {
+        ty::InstanceKind::Virtual(..)
+        | ty::InstanceKind::Intrinsic(_)
+        | ty::InstanceKind::LlvmIntrinsic(_) => {
             if !is_direct_call {
                 bug!("{:?} being reified", instance);
             }

compiler/rustc_monomorphize/src/partitioning.rs

@@ -631,6 +631,7 @@ fn characteristic_def_id_of_mono_item<'tcx>(
             let def_id = match instance.def {
                 ty::InstanceKind::Item(def) => def,
                 ty::InstanceKind::Intrinsic(..)
+                | ty::InstanceKind::LlvmIntrinsic(..)
                 | ty::InstanceKind::Virtual(..)
                 | ty::InstanceKind::Shim(ty::ShimKind::VTable(..))
                 | ty::InstanceKind::Shim(ty::ShimKind::Reify(..))
@@ -810,6 +811,7 @@ fn mono_item_visibility<'tcx>(
         | InstanceKind::Shim(ShimKind::FnPtr(..))
         | InstanceKind::Virtual(..)
         | InstanceKind::Intrinsic(..)
+        | InstanceKind::LlvmIntrinsic(..)
         | InstanceKind::Shim(ShimKind::ClosureOnce { .. })
         | InstanceKind::Shim(ShimKind::ConstructCoroutineInClosure { .. })
         | InstanceKind::Shim(ShimKind::DropGlue(..))

compiler/rustc_public/src/mir/mono.rs

@@ -32,6 +32,8 @@ pub enum InstanceKind {
     Item,
     /// A compiler intrinsic function.
     Intrinsic,
+    /// An LLVM intrinsic function.
+    LlvmIntrinsic,
     /// A virtual function definition stored in a VTable.
     /// The `idx` field indicates the position in the VTable for this instance.
     Virtual { idx: usize },
@@ -113,7 +115,10 @@ impl Instance {
             InstanceKind::Intrinsic => {
                 Some(with(|context| context.intrinsic(self.def.def_id()).unwrap().fn_name()))
             }
-            InstanceKind::Item | InstanceKind::Virtual { .. } | InstanceKind::Shim => None,
+            InstanceKind::LlvmIntrinsic
+            | InstanceKind::Item
+            | InstanceKind::Virtual { .. }
+            | InstanceKind::Shim => None,
         }
     }
 

compiler/rustc_public/src/unstable/convert/stable/ty.rs

@@ -970,6 +970,7 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
         let kind = match self.def {
             ty::InstanceKind::Item(..) => crate::mir::mono::InstanceKind::Item,
             ty::InstanceKind::Intrinsic(..) => crate::mir::mono::InstanceKind::Intrinsic,
+            ty::InstanceKind::LlvmIntrinsic(..) => crate::mir::mono::InstanceKind::LlvmIntrinsic,
             ty::InstanceKind::Virtual(_def_id, idx) => {
                 crate::mir::mono::InstanceKind::Virtual { idx }
             }

compiler/rustc_ty_utils/src/instance.rs

@@ -1,5 +1,6 @@
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::LangItem;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::bug;
@@ -91,6 +92,12 @@ fn resolve_instance_raw<'tcx>(
         } else if tcx.is_async_drop_in_place_coroutine(def_id) {
             let ty = args.type_at(0);
             ty::InstanceKind::Shim(ty::ShimKind::AsyncDropGlue(def_id, ty))
+        } else if tcx.def_kind(def_id) == DefKind::Fn
+            && let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name
+            && name.as_str().starts_with("llvm.")
+        {
+            debug!(" => LLVM intrinsic");
+            ty::InstanceKind::LlvmIntrinsic(def_id)
         } else {
             debug!(" => free item");
             ty::InstanceKind::Item(def_id)

src/tools/miri/src/intrinsics/mod.rs

@@ -10,7 +10,7 @@ pub use self::atomic::AtomicRmwOp;
 use rand::RngExt;
 use rustc_abi::Size;
 use rustc_middle::{mir, ty};
-use rustc_span::Symbol;
+use rustc_span::{Symbol, sym};
 
 use self::atomic::EvalContextExt as _;
 use self::math::EvalContextExt as _;

src/tools/miri/src/machine.rs

@@ -1300,6 +1300,17 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> {
         ecx.call_intrinsic(instance, args, dest, ret, unwind)
     }
 
+    #[inline(always)]
+    fn call_llvm_intrinsic(
+        ecx: &mut MiriInterpCx<'tcx>,
+        instance: ty::Instance<'tcx>,
+        args: &[OpTy<'tcx>],
+        dest: &PlaceTy<'tcx>,
+        ret: Option<mir::BasicBlock>,
+    ) -> InterpResult<'tcx, ()> {
+        ecx.call_llvm_intrinsic(instance, args, dest, ret)
+    }
+
     #[inline(always)]
     fn assert_panic(
         ecx: &mut MiriInterpCx<'tcx>,

src/tools/miri/src/shims/aarch64.rs

@@ -13,7 +13,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
         link_name: Symbol,
         args: &[OpTy<'tcx>],
         dest: &MPlaceTy<'tcx>,
-    ) -> InterpResult<'tcx, EmulateItemResult> {
+    ) -> InterpResult<'tcx, bool> {
         let this = self.eval_context_mut();
         // Prefix should have already been checked.
         let unprefixed_name = link_name.as_str().strip_prefix("llvm.aarch64.").unwrap();
@@ -273,8 +273,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                 this.write_scalar(Scalar::from_u128(result), &dest)?;
             }
 
-            _ => return interp_ok(EmulateItemResult::NotSupported),
+            _ => return interp_ok(false),
         }
-        interp_ok(EmulateItemResult::NeedsReturn)
+        interp_ok(true)
     }
 }