diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index ec17e72900dfb..ce780647662b8 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -440,18 +440,6 @@ pub(crate) fn codegen_terminator_call<'tcx>( } } - if fx.tcx.symbol_name(instance).name.starts_with("llvm.") { - crate::intrinsics::codegen_llvm_intrinsic_call( - fx, - fx.tcx.symbol_name(instance).name, - args, - ret_place, - target, - source_info.span, - ); - return; - } - match instance.def { InstanceKind::Intrinsic(_) => { match crate::intrinsics::codegen_intrinsic_call( @@ -466,6 +454,17 @@ pub(crate) fn codegen_terminator_call<'tcx>( Err(instance) => Some(instance), } } + InstanceKind::LlvmIntrinsic(_) => { + crate::intrinsics::codegen_llvm_intrinsic_call( + fx, + fx.tcx.symbol_name(instance).name, + args, + ret_place, + target, + source_info.span, + ); + return; + } // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, // it is `func returning noop future` InstanceKind::Shim(ShimKind::DropGlue(_, None)) => { diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 0b86c421d16b5..01c132fd09ee5 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -892,12 +892,8 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ) -> bool { - fn is_llvm_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - if let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name { - name.as_str().starts_with("llvm.") - } else { - false - } + if let ty::InstanceKind::LlvmIntrinsic(_) = instance.def { + return false; } fn is_extern_call_to_local_crate<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> bool { @@ -910,7 +906,6 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>( let def_id = instance.def_id(); !def_id.is_local() && tcx.is_compiler_builtins(LOCAL_CRATE) - && !is_llvm_intrinsic(tcx, def_id) && !tcx.should_codegen_locally(instance) && !is_extern_call_to_local_crate(tcx, instance) } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index fad93de43b272..c05b89a552631 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1112,8 +1112,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; if let Some(instance) = instance + && let ty::InstanceKind::LlvmIntrinsic(_) = instance.def && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name - && name.as_str().starts_with("llvm.") // This is the only LLVM intrinsic we use that unwinds // FIXME either add unwind support to codegen_llvm_intrinsic_call or replace usage of // this intrinsic with something else diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs index 7c5f867929f04..b4f54f4c79f51 100644 --- a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs @@ -105,6 +105,16 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine { unimplemented!() } + fn call_llvm_intrinsic( + _ecx: &mut InterpCx<'tcx, Self>, + _instance: ty::Instance<'tcx>, + _args: &[interpret::OpTy<'tcx, Self::Provenance>], + _destination: &interpret::PlaceTy<'tcx, Self::Provenance>, + _target: Option, + ) -> interpret::InterpResult<'tcx> { + unimplemented!() + } + fn assert_panic( _ecx: &mut InterpCx<'tcx, Self>, _msg: &rustc_middle::mir::AssertMessage<'tcx>, diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index c5da3253ef2c7..545a3a3ff523a 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -745,6 +745,18 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { interp_ok(None) } + fn call_llvm_intrinsic( + ecx: &mut InterpCx<'tcx, Self>, + instance: ty::Instance<'tcx>, + _args: &[OpTy<'tcx>], + _dest: &PlaceTy<'tcx, Self::Provenance>, + _target: Option, + ) -> InterpResult<'tcx> { + let intrinsic_name = ecx.tcx.codegen_fn_attrs(instance.def_id()).symbol_name.unwrap(); + + throw_unsup_format!("LLVM intrinsic `{intrinsic_name}` is not supported at compile-time"); + } + fn assert_panic( ecx: &mut InterpCx<'tcx, Self>, msg: &AssertMessage<'tcx>, diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index e2fe8c3a79690..80cd892c799cd 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -652,6 +652,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } } + ty::InstanceKind::LlvmIntrinsic(_) => { + // FIXME: Should `InPlace` arguments be reset to uninit? + M::call_llvm_intrinsic( + self, + instance, + &Self::copy_fn_args(args), + destination, + target, + ) + } ty::InstanceKind::Shim(ty::ShimKind::VTable(..)) | ty::InstanceKind::Shim(ty::ShimKind::Reify(..)) | ty::InstanceKind::Shim(ty::ShimKind::ClosureOnce { .. }) diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index debcc2ac338b5..40251cbc8155b 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -247,6 +247,16 @@ pub trait Machine<'tcx>: Sized { unwind: mir::UnwindAction, ) -> InterpResult<'tcx, Option>>; + /// Directly process an LLVM intrinsic without pushing a stack frame. It is the hook's + /// responsibility to advance the instruction pointer as appropriate. + fn call_llvm_intrinsic( + ecx: &mut InterpCx<'tcx, Self>, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx, Self::Provenance>], + destination: &PlaceTy<'tcx, Self::Provenance>, + target: Option, + ) -> InterpResult<'tcx>; + /// Check whether the given function may be executed on the current machine, in terms of the /// target features is requires. fn check_fn_target_features( diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 22f2d5c1afcac..56f6ac4368d5f 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -347,6 +347,7 @@ macro_rules! make_mir_visitor { ty::InstanceKind::Item(_def_id) => {} ty::InstanceKind::Intrinsic(_def_id) + | ty::InstanceKind::LlvmIntrinsic(_def_id) | ty::InstanceKind::Shim(ty::ShimKind::VTable(_def_id)) | ty::InstanceKind::Shim(ty::ShimKind::Reify(_def_id, _)) | ty::InstanceKind::Virtual(_def_id, _) diff --git a/compiler/rustc_middle/src/mono.rs b/compiler/rustc_middle/src/mono.rs index 6df67257a2ea7..bad6986a2630c 100644 --- a/compiler/rustc_middle/src/mono.rs +++ b/compiler/rustc_middle/src/mono.rs @@ -524,6 +524,7 @@ impl<'tcx> CodegenUnit<'tcx> { MonoItem::Fn(ref instance) => match instance.def { InstanceKind::Item(def) => def.as_local().map(|_| def), InstanceKind::Intrinsic(..) + | InstanceKind::LlvmIntrinsic(..) | InstanceKind::Virtual(..) | InstanceKind::Shim(ShimKind::VTable(..)) | InstanceKind::Shim(ShimKind::Reify(..)) diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index c59e71b19be78..6a04357827360 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -70,11 +70,18 @@ pub enum InstanceKind<'tcx> { /// An intrinsic `fn` item (with`#[rustc_intrinsic]`). /// - /// Alongside `Virtual`, this is the only `InstanceKind` that does not have its own callable MIR. - /// Instead, codegen and const eval "magically" evaluate calls to intrinsics purely in the - /// caller. + /// Alongside `LlvmIntrinsic` and `Virtual`, this is the only `InstanceKind` + /// that does not have its own callable MIR. Instead, codegen and const eval + /// "magically" evaluate calls to intrinsics purely in the caller. Intrinsic(DefId), + /// An LLVM intrinsic `fn` item (with `extern "unadjusted"`). + /// + /// Alongside `Intrinsic` and `Virtual`, this is the only `InstanceKind` + /// that does not have its own callable MIR. Instead, codegen and const eval + /// "magically" evaluate calls to LLVM intrinsics purely in the caller. + LlvmIntrinsic(DefId), + /// Dynamic dispatch to `::fn`. /// /// This `InstanceKind` may have a callable MIR as the default implementation. @@ -253,7 +260,8 @@ impl<'tcx> InstanceKind<'tcx> { match self { InstanceKind::Item(def_id) | InstanceKind::Virtual(def_id, _) - | InstanceKind::Intrinsic(def_id) => def_id, + | InstanceKind::Intrinsic(def_id) + | InstanceKind::LlvmIntrinsic(def_id) => def_id, InstanceKind::Shim(shim) => shim.def_id(), } } @@ -262,7 +270,9 @@ impl<'tcx> InstanceKind<'tcx> { pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option { match self { InstanceKind::Item(def) => Some(def), - InstanceKind::Virtual(..) | InstanceKind::Intrinsic(..) => None, + InstanceKind::Virtual(..) + | InstanceKind::Intrinsic(..) + | InstanceKind::LlvmIntrinsic(..) => None, InstanceKind::Shim(shim) => shim.def_id_if_not_guaranteed_local_codegen(), } } @@ -280,7 +290,9 @@ impl<'tcx> InstanceKind<'tcx> { DefPathData::Ctor | DefPathData::Closure ), InstanceKind::Shim(shim) => shim.requires_inline(), - InstanceKind::Virtual(..) | InstanceKind::Intrinsic(..) => true, + InstanceKind::Virtual(..) + | InstanceKind::Intrinsic(..) + | InstanceKind::LlvmIntrinsic(..) => true, } } @@ -308,7 +320,10 @@ impl<'tcx> InstanceKind<'tcx> { /// body should perform necessary instantiations. pub fn has_polymorphic_mir_body(&self) -> bool { match *self { - InstanceKind::Item(_) | InstanceKind::Intrinsic(..) | InstanceKind::Virtual(..) => true, + InstanceKind::Item(_) + | InstanceKind::Intrinsic(..) + | InstanceKind::LlvmIntrinsic(..) + | InstanceKind::Virtual(..) => true, InstanceKind::Shim(shim) => shim.has_polymorphic_mir_body(), } } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index c482e6bb87345..4c14858101bf1 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1833,7 +1833,9 @@ impl<'tcx> TyCtxt<'tcx> { _ => self.optimized_mir(def), } } - ty::InstanceKind::Intrinsic(..) => bug!("intrinsics have no instance MIR"), + ty::InstanceKind::Intrinsic(..) | ty::InstanceKind::LlvmIntrinsic(..) => { + bug!("intrinsics have no instance MIR") + } ty::InstanceKind::Virtual(..) => bug!("virtual dispatches have no instance MIR"), ty::InstanceKind::Shim(shim) => self.mir_shims(shim), }; diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs index bec1ffc642769..ccdac57cc8dcd 100644 --- a/compiler/rustc_middle/src/ty/print/mod.rs +++ b/compiler/rustc_middle/src/ty/print/mod.rs @@ -368,6 +368,7 @@ impl<'tcx, P: Printer<'tcx> + std::fmt::Write> Print

for ty::Instance<'tcx> { match self.def { ty::InstanceKind::Item(_) => {} ty::InstanceKind::Intrinsic(_) => cx.write_str(" - intrinsic")?, + ty::InstanceKind::LlvmIntrinsic(_) => cx.write_str(" - LLVM intrinsic")?, ty::InstanceKind::Virtual(_, num) => cx.write_str(&format!(" - virtual#{num}"))?, ty::InstanceKind::Shim(shim) => { cx.write_str(" - ")?; diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index c36b687111c01..df47eb9c3f9a2 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -730,7 +730,7 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>( } } // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => { + InstanceKind::Intrinsic(_) | InstanceKind::LlvmIntrinsic(_) | InstanceKind::Virtual(..) => { debug!("instance without MIR (intrinsic / virtual)"); return Err("implementation limitation -- cannot inline intrinsic"); } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index b1a9258d48bb3..c3bdef3e43cf8 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -21,7 +21,9 @@ fn should_recurse<'tcx>(tcx: TyCtxt<'tcx>, callee: ty::Instance<'tcx>) -> bool { } // These have no own callable MIR. - InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => return false, + InstanceKind::Intrinsic(_) | InstanceKind::LlvmIntrinsic(_) | InstanceKind::Virtual(..) => { + return false; + } // These have MIR and if that MIR is inlined, instantiated and then inlining is run // again, a function item can end up getting inlined. Thus we'll be able to cause diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 9b18b22399ae8..6fb789939f416 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1018,7 +1018,9 @@ fn visit_instance_use<'tcx>( } match instance.def { - ty::InstanceKind::Virtual(..) | ty::InstanceKind::Intrinsic(_) => { + ty::InstanceKind::Virtual(..) + | ty::InstanceKind::Intrinsic(_) + | ty::InstanceKind::LlvmIntrinsic(_) => { if !is_direct_call { bug!("{:?} being reified", instance); } diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index bf4a2bdd15107..1e8e1486f107f 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -631,6 +631,7 @@ fn characteristic_def_id_of_mono_item<'tcx>( let def_id = match instance.def { ty::InstanceKind::Item(def) => def, ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::LlvmIntrinsic(..) | ty::InstanceKind::Virtual(..) | ty::InstanceKind::Shim(ty::ShimKind::VTable(..)) | ty::InstanceKind::Shim(ty::ShimKind::Reify(..)) @@ -810,6 +811,7 @@ fn mono_item_visibility<'tcx>( | InstanceKind::Shim(ShimKind::FnPtr(..)) | InstanceKind::Virtual(..) | InstanceKind::Intrinsic(..) + | InstanceKind::LlvmIntrinsic(..) | InstanceKind::Shim(ShimKind::ClosureOnce { .. }) | InstanceKind::Shim(ShimKind::ConstructCoroutineInClosure { .. }) | InstanceKind::Shim(ShimKind::DropGlue(..)) diff --git a/compiler/rustc_public/src/mir/mono.rs b/compiler/rustc_public/src/mir/mono.rs index ab939a5535149..b3caee5095733 100644 --- a/compiler/rustc_public/src/mir/mono.rs +++ b/compiler/rustc_public/src/mir/mono.rs @@ -32,6 +32,8 @@ pub enum InstanceKind { Item, /// A compiler intrinsic function. Intrinsic, + /// An LLVM intrinsic function. + LlvmIntrinsic, /// A virtual function definition stored in a VTable. /// The `idx` field indicates the position in the VTable for this instance. Virtual { idx: usize }, @@ -113,7 +115,10 @@ impl Instance { InstanceKind::Intrinsic => { Some(with(|context| context.intrinsic(self.def.def_id()).unwrap().fn_name())) } - InstanceKind::Item | InstanceKind::Virtual { .. } | InstanceKind::Shim => None, + InstanceKind::LlvmIntrinsic + | InstanceKind::Item + | InstanceKind::Virtual { .. } + | InstanceKind::Shim => None, } } diff --git a/compiler/rustc_public/src/unstable/convert/stable/ty.rs b/compiler/rustc_public/src/unstable/convert/stable/ty.rs index e0e212ee47d59..436fdd766899f 100644 --- a/compiler/rustc_public/src/unstable/convert/stable/ty.rs +++ b/compiler/rustc_public/src/unstable/convert/stable/ty.rs @@ -970,6 +970,7 @@ impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> { let kind = match self.def { ty::InstanceKind::Item(..) => crate::mir::mono::InstanceKind::Item, ty::InstanceKind::Intrinsic(..) => crate::mir::mono::InstanceKind::Intrinsic, + ty::InstanceKind::LlvmIntrinsic(..) => crate::mir::mono::InstanceKind::LlvmIntrinsic, ty::InstanceKind::Virtual(_def_id, idx) => { crate::mir::mono::InstanceKind::Virtual { idx } } diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 34a3a96f89c40..d9069ac127729 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -1,5 +1,6 @@ use rustc_errors::ErrorGuaranteed; use rustc_hir::LangItem; +use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_infer::infer::TyCtxtInferExt; use rustc_middle::bug; @@ -91,6 +92,12 @@ fn resolve_instance_raw<'tcx>( } else if tcx.is_async_drop_in_place_coroutine(def_id) { let ty = args.type_at(0); ty::InstanceKind::Shim(ty::ShimKind::AsyncDropGlue(def_id, ty)) + } else if tcx.def_kind(def_id) == DefKind::Fn + && let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name + && name.as_str().starts_with("llvm.") + { + debug!(" => LLVM intrinsic"); + ty::InstanceKind::LlvmIntrinsic(def_id) } else { debug!(" => free item"); ty::InstanceKind::Item(def_id) diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs index bc171f8fdb87c..7d7f4e6be7fc3 100644 --- a/src/tools/miri/src/intrinsics/mod.rs +++ b/src/tools/miri/src/intrinsics/mod.rs @@ -10,7 +10,7 @@ pub use self::atomic::AtomicRmwOp; use rand::RngExt; use rustc_abi::Size; use rustc_middle::{mir, ty}; -use rustc_span::Symbol; +use rustc_span::{Symbol, sym}; use self::atomic::EvalContextExt as _; use self::math::EvalContextExt as _; diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index cc6a15d3f7115..cdf7dcda2444f 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -1300,6 +1300,17 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { ecx.call_intrinsic(instance, args, dest, ret, unwind) } + #[inline(always)] + fn call_llvm_intrinsic( + ecx: &mut MiriInterpCx<'tcx>, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + dest: &PlaceTy<'tcx>, + ret: Option, + ) -> InterpResult<'tcx, ()> { + ecx.call_llvm_intrinsic(instance, args, dest, ret) + } + #[inline(always)] fn assert_panic( ecx: &mut MiriInterpCx<'tcx>, diff --git a/src/tools/miri/src/shims/aarch64.rs b/src/tools/miri/src/shims/aarch64.rs index 290b8e108f3e6..afaace872b52b 100644 --- a/src/tools/miri/src/shims/aarch64.rs +++ b/src/tools/miri/src/shims/aarch64.rs @@ -13,7 +13,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. let unprefixed_name = link_name.as_str().strip_prefix("llvm.aarch64.").unwrap(); @@ -273,8 +273,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_u128(result), &dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index a1a341ffff3e0..cef23e6648964 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -241,6 +241,109 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { Some(instance) => interp_ok(Some((this.load_mir(instance.def, None)?, instance))), } } + + // FIXME move this and the LLVM intrinsic impls to the intrinsics module + fn call_llvm_intrinsic( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + dest: &PlaceTy<'tcx>, + ret: Option, + ) -> InterpResult<'tcx> { + let this = self.eval_context_mut(); + + let link_name = this.tcx.codegen_fn_attrs(instance.def_id()).symbol_name.unwrap(); + + // FIXME: avoid allocating memory + let dest = this.force_allocation(dest)?; + + let handled = match link_name.as_str() { + // LLVM intrinsics + "llvm.prefetch.p0" => { + let [p, rw, loc, ty] = this.check_shim_sig_unadjusted(link_name, args)?; + + let _ = this.read_pointer(p)?; + let rw = this.read_scalar(rw)?.to_i32()?; + let loc = this.read_scalar(loc)?.to_i32()?; + let ty = this.read_scalar(ty)?.to_i32()?; + + if ty == 1 { + // Data cache prefetch. + // Notably, we do not have to check the pointer, this operation is never UB! + + if !matches!(rw, 0 | 1) { + throw_unsup_format!("invalid `rw` value passed to `llvm.prefetch`: {}", rw); + } + if !matches!(loc, 0..=3) { + throw_unsup_format!( + "invalid `loc` value passed to `llvm.prefetch`: {}", + loc + ); + } + } else { + throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty); + } + + true + } + // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm + // `{i,u}8x16_popcnt` functions. + name if name.starts_with("llvm.ctpop.v") + && this.tcx.sess.target.endian == Endian::Little => + { + let [op] = this.check_shim_sig_unadjusted(link_name, args)?; + + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(&dest)?; + + assert_eq!(dest_len, op_len); + + for i in 0..dest_len { + let op = this.read_immediate(&this.project_index(&op, i)?)?; + // Use `to_uint` to get a zero-extended `u128`. Those + // extra zeros will not affect `count_ones`. + let res = op.to_scalar().to_uint(op.layout.size)?.count_ones(); + + this.write_scalar( + Scalar::from_uint(res, op.layout.size), + &this.project_index(&dest, i)?, + )?; + } + + true + } + + // Target-specific shims + name if name.starts_with("llvm.x86.") + && matches!(this.tcx.sess.target.arch, Arch::X86 | Arch::X86_64) + && this.tcx.sess.target.endian == Endian::Little => + shims::x86::EvalContextExt::emulate_x86_intrinsic(this, link_name, args, &dest)?, + name if name.starts_with("llvm.aarch64.") + && this.tcx.sess.target.arch == Arch::AArch64 + && this.tcx.sess.target.endian == Endian::Little => + shims::aarch64::EvalContextExt::emulate_aarch64_intrinsic( + this, link_name, args, &dest, + )?, + name if name.starts_with("llvm.loongarch.") + && matches!(this.tcx.sess.target.arch, Arch::LoongArch32 | Arch::LoongArch64) + && this.tcx.sess.target.endian == Endian::Little => + shims::loongarch::EvalContextExt::emulate_loongarch_intrinsic( + this, link_name, args, &dest, + )?, + _ => false, + }; + + // The rest either implements the logic, or falls back to `lookup_exported_symbol`. + if handled { + trace!("{:?}", this.dump_place(&dest.clone().into())); + this.return_to_block(ret) + } else { + throw_machine_stop!(TerminationInfo::UnsupportedForeignItem(format!( + "can't call LLVM intrinsic `{link_name}` on architecture `{arch}`", + arch = this.tcx.sess.target.arch, + ))); + } + } } impl<'tcx> EvalContextExtPriv<'tcx> for crate::MiriInterpCx<'tcx> {} @@ -803,83 +906,6 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_pointer(ptr_dest, dest)?; } - // LLVM intrinsics - "llvm.prefetch.p0" => { - let [p, rw, loc, ty] = this.check_shim_sig_unadjusted(link_name, args)?; - - let _ = this.read_pointer(p)?; - let rw = this.read_scalar(rw)?.to_i32()?; - let loc = this.read_scalar(loc)?.to_i32()?; - let ty = this.read_scalar(ty)?.to_i32()?; - - if ty == 1 { - // Data cache prefetch. - // Notably, we do not have to check the pointer, this operation is never UB! - - if !matches!(rw, 0 | 1) { - throw_unsup_format!("invalid `rw` value passed to `llvm.prefetch`: {}", rw); - } - if !matches!(loc, 0..=3) { - throw_unsup_format!( - "invalid `loc` value passed to `llvm.prefetch`: {}", - loc - ); - } - } else { - throw_unsup_format!("unsupported `llvm.prefetch` type argument: {}", ty); - } - } - // Used to implement the x86 `_mm{,256,512}_popcnt_epi{8,16,32,64}` and wasm - // `{i,u}8x16_popcnt` functions. - name if name.starts_with("llvm.ctpop.v") - && this.tcx.sess.target.endian == Endian::Little => - { - let [op] = this.check_shim_sig_unadjusted(link_name, args)?; - - let (op, op_len) = this.project_to_simd(op)?; - let (dest, dest_len) = this.project_to_simd(dest)?; - - assert_eq!(dest_len, op_len); - - for i in 0..dest_len { - let op = this.read_immediate(&this.project_index(&op, i)?)?; - // Use `to_uint` to get a zero-extended `u128`. Those - // extra zeros will not affect `count_ones`. - let res = op.to_scalar().to_uint(op.layout.size)?.count_ones(); - - this.write_scalar( - Scalar::from_uint(res, op.layout.size), - &this.project_index(&dest, i)?, - )?; - } - } - - // Target-specific shims - name if name.starts_with("llvm.x86.") - && matches!(this.tcx.sess.target.arch, Arch::X86 | Arch::X86_64) - && this.tcx.sess.target.endian == Endian::Little => - { - return shims::x86::EvalContextExt::emulate_x86_intrinsic( - this, link_name, args, dest, - ); - } - name if name.starts_with("llvm.aarch64.") - && this.tcx.sess.target.arch == Arch::AArch64 - && this.tcx.sess.target.endian == Endian::Little => - { - return shims::aarch64::EvalContextExt::emulate_aarch64_intrinsic( - this, link_name, args, dest, - ); - } - name if name.starts_with("llvm.loongarch.") - && matches!(this.tcx.sess.target.arch, Arch::LoongArch32 | Arch::LoongArch64) - && this.tcx.sess.target.endian == Endian::Little => - { - return shims::loongarch::EvalContextExt::emulate_loongarch_intrinsic( - this, link_name, args, dest, - ); - } - // Fallback to shims in submodules. _ => { // Math shims diff --git a/src/tools/miri/src/shims/loongarch.rs b/src/tools/miri/src/shims/loongarch.rs index 461dd08dd2157..d45581c4c09f9 100644 --- a/src/tools/miri/src/shims/loongarch.rs +++ b/src/tools/miri/src/shims/loongarch.rs @@ -11,7 +11,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. let unprefixed_name = link_name.as_str().strip_prefix("llvm.loongarch.").unwrap(); @@ -67,8 +67,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let result = compute_crc32(crc, data, bit_size, polynomial); this.write_scalar(Scalar::from_u32(result), dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/aesni.rs b/src/tools/miri/src/shims/x86/aesni.rs index 0ae76b62d2624..b706639c75ead 100644 --- a/src/tools/miri/src/shims/x86/aesni.rs +++ b/src/tools/miri/src/shims/x86/aesni.rs @@ -10,7 +10,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "aes")?; // Prefix should have already been checked. @@ -112,9 +112,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } // TODO: Implement the `llvm.x86.aesni.aeskeygenassist` when possible // with an external crate. - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/avx.rs b/src/tools/miri/src/shims/x86/avx.rs index 25fe93a20f9db..aad278948338f 100644 --- a/src/tools/miri/src/shims/x86/avx.rs +++ b/src/tools/miri/src/shims/x86/avx.rs @@ -14,7 +14,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "avx")?; // Prefix should have already been checked. @@ -243,8 +243,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // The only thing that needs to be ensured is the correct calling convention. let [] = this.check_shim_sig_unadjusted(link_name, args)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/avx2.rs b/src/tools/miri/src/shims/x86/avx2.rs index 160bce2dec98b..3ee66862cc5cf 100644 --- a/src/tools/miri/src/shims/x86/avx2.rs +++ b/src/tools/miri/src/shims/x86/avx2.rs @@ -14,7 +14,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "avx2")?; // Prefix should have already been checked. @@ -216,8 +216,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { pmaddwd(this, left, right, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/avx512.rs b/src/tools/miri/src/shims/x86/avx512.rs index 6dde7558d829c..d69b1ff112753 100644 --- a/src/tools/miri/src/shims/x86/avx512.rs +++ b/src/tools/miri/src/shims/x86/avx512.rs @@ -12,7 +12,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.avx512.").unwrap(); @@ -178,9 +178,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { packusdw(this, a, b, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/bmi.rs b/src/tools/miri/src/shims/x86/bmi.rs index b14d69199ab6b..4e82d7a6693fa 100644 --- a/src/tools/miri/src/shims/x86/bmi.rs +++ b/src/tools/miri/src/shims/x86/bmi.rs @@ -10,7 +10,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. @@ -29,7 +29,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.expect_target_feature_for_intrinsic(link_name, target_feature)?; if is_64_bit && this.tcx.sess.target.arch != Arch::X86_64 { - return interp_ok(EmulateItemResult::NotSupported); + return interp_ok(false); } let [left, right] = this.check_shim_sig_unadjusted(link_name, args)?; @@ -92,7 +92,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } result } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), }; let result = if is_64_bit { @@ -102,6 +102,6 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { }; this.write_scalar(result, dest)?; - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/gfni.rs b/src/tools/miri/src/shims/x86/gfni.rs index 562d9f76ffc6e..71ba283992c62 100644 --- a/src/tools/miri/src/shims/x86/gfni.rs +++ b/src/tools/miri/src/shims/x86/gfni.rs @@ -9,7 +9,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. @@ -58,9 +58,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_u8(gf2p8_mul(left, right)), &dest)?; } } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/mod.rs b/src/tools/miri/src/shims/x86/mod.rs index d76d35cb722bc..c320d5af9b976 100644 --- a/src/tools/miri/src/shims/x86/mod.rs +++ b/src/tools/miri/src/shims/x86/mod.rs @@ -30,7 +30,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); // Prefix should have already been checked. let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.").unwrap(); @@ -42,7 +42,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-8/subborrow-u32-subborrow-u64.html "addcarry.32" | "addcarry.64" | "subborrow.32" | "subborrow.64" => { if unprefixed_name.ends_with("64") && this.tcx.sess.target.arch != Arch::X86_64 { - return interp_ok(EmulateItemResult::NotSupported); + return interp_ok(false); } let [cb_in, a, b] = this.check_shim_sig_unadjusted(link_name, args)?; @@ -147,9 +147,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ); } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sha.rs b/src/tools/miri/src/shims/x86/sha.rs index 982e3a08e4826..3809e23ce8bb4 100644 --- a/src/tools/miri/src/shims/x86/sha.rs +++ b/src/tools/miri/src/shims/x86/sha.rs @@ -15,7 +15,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sha")?; // Prefix should have already been checked. @@ -104,9 +104,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let result = sha256msg2(a, b); write(this, &dest, result)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sse.rs b/src/tools/miri/src/shims/x86/sse.rs index 81078bda99c8c..c6e0c7d9a7b59 100644 --- a/src/tools/miri/src/shims/x86/sse.rs +++ b/src/tools/miri/src/shims/x86/sse.rs @@ -14,7 +14,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sse")?; // Prefix should have already been checked. @@ -171,8 +171,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_immediate(*res, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sse2.rs b/src/tools/miri/src/shims/x86/sse2.rs index 2ee4d276a3af6..f0017b443d0a3 100644 --- a/src/tools/miri/src/shims/x86/sse2.rs +++ b/src/tools/miri/src/shims/x86/sse2.rs @@ -14,7 +14,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sse2")?; // Prefix should have already been checked. @@ -272,8 +272,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { pmaddwd(this, left, right, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sse3.rs b/src/tools/miri/src/shims/x86/sse3.rs index 5890af49088df..c470ad54b20a4 100644 --- a/src/tools/miri/src/shims/x86/sse3.rs +++ b/src/tools/miri/src/shims/x86/sse3.rs @@ -9,7 +9,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sse3")?; // Prefix should have already been checked. @@ -28,8 +28,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.mem_copy(src_ptr, dest.ptr(), dest.layout.size, /*nonoverlapping*/ true)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sse41.rs b/src/tools/miri/src/shims/x86/sse41.rs index aee2e115a0ac5..68aa0f71160e2 100644 --- a/src/tools/miri/src/shims/x86/sse41.rs +++ b/src/tools/miri/src/shims/x86/sse41.rs @@ -10,7 +10,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sse4.1")?; // Prefix should have already been checked. @@ -155,8 +155,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(Scalar::from_i32(res.into()), dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/sse42.rs b/src/tools/miri/src/shims/x86/sse42.rs index f6152c60f8370..af28897305bb9 100644 --- a/src/tools/miri/src/shims/x86/sse42.rs +++ b/src/tools/miri/src/shims/x86/sse42.rs @@ -279,7 +279,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "sse4.2")?; // Prefix should have already been checked. @@ -428,7 +428,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { }; if bit_size == 64 && this.tcx.sess.target.arch != Arch::X86_64 { - return interp_ok(EmulateItemResult::NotSupported); + return interp_ok(false); } let [left, right] = this.check_shim_sig_unadjusted(link_name, args)?; @@ -460,8 +460,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { this.write_scalar(result, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } } diff --git a/src/tools/miri/src/shims/x86/ssse3.rs b/src/tools/miri/src/shims/x86/ssse3.rs index 5b4746e5b1a04..dadd0cc7315a8 100644 --- a/src/tools/miri/src/shims/x86/ssse3.rs +++ b/src/tools/miri/src/shims/x86/ssse3.rs @@ -11,7 +11,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { link_name: Symbol, args: &[OpTy<'tcx>], dest: &MPlaceTy<'tcx>, - ) -> InterpResult<'tcx, EmulateItemResult> { + ) -> InterpResult<'tcx, bool> { let this = self.eval_context_mut(); this.expect_target_feature_for_intrinsic(link_name, "ssse3")?; // Prefix should have already been checked. @@ -67,8 +67,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { psign(this, left, right, dest)?; } - _ => return interp_ok(EmulateItemResult::NotSupported), + _ => return interp_ok(false), } - interp_ok(EmulateItemResult::NeedsReturn) + interp_ok(true) } }