diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index c9f84066da8d..ff030470634f 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -2942,14 +2942,11 @@ impl MachInstEmit for Inst { let offset = sink.cur_offset(); sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } if info.callee_pop_size > 0 { @@ -2989,14 +2986,11 @@ impl MachInstEmit for Inst { let offset = sink.cur_offset(); sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } if info.callee_pop_size > 0 { @@ -3035,7 +3029,7 @@ impl MachInstEmit for Inst { // for the target, but rather a function relocation. sink.add_reloc(Reloc::Arm64Call, &info.dest, 0); sink.put4(enc_jump26(0b000101, 0)); - sink.add_call_site(); + sink.add_call_site(&[]); // `emit_return_call_common_sequence` emits an island if // necessary, so we can safely disable the worst-case-size check @@ -3050,7 +3044,7 @@ impl MachInstEmit for Inst { targets: vec![], } .emit(sink, emit_info, state); - sink.add_call_site(); + sink.add_call_site(&[]); // `emit_return_call_common_sequence` emits an island if // necessary, so we can safely disable the worst-case-size check diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index d0b741467758..bf226a0e7dda 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -182,14 +182,11 @@ fn pulley_emit

( let offset = sink.cur_offset(); sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } let adjust = -i32::try_from(info.callee_pop_size).unwrap(); @@ -226,14 +223,10 @@ fn pulley_emit

( sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } let adjust = -i32::try_from(info.callee_pop_size).unwrap(); @@ -295,7 +288,7 @@ fn pulley_emit

( let offset = sink.cur_offset(); sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); + sink.add_call_site(&[]); // If a callee pop is happening here that means that something has // messed up, these are expected to be "very simple" signatures. diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs index 966176c6374e..805f88ac8690 100644 --- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs +++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs @@ -1123,7 +1123,6 @@ impl Inst { } &Inst::Call { ref info } => { - sink.add_call_site(); sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0); Inst::construct_auipc_and_jalr(Some(writable_link_reg()), writable_link_reg(), 0) @@ -1135,12 +1134,10 @@ impl Inst { sink.push_user_stack_map(state, offset, s); } - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); @@ -1181,14 +1178,10 @@ impl Inst { sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } let callee_pop_size = i32::try_from(info.callee_pop_size).unwrap(); @@ -1220,7 +1213,7 @@ impl Inst { &Inst::ReturnCall { ref info } => { emit_return_call_common_sequence(sink, emit_info, state, info); - sink.add_call_site(); + sink.add_call_site(&[]); sink.add_reloc(Reloc::RiscvCallPlt, &info.dest, 0); Inst::construct_auipc_and_jalr(None, writable_spilltmp_reg(), 0) .into_iter() diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs index aa13cadeb397..45cb6b88c705 100644 --- a/cranelift/codegen/src/isa/s390x/inst/emit.rs +++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs @@ -3216,14 +3216,11 @@ impl Inst { sink.push_user_stack_map(state, offset, s); } put(sink, enc); - sink.add_call_site(); - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } state.nominal_sp_offset -= info.callee_pop_size; @@ -3264,7 +3261,7 @@ impl Inst { } }; put(sink, enc); - sink.add_call_site(); + sink.add_call_site(&[]); } &Inst::ElfTlsGetOffset { ref symbol, .. } => { let opcode = 0xc05; // BRASL @@ -3281,7 +3278,7 @@ impl Inst { } put(sink, &enc_ril_b(opcode, gpr(14), 0)); - sink.add_call_site(); + sink.add_call_site(&[]); } &Inst::Args { .. } => {} &Inst::Rets { .. } => {} diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs index b086c215c92d..56ffc3e11ca4 100644 --- a/cranelift/codegen/src/isa/x64/inst/emit.rs +++ b/cranelift/codegen/src/isa/x64/inst/emit.rs @@ -1611,14 +1611,11 @@ pub(crate) fn emit( // beginning of the immediate field. emit_reloc(sink, Reloc::X86CallPCRel4, &call_info.dest, -4); sink.put4(0); - sink.add_call_site(); - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = call_info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } // Reclaim the outgoing argument area that was released by the callee, to ensure that @@ -1663,7 +1660,7 @@ pub(crate) fn emit( // beginning of the immediate field. emit_reloc(sink, Reloc::X86CallPCRel4, &call_info.dest, -4); sink.put4(0); - sink.add_call_site(); + sink.add_call_site(&[]); } Inst::ReturnCallUnknown { info: call_info } => { @@ -1675,7 +1672,7 @@ pub(crate) fn emit( target: RegMem::reg(callee), } .emit(sink, info, state); - sink.add_call_site(); + sink.add_call_site(&[]); } Inst::CallUnknown { @@ -1717,14 +1714,10 @@ pub(crate) fn emit( sink.push_user_stack_map(state, offset, s); } - sink.add_call_site(); - - // Add exception info, if any, at this point (which will - // be the return address on stack). if let Some(try_call) = call_info.try_call_info.as_ref() { - for &(tag, label) in &try_call.exception_dests { - sink.add_exception_handler(tag, label); - } + sink.add_call_site(&try_call.exception_dests); + } else { + sink.add_call_site(&[]); } // Reclaim the outgoing argument area that was released by the callee, to ensure that diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs index 2b1ac7ca07e3..ddfbc05d8f5c 100644 --- a/cranelift/codegen/src/machinst/buffer.rs +++ b/cranelift/codegen/src/machinst/buffer.rs @@ -172,7 +172,7 @@ use crate::binemit::{Addend, CodeOffset, Reloc}; use crate::ir::function::FunctionParameters; -use crate::ir::{ExternalName, RelSourceLoc, SourceLoc, TrapCode}; +use crate::ir::{ExceptionTag, ExternalName, RelSourceLoc, SourceLoc, TrapCode}; use crate::isa::unwind::UnwindInst; use crate::machinst::{ BlockIndex, MachInstLabelUse, TextSectionBuilder, VCodeConstant, VCodeConstants, VCodeInst, @@ -180,6 +180,7 @@ use crate::machinst::{ use crate::trace; use crate::{ir, MachInstEmitState}; use crate::{timing, VCodeConstantData}; +use core::ops::Range; use cranelift_control::ControlPlane; use cranelift_entity::packed_option::PackedOption; use cranelift_entity::{entity_impl, PrimaryMap}; @@ -250,6 +251,8 @@ pub struct MachBuffer { traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. call_sites: SmallVec<[MachCallSite; 16]>, + /// Any exception-handler records referred to at call sites. + exception_handlers: SmallVec<[(PackedOption, MachLabel); 16]>, /// Any source location mappings referring to this code. srclocs: SmallVec<[MachSrcLoc; 64]>, /// Any user stack maps for this code. @@ -259,8 +262,6 @@ pub struct MachBuffer { user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, - /// Any exception handler targets at a given location. - exception_handlers: SmallVec<[(CodeOffset, PackedOption, MachLabel); 8]>, /// The current source location in progress (after `start_srcloc()` and /// before `end_srcloc()`). This is a (start_offset, src_loc) tuple. cur_srcloc: Option<(CodeOffset, RelSourceLoc)>, @@ -332,6 +333,7 @@ impl MachBufferFinalized { relocs: self.relocs, traps: self.traps, call_sites: self.call_sites, + exception_handlers: self.exception_handlers, srclocs: self .srclocs .into_iter() @@ -339,7 +341,6 @@ impl MachBufferFinalized { .collect(), user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, - exception_handlers: self.exception_handlers, alignment: self.alignment, } } @@ -363,6 +364,8 @@ pub struct MachBufferFinalized { pub(crate) traps: SmallVec<[MachTrap; 16]>, /// Any call site records referring to this code. pub(crate) call_sites: SmallVec<[MachCallSite; 16]>, + /// Any exception-handler records referred to at call sites. + pub(crate) exception_handlers: SmallVec<[(PackedOption, CodeOffset); 16]>, /// Any source location mappings referring to this code. pub(crate) srclocs: SmallVec<[T::MachSrcLocType; 64]>, /// Any user stack maps for this code. @@ -372,8 +375,6 @@ pub struct MachBufferFinalized { pub(crate) user_stack_maps: SmallVec<[(CodeOffset, u32, ir::UserStackMap); 8]>, /// Any unwind info at a given location. pub unwind_info: SmallVec<[(CodeOffset, UnwindInst); 8]>, - /// Any exception handler targets at a given location. - pub exception_handlers: SmallVec<[(CodeOffset, PackedOption, CodeOffset); 8]>, /// The required alignment of this buffer. pub alignment: u32, } @@ -425,7 +426,7 @@ pub struct OpenPatchRegion(usize); /// the [`PatchRegion::patch`] function can be used to get a mutable buffer to the instruction /// bytes, and the constants uses can be updated directly. pub struct PatchRegion { - range: std::ops::Range, + range: Range, } impl PatchRegion { @@ -445,10 +446,10 @@ impl MachBuffer { relocs: SmallVec::new(), traps: SmallVec::new(), call_sites: SmallVec::new(), + exception_handlers: SmallVec::new(), srclocs: SmallVec::new(), user_stack_maps: SmallVec::new(), unwind_info: SmallVec::new(), - exception_handlers: SmallVec::new(), cur_srcloc: None, label_offsets: SmallVec::new(), label_aliases: SmallVec::new(), @@ -1526,10 +1527,10 @@ impl MachBuffer { }) .collect(); - let exception_handlers = self + let finalized_exception_handlers = self .exception_handlers .iter() - .map(|&(off, tag, target)| (off, tag, self.resolve_label_offset(target))) + .map(|(tag, label)| (*tag, self.resolve_label_offset(*label))) .collect(); let mut srclocs = self.srclocs; @@ -1540,10 +1541,10 @@ impl MachBuffer { relocs: finalized_relocs, traps: self.traps, call_sites: self.call_sites, + exception_handlers: finalized_exception_handlers, srclocs, user_stack_maps: self.user_stack_maps, unwind_info: self.unwind_info, - exception_handlers, alignment, } } @@ -1616,10 +1617,18 @@ impl MachBuffer { }); } - /// Add a call-site record at the current offset. - pub fn add_call_site(&mut self) { + /// Add a call-site record at the current offset, optionally with exception handlers. + pub fn add_call_site( + &mut self, + exception_handlers: &[(PackedOption, MachLabel)], + ) { + let start = u32::try_from(self.exception_handlers.len()).unwrap(); + self.exception_handlers + .extend(exception_handlers.into_iter().copied()); + let end = u32::try_from(self.exception_handlers.len()).unwrap(); self.call_sites.push(MachCallSite { ret_addr: self.data.len() as CodeOffset, + exception_handler_range: start..end, }); } @@ -1628,16 +1637,6 @@ impl MachBuffer { self.unwind_info.push((self.cur_offset(), unwind)); } - /// Add an exception handler record at the current offset. - pub fn add_exception_handler( - &mut self, - tag: PackedOption, - target: MachLabel, - ) { - self.exception_handlers - .push((self.cur_offset(), tag, target)); - } - /// Set the `SourceLoc` for code from this offset until the offset at the /// next call to `end_srcloc()`. /// Returns the current [CodeOffset] and [RelSourceLoc]. @@ -1769,9 +1768,26 @@ impl MachBufferFinalized { mem::take(&mut self.user_stack_maps) } - /// Get the list of call sites for this code. - pub fn call_sites(&self) -> &[MachCallSite] { - &self.call_sites[..] + /// Get the list of call sites for this code, along with + /// associated exception handlers. + /// + /// Each item yielded by the returned iterator is a struct with: + /// + /// - The call site metadata record, with a `ret_addr` field + /// directly accessible and denoting the offset of the return + /// address into this buffer's code. + /// - The slice of pairs of exception tags and code offsets + /// denoting exception-handler entry points associated with this + /// call site. + pub fn call_sites(&self) -> impl Iterator> + '_ { + self.call_sites.iter().map(|call_site| { + let range = call_site.exception_handler_range.clone(); + let range = usize::try_from(range.start).unwrap()..usize::try_from(range.end).unwrap(); + FinalizedMachCallSite { + ret_addr: call_site.ret_addr, + exception_handlers: &self.exception_handlers[range], + } + }) } } @@ -1935,8 +1951,25 @@ pub struct MachTrap { derive(serde_derive::Serialize, serde_derive::Deserialize) )] pub struct MachCallSite { - /// The offset of the call's return address, *relative to the containing section*. + /// The offset of the call's return address, *relative to the + /// start of the buffer*. + pub ret_addr: CodeOffset, + + /// Range in `exception_handlers` corresponding to the exception + /// handlers for this callsite. + exception_handler_range: Range, +} + +/// A call site record resulting from a compilation. +#[derive(Clone, Debug, PartialEq)] +pub struct FinalizedMachCallSite<'a> { + /// The offset of the call's return address, *relative to the + /// start of the buffer*. pub ret_addr: CodeOffset, + + /// Exception handlers at this callsite, with target offsets + /// *relative to the start of the buffer*. + pub exception_handlers: &'a [(PackedOption, CodeOffset)], } /// A source-location mapping resulting from a compilation. @@ -2473,7 +2506,7 @@ mod test { let ctrl_plane = &mut Default::default(); let constants = Default::default(); - buf.reserve_labels_for_blocks(1); + buf.reserve_labels_for_blocks(3); buf.bind_label(label(0), ctrl_plane); buf.put1(1); @@ -2481,7 +2514,10 @@ mod test { buf.put1(2); buf.add_trap(TrapCode::INTEGER_OVERFLOW); buf.add_trap(TrapCode::INTEGER_DIVISION_BY_ZERO); - buf.add_call_site(); + buf.add_call_site(&[ + (None.into(), label(1)), + (Some(ExceptionTag::new(42)).into(), label(2)), + ]); buf.add_reloc( Reloc::Abs4, &ExternalName::User(UserExternalNameRef::new(0)), @@ -2494,10 +2530,14 @@ mod test { 1, ); buf.put1(4); + buf.bind_label(label(1), ctrl_plane); + buf.put1(0xff); + buf.bind_label(label(2), ctrl_plane); + buf.put1(0xff); let buf = buf.finish(&constants, ctrl_plane); - assert_eq!(buf.data(), &[1, 2, 3, 4]); + assert_eq!(buf.data(), &[1, 2, 3, 4, 0xff, 0xff]); assert_eq!( buf.traps() .iter() @@ -2509,12 +2549,11 @@ mod test { (2, TrapCode::INTEGER_DIVISION_BY_ZERO) ] ); + let call_sites: Vec<_> = buf.call_sites().collect(); + assert_eq!(call_sites[0].ret_addr, 2); assert_eq!( - buf.call_sites() - .iter() - .map(|call_site| call_site.ret_addr) - .collect::>(), - vec![2] + call_sites[0].exception_handlers, + &[(None.into(), 4), (Some(ExceptionTag::new(42)).into(), 5)] ); assert_eq!( buf.relocs()