Cranelift: remove return-value instructions after calls at callsites.

cranelift/codegen/src/isa/aarch64/abi.rs

@@ -1149,6 +1149,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
+        stackslots_size: u32,
         fixed_frame_storage_size: u32,
         outgoing_args_size: u32,
     ) -> FrameLayout {
@@ -1188,6 +1189,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
             setup_area_size,
             clobber_size,
             fixed_frame_storage_size,
+            stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
         }

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -2951,6 +2951,20 @@ impl MachInstEmit for Inst {
                         inst.emit(sink, emit_info, state);
                     }
                 }
+
+                // Load any stack-carried return values.
+                info.emit_retval_loads::<AArch64MachineDeps, _, _>(
+                    // Use x9 as a temp if needed: clobbered, not a
+                    // retval.
+                    regs::writable_xreg(9),
+                    state.frame_layout().stackslots_size,
+                    |inst| inst.emit(sink, emit_info, state),
+                    |needed_space| Some(Inst::EmitIsland { needed_space }),
+                );
+
+                // We produce an island above if needed, so disable
+                // the worst-case-size check in this case.
+                start_off = sink.cur_offset();
             }
             &Inst::CallInd { ref info } => {
                 let user_stack_map = state.take_stack_map();
@@ -2970,6 +2984,20 @@ impl MachInstEmit for Inst {
                         inst.emit(sink, emit_info, state);
                     }
                 }
+
+                // Load any stack-carried return values.
+                info.emit_retval_loads::<AArch64MachineDeps, _, _>(
+                    // Use x9 as a temp if needed: clobbered, not a
+                    // retval.
+                    regs::writable_xreg(9),
+                    state.frame_layout().stackslots_size,
+                    |inst| inst.emit(sink, emit_info, state),
+                    |needed_space| Some(Inst::EmitIsland { needed_space }),
+                );
+
+                // We produce an island above if needed, so disable
+                // the worst-case-size check in this case.
+                start_off = sink.cur_offset();
             }
             &Inst::ReturnCall { ref info } => {
                 emit_return_call_common_sequence(sink, emit_info, state, info);

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -839,8 +839,11 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -852,8 +855,11 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }

cranelift/codegen/src/isa/pulley_shared/abi.rs

@@ -544,6 +544,7 @@ where
         is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
+        stackslots_size: u32,
         fixed_frame_storage_size: u32,
         outgoing_args_size: u32,
     ) -> FrameLayout {
@@ -578,6 +579,7 @@ where
             setup_area_size: setup_area_size.into(),
             clobber_size,
             fixed_frame_storage_size,
+            stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
         }

cranelift/codegen/src/isa/pulley_shared/inst.isle

@@ -89,6 +89,9 @@
     ;; emit this instruction and auto-generated methods for other various
     ;; bits and pieces of boilerplate in the backend.
     (Raw (raw RawInst))
+
+    ;; Island generation prior to variable-length instructions.
+    (EmitIsland (space_needed u32))
   )
 )
 

cranelift/codegen/src/isa/pulley_shared/inst/emit.rs

@@ -188,6 +188,20 @@ fn pulley_emit<P>(
             for i in PulleyMachineDeps::<P>::gen_sp_reg_adjust(adjust) {
                 <InstAndKind<P>>::from(i).emit(sink, emit_info, state);
             }
+
+            // Load any stack-carried return values.
+            info.emit_retval_loads::<PulleyMachineDeps<P>, _, _>(
+                // Use x15 as a temp if needed: clobbered, not a
+                // retval.
+                Writable::from_reg(regs::x_reg(15)),
+                state.frame_layout().stackslots_size,
+                |inst| inst.emit(sink, emit_info, state),
+                |space_needed| Some(<InstAndKind<P>>::from(Inst::EmitIsland { space_needed })),
+            );
+
+            // We produce an island above if needed, so disable
+            // the worst-case-size check in this case.
+            *start_offset = sink.cur_offset();
         }
 
         Inst::IndirectCall { info } => {
@@ -204,6 +218,20 @@ fn pulley_emit<P>(
             for i in PulleyMachineDeps::<P>::gen_sp_reg_adjust(adjust) {
                 <InstAndKind<P>>::from(i).emit(sink, emit_info, state);
             }
+
+            // Load any stack-carried return values.
+            info.emit_retval_loads::<PulleyMachineDeps<P>, _, _>(
+                // Use x15 as a temp if needed: clobbered, not a
+                // retval.
+                Writable::from_reg(regs::x_reg(15)),
+                state.frame_layout().stackslots_size,
+                |inst| inst.emit(sink, emit_info, state),
+                |space_needed| Some(<InstAndKind<P>>::from(Inst::EmitIsland { space_needed })),
+            );
+
+            // We produce an island above if needed, so disable
+            // the worst-case-size check in this case.
+            *start_offset = sink.cur_offset();
         }
 
         Inst::ReturnCall { info } => {
@@ -517,6 +545,13 @@ fn pulley_emit<P>(
             }
             super::generated::emit(raw, sink)
         }
+
+        Inst::EmitIsland { space_needed } => {
+            let label = sink.get_label();
+            <InstAndKind<P>>::from(Inst::Jump { label }).emit(sink, emit_info, state);
+            sink.emit_island(space_needed + 8, &mut state.ctrl_plane);
+            sink.bind_label(label, &mut state.ctrl_plane);
+        }
     }
 }
 

cranelift/codegen/src/isa/pulley_shared/inst/mod.rs

@@ -169,8 +169,11 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -179,8 +182,11 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -190,8 +196,11 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -298,6 +307,8 @@ fn pulley_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
         }
 
         Inst::Raw { raw } => generated::get_operands(raw, collector),
+
+        Inst::EmitIsland { .. } => {}
     }
 }
 
@@ -745,6 +756,8 @@ impl Inst {
                 format!("br_table {idx} {default:?} {targets:?}")
             }
             Inst::Raw { raw } => generated::print(raw),
+
+            Inst::EmitIsland { space_needed } => format!("emit_island {space_needed}"),
         }
     }
 }

cranelift/codegen/src/isa/riscv64/abi.rs

@@ -649,6 +649,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
+        stackslots_size: u32,
         fixed_frame_storage_size: u32,
         outgoing_args_size: u32,
     ) -> FrameLayout {
@@ -684,6 +685,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
             setup_area_size,
             clobber_size,
             fixed_frame_storage_size,
+            stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
         }

cranelift/codegen/src/isa/riscv64/inst.isle

@@ -345,6 +345,10 @@
       (flags MemFlags)
       (mask VecOpMasking)
       (vstate VState))
+
+    (EmitIsland
+     ;; The needed space before the next deadline.
+     (needed_space u32))
 ))
 
 (type AtomicOP (enum

cranelift/codegen/src/isa/riscv64/inst/emit.rs

@@ -210,6 +210,7 @@ impl Inst {
             // some cases.
             Inst::VecLoad { vstate, .. }
             | Inst::VecStore { vstate, .. } => Some(vstate),
+            Inst::EmitIsland { .. } => None,
         }
     }
 }
@@ -1133,6 +1134,18 @@ impl Inst {
                         inst.emit(sink, emit_info, state);
                     }
                 }
+
+                // Load any stack-carried return values.
+                info.emit_retval_loads::<Riscv64MachineDeps, _, _>(
+                    // Use x12 as a temp if needed: clobbered, not a
+                    // retval.
+                    Writable::from_reg(regs::x_reg(12)),
+                    state.frame_layout().stackslots_size,
+                    |inst| inst.emit(sink, emit_info, state),
+                    |needed_space| Some(Inst::EmitIsland { needed_space }),
+                );
+
+                *start_off = sink.cur_offset();
             }
             &Inst::CallInd { ref info } => {
                 Inst::Jalr {
@@ -1155,6 +1168,18 @@ impl Inst {
                         inst.emit(sink, emit_info, state);
                     }
                 }
+
+                // Load any stack-carried return values.
+                info.emit_retval_loads::<Riscv64MachineDeps, _, _>(
+                    // Use x12 as a temp if needed: clobbered, not a
+                    // retval.
+                    Writable::from_reg(regs::x_reg(12)),
+                    state.frame_layout().stackslots_size,
+                    |inst| inst.emit(sink, emit_info, state),
+                    |needed_space| Some(Inst::EmitIsland { needed_space }),
+                );
+
+                *start_off = sink.cur_offset();
             }
 
             &Inst::ReturnCall { ref info } => {
@@ -2577,7 +2602,14 @@ impl Inst {
                     to.nf(),
                 ));
             }
-        };
+
+            Inst::EmitIsland { needed_space } => {
+                let jump_around_label = sink.get_label();
+                Inst::gen_jump(jump_around_label).emit(sink, emit_info, state);
+                sink.emit_island(needed_space + 4, &mut state.ctrl_plane);
+                sink.bind_label(jump_around_label, &mut state.ctrl_plane);
+            }
+        }
     }
 }
 

cranelift/codegen/src/isa/riscv64/inst/mod.rs

@@ -335,8 +335,11 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -348,8 +351,11 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             for CallArgPair { vreg, preg } in uses {
                 collector.reg_fixed_use(vreg, *preg);
             }
-            for CallRetPair { vreg, preg } in defs {
-                collector.reg_fixed_def(vreg, *preg);
+            for CallRetPair { vreg, location } in defs {
+                match location {
+                    RetLocation::Reg(preg) => collector.reg_fixed_def(vreg, *preg),
+                    RetLocation::Stack(..) => collector.any_def(vreg),
+                }
             }
             collector.reg_clobbers(info.clobbers);
         }
@@ -680,6 +686,7 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_use(from);
             vec_mask_operands(mask, collector);
         }
+        Inst::EmitIsland { .. } => {}
     }
 }
 
@@ -1612,6 +1619,9 @@ impl Inst {
 
                 format!("vs{eew}.v {vs3},{dst}{mask} {vstate}")
             }
+            Inst::EmitIsland { needed_space } => {
+                format!("emit_island {needed_space}")
+            }
         }
     }
 }

cranelift/codegen/src/isa/s390x/abi.rs

@@ -914,6 +914,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         _is_leaf: bool,
         incoming_args_size: u32,
         tail_args_size: u32,
+        stackslots_size: u32,
         fixed_frame_storage_size: u32,
         mut outgoing_args_size: u32,
     ) -> FrameLayout {
@@ -985,6 +986,7 @@ impl ABIMachineSpec for S390xMachineDeps {
             setup_area_size: 0,
             clobber_size,
             fixed_frame_storage_size,
+            stackslots_size,
             outgoing_args_size,
             clobbered_callee_saves: regs,
         }