From 0166f898e46c6ece930361691af37ed80c4d26bf Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Fri, 6 Mar 2026 20:41:00 +0100 Subject: [PATCH 01/15] [RISCV] Fix musttail with indirect arguments by forwarding incoming pointers When a musttail call has arguments passed indirectly (CCValAssign::Indirect), the current code creates a new stack temporary and copies the data there. The tail call then deallocates the caller's stack frame, leaving the pointer dangling. Fix this by forwarding the original incoming indirect pointer instead of re-spilling. Since musttail guarantees matching prototypes, incoming and outgoing indirect args have a 1:1 correspondence, and the incoming pointer (from the caller's caller's frame) remains valid after the tail call. This also subsumes the non-musttail indirect args fix: non-musttail calls with indirect args are rejected from tail call optimization (the pointer would dangle), while musttail calls forward the incoming pointer. Fixes #185089. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 118 +++++++++++------- .../Target/RISCV/RISCVMachineFunctionInfo.h | 13 ++ .../CodeGen/RISCV/musttail-indirect-args.ll | 66 ++++++++++ llvm/test/CodeGen/RISCV/tail-calls.ll | 18 ++- 4 files changed, 167 insertions(+), 48 deletions(-) create mode 100644 llvm/test/CodeGen/RISCV/musttail-indirect-args.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 17d7db95886ab..68cff3332b873 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24454,6 +24454,8 @@ SDValue RISCVTargetLowering::LowerFormalArguments( ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL, *this); if (VA.getLocInfo() == CCValAssign::Indirect) { + // Save the incoming indirect pointer for musttail forwarding. + RVFI->addIncomingIndirectArg(ArgValue); // If the original argument was split and passed by reference (e.g. i128 // on RV32), we need to load all parts of it here (using the same // address). Vectors may be partly split to registers and partly to the @@ -24578,6 +24580,19 @@ bool RISCVTargetLowering::isEligibleForTailCallOptimization( if (CCInfo.getStackSize() > RVFI->getArgumentStackSize()) return false; + // Do not tail call optimize if any argument needs to be passed indirectly. + // The caller allocates stack space and passes a pointer to the callee. On a + // tail call the caller's stack frame is deallocated before the callee + // executes, invalidating the pointer (use-after-free). + // musttail is excluded: callers forward incoming indirect pointers that + // point to the caller's caller's frame, which remains valid. + if (!CLI.CB || !CLI.CB->isMustTailCall()) { + for (const auto &VA : ArgLocs) { + if (VA.getLocInfo() == CCValAssign::Indirect) + return false; + } + } + // Do not tail call opt if either caller or callee uses struct return // semantics. auto IsCallerStructRet = Caller.hasStructRetAttr(); @@ -24765,51 +24780,70 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Promote the value if needed. // For now, only handle fully promoted and indirect arguments. if (VA.getLocInfo() == CCValAssign::Indirect) { - // Store the argument in a stack slot and pass its address. - Align StackAlign = - std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG), - getPrefTypeAlign(ArgValue.getValueType(), DAG)); - TypeSize StoredSize = ArgValue.getValueType().getStoreSize(); - // If the original argument was split (e.g. i128), we need - // to store the required parts of it here (and pass just one address). - // Vectors may be partly split to registers and partly to the stack, in - // which case the base address is partly offset and subsequent stores are - // relative to that. - unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; - unsigned ArgPartOffset = Outs[OutIdx].PartOffset; - assert(VA.getValVT().isVector() || ArgPartOffset == 0); - // Calculate the total size to store. We don't have access to what we're - // actually storing other than performing the loop and collecting the - // info. - SmallVector> Parts; - while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { - SDValue PartValue = OutVals[OutIdx + 1]; - unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; - SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); - EVT PartVT = PartValue.getValueType(); - if (PartVT.isScalableVector()) - Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); - StoredSize += PartVT.getStoreSize(); - StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG)); - Parts.push_back(std::make_pair(PartValue, Offset)); - ++i; - ++OutIdx; - } - SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign); - int FI = cast(SpillSlot)->getIndex(); - MemOpChains.push_back( - DAG.getStore(Chain, DL, ArgValue, SpillSlot, - MachinePointerInfo::getFixedStack(MF, FI))); - for (const auto &Part : Parts) { - SDValue PartValue = Part.first; - SDValue PartOffset = Part.second; - SDValue Address = - DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); + // For musttail calls, forward the incoming indirect pointer instead + // of creating a new stack temporary. The incoming pointer points to + // the caller's caller's frame, which remains valid after a tail call. + if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { + unsigned IndirectIdx = 0; + for (unsigned k = 0; k < OutIdx; ++k) { + if (ArgLocs[k].getLocInfo() == CCValAssign::Indirect) + ++IndirectIdx; + } + ArgValue = RVFI->getIncomingIndirectArg(IndirectIdx); + // Skip any split parts of this argument (they are covered by the + // forwarded pointer). + unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { + ++i; + ++OutIdx; + } + } else { + // Store the argument in a stack slot and pass its address. + Align StackAlign = + std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG), + getPrefTypeAlign(ArgValue.getValueType(), DAG)); + TypeSize StoredSize = ArgValue.getValueType().getStoreSize(); + // If the original argument was split (e.g. i128), we need + // to store the required parts of it here (and pass just one address). + // Vectors may be partly split to registers and partly to the stack, in + // which case the base address is partly offset and subsequent stores + // are relative to that. + unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; + unsigned ArgPartOffset = Outs[OutIdx].PartOffset; + assert(VA.getValVT().isVector() || ArgPartOffset == 0); + // Calculate the total size to store. We don't have access to what + // we're actually storing other than performing the loop and collecting + // the info. + SmallVector> Parts; + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { + SDValue PartValue = OutVals[OutIdx + 1]; + unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; + SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); + EVT PartVT = PartValue.getValueType(); + if (PartVT.isScalableVector()) + Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); + StoredSize += PartVT.getStoreSize(); + StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG)); + Parts.push_back(std::make_pair(PartValue, Offset)); + ++i; + ++OutIdx; + } + SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign); + int FI = cast(SpillSlot)->getIndex(); MemOpChains.push_back( - DAG.getStore(Chain, DL, PartValue, Address, + DAG.getStore(Chain, DL, ArgValue, SpillSlot, MachinePointerInfo::getFixedStack(MF, FI))); + for (const auto &Part : Parts) { + SDValue PartValue = Part.first; + SDValue PartOffset = Part.second; + SDValue Address = + DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); + MemOpChains.push_back( + DAG.getStore(Chain, DL, PartValue, Address, + MachinePointerInfo::getFixedStack(MF, FI))); + } + ArgValue = SpillSlot; } - ArgValue = SpillSlot; } else { ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget); } diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h index e23f162a317ef..65b7226025da5 100644 --- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h +++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h @@ -73,6 +73,9 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { /// Incoming ByVal arguments SmallVector IncomingByValArgs; + /// Incoming indirect argument pointers (for musttail forwarding) + SmallVector IncomingIndirectArgs; + /// Is there any vector argument or return? bool IsVectorCall = false; @@ -157,6 +160,16 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { SDValue getIncomingByValArgs(unsigned Idx) { return IncomingByValArgs[Idx]; } unsigned getIncomingByValArgsSize() const { return IncomingByValArgs.size(); } + void addIncomingIndirectArg(SDValue Val) { + IncomingIndirectArgs.push_back(Val); + } + SDValue getIncomingIndirectArg(unsigned Idx) { + return IncomingIndirectArgs[Idx]; + } + unsigned getIncomingIndirectArgsSize() const { + return IncomingIndirectArgs.size(); + } + enum class PushPopKind { None = 0, StdExtZcmp, VendorXqccmp }; PushPopKind getPushPopKind(const MachineFunction &MF) const; diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll new file mode 100644 index 0000000000000..447cacf13e2b8 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -0,0 +1,66 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=riscv32 %s -o - | FileCheck %s --check-prefix=RV32 +; RUN: llc -mtriple=riscv64 %s -o - | FileCheck %s --check-prefix=RV64 + +; Test that musttail with indirect args (fp128 on RV32) forwards the incoming +; pointer instead of creating a new stack temporary. Without this fix, the +; pointer would dangle after the tail call deallocates the caller's frame. + +declare i32 @callee_musttail_indirect(fp128 %a) + +; fp128 is indirect on RV32 (too large for registers), direct on RV64. +; On RV32, musttail must forward the incoming indirect pointer (a0) directly. +define i32 @caller_musttail_indirect(fp128 %a) nounwind { +; RV32-LABEL: caller_musttail_indirect: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_indirect +; +; RV64-LABEL: caller_musttail_indirect: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_indirect + %call = musttail call i32 @callee_musttail_indirect(fp128 %a) + ret i32 %call +} + +; Verify that non-musttail tail call with indirect args does NOT tail call +; (this is the PR #184972 fix - indirect args are unsafe for regular tail calls). +define void @caller_no_musttail_indirect() nounwind { +; RV32-LABEL: caller_no_musttail_indirect: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -32 +; RV32-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32-NEXT: lui a1, 262128 +; RV32-NEXT: mv a0, sp +; RV32-NEXT: sw zero, 0(sp) +; RV32-NEXT: sw zero, 4(sp) +; RV32-NEXT: sw zero, 8(sp) +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: call callee_musttail_indirect +; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 32 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_no_musttail_indirect: +; RV64: # %bb.0: +; RV64-NEXT: lui a1, 16383 +; RV64-NEXT: slli a1, a1, 36 +; RV64-NEXT: li a0, 0 +; RV64-NEXT: tail callee_musttail_indirect + %call = tail call i32 @callee_musttail_indirect(fp128 0xL00000000000000003FFF000000000000) + ret void +} + +; Test musttail with i128 on RV32 (indirect, split into 4 x i32 parts). +declare i64 @callee_musttail_i128(i128 %a) + +define i64 @caller_musttail_i128(i128 %a) nounwind { +; RV32-LABEL: caller_musttail_i128: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_i128 +; +; RV64-LABEL: caller_musttail_i128: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_i128 + %call = musttail call i64 @callee_musttail_i128(i128 %a) + ret i64 %call +} diff --git a/llvm/test/CodeGen/RISCV/tail-calls.ll b/llvm/test/CodeGen/RISCV/tail-calls.ll index 33feba3c6fba1..79855aa03adcf 100644 --- a/llvm/test/CodeGen/RISCV/tail-calls.ll +++ b/llvm/test/CodeGen/RISCV/tail-calls.ll @@ -247,20 +247,24 @@ declare i32 @callee_indirect_args(fp128 %a) define void @caller_indirect_args() nounwind { ; CHECK-LABEL: caller_indirect_args: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: addi sp, sp, -16 +; CHECK-NEXT: addi sp, sp, -32 +; CHECK-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; CHECK-NEXT: lui a1, 262128 ; CHECK-NEXT: mv a0, sp ; CHECK-NEXT: sw zero, 0(sp) ; CHECK-NEXT: sw zero, 4(sp) ; CHECK-NEXT: sw zero, 8(sp) ; CHECK-NEXT: sw a1, 12(sp) -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: tail callee_indirect_args +; CHECK-NEXT: call callee_indirect_args +; CHECK-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; CHECK-NEXT: addi sp, sp, 32 +; CHECK-NEXT: ret ; ; CHECK-LARGE-ZICFILP-LABEL: caller_indirect_args: ; CHECK-LARGE-ZICFILP: # %bb.0: # %entry ; CHECK-LARGE-ZICFILP-NEXT: lpad 0 -; CHECK-LARGE-ZICFILP-NEXT: addi sp, sp, -16 +; CHECK-LARGE-ZICFILP-NEXT: addi sp, sp, -32 +; CHECK-LARGE-ZICFILP-NEXT: sw ra, 28(sp) # 4-byte Folded Spill ; CHECK-LARGE-ZICFILP-NEXT: lui a1, 262128 ; CHECK-LARGE-ZICFILP-NEXT: .Lpcrel_hi9: ; CHECK-LARGE-ZICFILP-NEXT: auipc a0, %pcrel_hi(.LCPI7_0) @@ -270,8 +274,10 @@ define void @caller_indirect_args() nounwind { ; CHECK-LARGE-ZICFILP-NEXT: sw zero, 4(sp) ; CHECK-LARGE-ZICFILP-NEXT: sw zero, 8(sp) ; CHECK-LARGE-ZICFILP-NEXT: sw a1, 12(sp) -; CHECK-LARGE-ZICFILP-NEXT: addi sp, sp, 16 -; CHECK-LARGE-ZICFILP-NEXT: jr t2 +; CHECK-LARGE-ZICFILP-NEXT: jalr t2 +; CHECK-LARGE-ZICFILP-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; CHECK-LARGE-ZICFILP-NEXT: addi sp, sp, 32 +; CHECK-LARGE-ZICFILP-NEXT: ret entry: %call = tail call i32 @callee_indirect_args(fp128 0xL00000000000000003FFF000000000000) ret void From d94c5a02c11d56c4f2469bd05ae44083d98f743d Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sat, 7 Mar 2026 08:29:22 +0100 Subject: [PATCH 02/15] [RISCV] Address review: use DenseMap for indirect args, add tests - Use DenseMap keyed by OrigArgIndex instead of SmallVector for incoming indirect arg pointers. This avoids a fragile O(n) counting loop in LowerCall and directly maps argument indices. - Add test cases for: two indirect args (DenseMap multi-key), mixed direct+indirect, and i128 split+trailing direct arg. - Add test for non-musttail forwarding of indirect arg (shows it correctly falls back to call). Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 13 +--- .../Target/RISCV/RISCVMachineFunctionInfo.h | 19 ++--- .../CodeGen/RISCV/musttail-indirect-args.ll | 77 +++++++++++++++++++ 3 files changed, 91 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 68cff3332b873..e49628550bc5f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24454,8 +24454,6 @@ SDValue RISCVTargetLowering::LowerFormalArguments( ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL, *this); if (VA.getLocInfo() == CCValAssign::Indirect) { - // Save the incoming indirect pointer for musttail forwarding. - RVFI->addIncomingIndirectArg(ArgValue); // If the original argument was split and passed by reference (e.g. i128 // on RV32), we need to load all parts of it here (using the same // address). Vectors may be partly split to registers and partly to the @@ -24464,6 +24462,8 @@ SDValue RISCVTargetLowering::LowerFormalArguments( InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, MachinePointerInfo())); unsigned ArgIndex = Ins[InsIdx].OrigArgIndex; + // Save the incoming indirect pointer for musttail forwarding. + RVFI->setIncomingIndirectArg(ArgIndex, ArgValue); unsigned ArgPartOffset = Ins[InsIdx].PartOffset; assert(VA.getValVT().isVector() || ArgPartOffset == 0); while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) { @@ -24784,15 +24784,10 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // of creating a new stack temporary. The incoming pointer points to // the caller's caller's frame, which remains valid after a tail call. if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { - unsigned IndirectIdx = 0; - for (unsigned k = 0; k < OutIdx; ++k) { - if (ArgLocs[k].getLocInfo() == CCValAssign::Indirect) - ++IndirectIdx; - } - ArgValue = RVFI->getIncomingIndirectArg(IndirectIdx); + unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; + ArgValue = RVFI->getIncomingIndirectArg(ArgIndex); // Skip any split parts of this argument (they are covered by the // forwarded pointer). - unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { ++i; ++OutIdx; diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h index 65b7226025da5..9db41bc04094e 100644 --- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h +++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h @@ -14,6 +14,7 @@ #define LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H #include "RISCVSubtarget.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MIRYamlMapping.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -73,8 +74,9 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { /// Incoming ByVal arguments SmallVector IncomingByValArgs; - /// Incoming indirect argument pointers (for musttail forwarding) - SmallVector IncomingIndirectArgs; + /// Incoming indirect argument pointers, keyed by OrigArgIndex. + /// Used for musttail forwarding of indirect args. + DenseMap IncomingIndirectArgs; /// Is there any vector argument or return? bool IsVectorCall = false; @@ -160,14 +162,13 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { SDValue getIncomingByValArgs(unsigned Idx) { return IncomingByValArgs[Idx]; } unsigned getIncomingByValArgsSize() const { return IncomingByValArgs.size(); } - void addIncomingIndirectArg(SDValue Val) { - IncomingIndirectArgs.push_back(Val); + void setIncomingIndirectArg(unsigned ArgIndex, SDValue Val) { + IncomingIndirectArgs[ArgIndex] = Val; } - SDValue getIncomingIndirectArg(unsigned Idx) { - return IncomingIndirectArgs[Idx]; - } - unsigned getIncomingIndirectArgsSize() const { - return IncomingIndirectArgs.size(); + SDValue getIncomingIndirectArg(unsigned ArgIndex) const { + auto It = IncomingIndirectArgs.find(ArgIndex); + assert(It != IncomingIndirectArgs.end() && "No incoming indirect arg"); + return It->second; } enum class PushPopKind { None = 0, StdExtZcmp, VendorXqccmp }; diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index 447cacf13e2b8..97816c4c41e9d 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -50,6 +50,67 @@ define void @caller_no_musttail_indirect() nounwind { ret void } +; Verify that non-musttail tail call forwarding an indirect arg from the +; caller's own parameters also does NOT tail call (the arg lives on the +; caller's frame, which would be deallocated). +define i32 @caller_no_musttail_forward_indirect(fp128 %a) nounwind { +; RV32-LABEL: caller_no_musttail_forward_indirect: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -32 +; RV32-NEXT: sw ra, 28(sp) # 4-byte Folded Spill +; RV32-NEXT: lw a1, 0(a0) +; RV32-NEXT: lw a2, 4(a0) +; RV32-NEXT: lw a3, 8(a0) +; RV32-NEXT: lw a4, 12(a0) +; RV32-NEXT: mv a0, sp +; RV32-NEXT: sw a1, 0(sp) +; RV32-NEXT: sw a2, 4(sp) +; RV32-NEXT: sw a3, 8(sp) +; RV32-NEXT: sw a4, 12(sp) +; RV32-NEXT: call callee_musttail_indirect +; RV32-NEXT: lw ra, 28(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 32 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_no_musttail_forward_indirect: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_indirect + %call = tail call i32 @callee_musttail_indirect(fp128 %a) + ret i32 %call +} + +; Test musttail with two indirect fp128 args on RV32. Both pointers must be +; forwarded. Exercises the DenseMap with two distinct OrigArgIndex values. +declare i32 @callee_musttail_two_indirect(fp128 %a, fp128 %b) + +define i32 @caller_musttail_two_indirect(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_two_indirect: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_two_indirect: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_two_indirect + %call = musttail call i32 @callee_musttail_two_indirect(fp128 %a, fp128 %b) + ret i32 %call +} + +; Test musttail with mixed direct (i32 in register) + indirect (fp128) args. +; Confirms OrigArgIndex lookup works when not all args are indirect. +declare i32 @callee_musttail_mixed(i32 %x, fp128 %a) + +define i32 @caller_musttail_mixed(i32 %x, fp128 %a) nounwind { +; RV32-LABEL: caller_musttail_mixed: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_mixed +; +; RV64-LABEL: caller_musttail_mixed: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_mixed + %call = musttail call i32 @callee_musttail_mixed(i32 %x, fp128 %a) + ret i32 %call +} + ; Test musttail with i128 on RV32 (indirect, split into 4 x i32 parts). declare i64 @callee_musttail_i128(i128 %a) @@ -64,3 +125,19 @@ define i64 @caller_musttail_i128(i128 %a) nounwind { %call = musttail call i64 @callee_musttail_i128(i128 %a) ret i64 %call } + +; Test musttail with i128 (indirect+split on RV32) plus a trailing i32 direct arg. +; Exercises the split-skip logic followed by a normal register arg. +declare i64 @callee_musttail_i128_and_i32(i128 %a, i32 %x) + +define i64 @caller_musttail_i128_and_i32(i128 %a, i32 %x) nounwind { +; RV32-LABEL: caller_musttail_i128_and_i32: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_i128_and_i32 +; +; RV64-LABEL: caller_musttail_i128_and_i32: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_i128_and_i32 + %call = musttail call i64 @callee_musttail_i128_and_i32(i128 %a, i32 %x) + ret i64 %call +} From ecb7952f4ef1f2eb475cc6d4f67055b08bc2c3a0 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sat, 7 Mar 2026 13:02:31 +0100 Subject: [PATCH 03/15] [RISCV] Fix musttail indirect arg forwarding when arguments are reordered Outs[].OrigArgIndex is the position in the call's argument list (callee perspective), but the incoming indirect arg map is keyed by the caller's formal parameter index. When musttail reorders arguments (e.g., `musttail call @f(fp128 %b, fp128 %a)`), these indices diverge, causing the wrong pointers to be forwarded. Fix by resolving the caller's formal parameter index via the IR: walk CLI.CB->args() to find the Argument at the matching call position and use its getArgNo() as the DenseMap lookup key. Add comprehensive tests for swapped, rotated, duplicated, and stack-spilled indirect args. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 23 +- .../CodeGen/RISCV/musttail-indirect-args.ll | 209 ++++++++++++++++++ 2 files changed, 229 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index e49628550bc5f..0dafa47f8cc00 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24784,11 +24784,28 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // of creating a new stack temporary. The incoming pointer points to // the caller's caller's frame, which remains valid after a tail call. if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { - unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; - ArgValue = RVFI->getIncomingIndirectArg(ArgIndex); + // Outs[OutIdx].OrigArgIndex is the position in the call's argument + // list (callee perspective), but the incoming indirect arg map is + // keyed by the caller's formal parameter index. When musttail + // reorders arguments, these differ. Resolve via the IR: find which + // formal parameter is being passed at this call position. + unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex; + unsigned FormalIdx = CallArgIdx; // default if lookup fails + unsigned Idx = 0; + for (const auto &CallArg : CLI.CB->args()) { + if (CallArg->getType()->isEmptyTy()) + continue; + if (Idx == CallArgIdx) { + if (const auto *FormalArg = dyn_cast(CallArg)) + FormalIdx = FormalArg->getArgNo(); + break; + } + ++Idx; + } + ArgValue = RVFI->getIncomingIndirectArg(FormalIdx); // Skip any split parts of this argument (they are covered by the // forwarded pointer). - while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { ++i; ++OutIdx; } diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index 97816c4c41e9d..c6d1743a27d26 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -141,3 +141,212 @@ define i64 @caller_musttail_i128_and_i32(i128 %a, i32 %x) nounwind { %call = musttail call i64 @callee_musttail_i128_and_i32(i128 %a, i32 %x) ret i64 %call } + +; Test musttail with two indirect args SWAPPED. The pointers must be exchanged +; before the tail call. This exercises the OrigArgIndex -> Argument::getArgNo() +; resolution in LowerCall. +define i32 @caller_musttail_two_indirect_swapped(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_two_indirect_swapped: +; RV32: # %bb.0: +; RV32-NEXT: mv a2, a0 +; RV32-NEXT: mv a0, a1 +; RV32-NEXT: mv a1, a2 +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_two_indirect_swapped: +; RV64: # %bb.0: +; RV64-NEXT: mv a4, a1 +; RV64-NEXT: mv a5, a0 +; RV64-NEXT: mv a0, a2 +; RV64-NEXT: mv a1, a3 +; RV64-NEXT: mv a2, a5 +; RV64-NEXT: mv a3, a4 +; RV64-NEXT: tail callee_musttail_two_indirect + %call = musttail call i32 @callee_musttail_two_indirect(fp128 %b, fp128 %a) + ret i32 %call +} + +; Test musttail with three indirect args rotated: call @f(%c, %a, %b). +; All three pointers need to be shuffled. +declare i32 @callee_musttail_three_indirect(fp128 %a, fp128 %b, fp128 %c) + +define i32 @caller_musttail_three_indirect_rotated(fp128 %a, fp128 %b, fp128 %c) nounwind { +; RV32-LABEL: caller_musttail_three_indirect_rotated: +; RV32: # %bb.0: +; RV32-NEXT: mv a3, a1 +; RV32-NEXT: mv a1, a0 +; RV32-NEXT: mv a0, a2 +; RV32-NEXT: mv a2, a3 +; RV32-NEXT: tail callee_musttail_three_indirect +; +; RV64-LABEL: caller_musttail_three_indirect_rotated: +; RV64: # %bb.0: +; RV64-NEXT: mv a6, a3 +; RV64-NEXT: mv a7, a2 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a0, a4 +; RV64-NEXT: mv a1, a5 +; RV64-NEXT: mv a4, a7 +; RV64-NEXT: mv a5, a6 +; RV64-NEXT: tail callee_musttail_three_indirect + %call = musttail call i32 @callee_musttail_three_indirect(fp128 %c, fp128 %a, fp128 %b) + ret i32 %call +} + +; Test musttail with mixed direct + indirect args where the indirect args +; are swapped but the direct arg stays in place. +declare i32 @callee_musttail_mixed_two_indirect(i32 %x, fp128 %a, fp128 %b) + +define i32 @caller_musttail_mixed_swap_indirect(i32 %x, fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_mixed_swap_indirect: +; RV32: # %bb.0: +; RV32-NEXT: mv a3, a1 +; RV32-NEXT: mv a1, a2 +; RV32-NEXT: mv a2, a3 +; RV32-NEXT: tail callee_musttail_mixed_two_indirect +; +; RV64-LABEL: caller_musttail_mixed_swap_indirect: +; RV64: # %bb.0: +; RV64-NEXT: mv a5, a2 +; RV64-NEXT: mv a6, a1 +; RV64-NEXT: mv a1, a3 +; RV64-NEXT: mv a2, a4 +; RV64-NEXT: mv a3, a6 +; RV64-NEXT: mv a4, a5 +; RV64-NEXT: tail callee_musttail_mixed_two_indirect + %call = musttail call i32 @callee_musttail_mixed_two_indirect(i32 %x, fp128 %b, fp128 %a) + ret i32 %call +} + +; Test musttail with swapped i128 on RV32 (split indirect args). +declare i64 @callee_musttail_two_i128(i128 %a, i128 %b) + +define i64 @caller_musttail_two_i128_swapped(i128 %a, i128 %b) nounwind { +; RV32-LABEL: caller_musttail_two_i128_swapped: +; RV32: # %bb.0: +; RV32-NEXT: mv a2, a0 +; RV32-NEXT: mv a0, a1 +; RV32-NEXT: mv a1, a2 +; RV32-NEXT: tail callee_musttail_two_i128 +; +; RV64-LABEL: caller_musttail_two_i128_swapped: +; RV64: # %bb.0: +; RV64-NEXT: mv a4, a1 +; RV64-NEXT: mv a5, a0 +; RV64-NEXT: mv a0, a2 +; RV64-NEXT: mv a1, a3 +; RV64-NEXT: mv a2, a5 +; RV64-NEXT: mv a3, a4 +; RV64-NEXT: tail callee_musttail_two_i128 + %call = musttail call i64 @callee_musttail_two_i128(i128 %b, i128 %a) + ret i64 %call +} + +; Test musttail passing the same indirect arg to both positions. +define i32 @caller_musttail_two_indirect_dup(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_two_indirect_dup: +; RV32: # %bb.0: +; RV32-NEXT: mv a1, a0 +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_two_indirect_dup: +; RV64: # %bb.0: +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: tail callee_musttail_two_indirect + %call = musttail call i32 @callee_musttail_two_indirect(fp128 %a, fp128 %a) + ret i32 %call +} + +; Test musttail with enough indirect args to spill to the stack (9 fp128 on +; RV32 uses a0-a7 for the first 8 pointers, 9th goes on the stack). +declare void @callee_musttail_nine_indirect(fp128, fp128, fp128, fp128, fp128, fp128, fp128, fp128, fp128) + +define void @caller_musttail_nine_indirect(fp128 %a, fp128 %b, fp128 %c, fp128 %d, fp128 %e, fp128 %f, fp128 %g, fp128 %h, fp128 %i) nounwind { +; RV32-LABEL: caller_musttail_nine_indirect: +; RV32: # %bb.0: +; RV32-NEXT: lw t0, 0(sp) +; RV32-NEXT: sw t0, 0(sp) +; RV32-NEXT: tail callee_musttail_nine_indirect +; +; RV64-LABEL: caller_musttail_nine_indirect: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -32 +; RV64-NEXT: sd s0, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s2, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: ld t0, 104(sp) +; RV64-NEXT: ld t1, 96(sp) +; RV64-NEXT: ld t2, 88(sp) +; RV64-NEXT: ld t3, 80(sp) +; RV64-NEXT: ld t4, 72(sp) +; RV64-NEXT: ld t5, 64(sp) +; RV64-NEXT: ld t6, 32(sp) +; RV64-NEXT: ld s0, 40(sp) +; RV64-NEXT: ld s1, 48(sp) +; RV64-NEXT: ld s2, 56(sp) +; RV64-NEXT: sd t6, 32(sp) +; RV64-NEXT: sd s0, 40(sp) +; RV64-NEXT: sd s1, 48(sp) +; RV64-NEXT: sd s2, 56(sp) +; RV64-NEXT: sd t5, 64(sp) +; RV64-NEXT: sd t4, 72(sp) +; RV64-NEXT: sd t3, 80(sp) +; RV64-NEXT: sd t2, 88(sp) +; RV64-NEXT: sd t1, 96(sp) +; RV64-NEXT: sd t0, 104(sp) +; RV64-NEXT: ld s0, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s2, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: tail callee_musttail_nine_indirect + musttail call void @callee_musttail_nine_indirect(fp128 %a, fp128 %b, fp128 %c, fp128 %d, fp128 %e, fp128 %f, fp128 %g, fp128 %h, fp128 %i) + ret void +} + +; Test musttail swapping the first (register) and last (stack-spilled) args. +define void @caller_musttail_nine_indirect_swap_first_last(fp128 %a, fp128 %b, fp128 %c, fp128 %d, fp128 %e, fp128 %f, fp128 %g, fp128 %h, fp128 %i) nounwind { +; RV32-LABEL: caller_musttail_nine_indirect_swap_first_last: +; RV32: # %bb.0: +; RV32-NEXT: lw t0, 0(sp) +; RV32-NEXT: sw a0, 0(sp) +; RV32-NEXT: mv a0, t0 +; RV32-NEXT: tail callee_musttail_nine_indirect +; +; RV64-LABEL: caller_musttail_nine_indirect_swap_first_last: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -32 +; RV64-NEXT: sd s0, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s2, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: ld t0, 96(sp) +; RV64-NEXT: ld t1, 104(sp) +; RV64-NEXT: ld t2, 88(sp) +; RV64-NEXT: ld t3, 80(sp) +; RV64-NEXT: ld t4, 72(sp) +; RV64-NEXT: ld t5, 64(sp) +; RV64-NEXT: ld t6, 32(sp) +; RV64-NEXT: ld s0, 40(sp) +; RV64-NEXT: ld s1, 48(sp) +; RV64-NEXT: ld s2, 56(sp) +; RV64-NEXT: sd t6, 32(sp) +; RV64-NEXT: sd s0, 40(sp) +; RV64-NEXT: sd s1, 48(sp) +; RV64-NEXT: sd s2, 56(sp) +; RV64-NEXT: sd t5, 64(sp) +; RV64-NEXT: sd t4, 72(sp) +; RV64-NEXT: sd t3, 80(sp) +; RV64-NEXT: sd t2, 88(sp) +; RV64-NEXT: sd a0, 96(sp) +; RV64-NEXT: sd a1, 104(sp) +; RV64-NEXT: mv a0, t0 +; RV64-NEXT: mv a1, t1 +; RV64-NEXT: ld s0, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s2, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: tail callee_musttail_nine_indirect + musttail call void @callee_musttail_nine_indirect(fp128 %i, fp128 %b, fp128 %c, fp128 %d, fp128 %e, fp128 %f, fp128 %g, fp128 %h, fp128 %a) + ret void +} From 568648b54aeb5d9308a2b51d289b9980af9b2b2f Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Mon, 9 Mar 2026 13:23:04 +0100 Subject: [PATCH 04/15] [RISCV] Handle computed values in musttail indirect arg forwarding When a musttail call passes a computed value (not a forwarded formal parameter) as an indirect argument, we must store the value into the incoming indirect pointer rather than blindly forwarding it. The incoming pointer points to the caller's caller's frame, which remains valid after the tail call and is writable per the RISC-V ABI. Previously, computed values were silently dropped: the incoming pointer was forwarded without modification, passing the original (stale) data. The fix distinguishes two cases via dyn_cast on the call arg: - Forwarded formal parameter: zero-copy pointer forwarding (unchanged) - Computed value: store into the incoming pointer, then forward it Add tests for computed fp128, computed i128 (split indirect on RV32), mixed computed+forwarded, and both-computed scenarios. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 60 ++- .../CodeGen/RISCV/musttail-indirect-args.ll | 359 ++++++++++++++++++ 2 files changed, 405 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 0dafa47f8cc00..fb0babc6b9dba 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24780,31 +24780,63 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Promote the value if needed. // For now, only handle fully promoted and indirect arguments. if (VA.getLocInfo() == CCValAssign::Indirect) { - // For musttail calls, forward the incoming indirect pointer instead - // of creating a new stack temporary. The incoming pointer points to - // the caller's caller's frame, which remains valid after a tail call. + // For musttail calls, reuse incoming indirect pointers instead of + // creating new stack temporaries. The incoming pointers point to the + // caller's caller's frame, which remains valid after a tail call. if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { - // Outs[OutIdx].OrigArgIndex is the position in the call's argument - // list (callee perspective), but the incoming indirect arg map is - // keyed by the caller's formal parameter index. When musttail - // reorders arguments, these differ. Resolve via the IR: find which - // formal parameter is being passed at this call position. unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex; - unsigned FormalIdx = CallArgIdx; // default if lookup fails + + // Resolve which formal parameter is being passed at this call + // position. Outs[].OrigArgIndex is the callee's argument position, + // but the incoming indirect arg map is keyed by the caller's formal + // parameter index. These differ when musttail reorders arguments. + const Argument *FormalArg = nullptr; unsigned Idx = 0; for (const auto &CallArg : CLI.CB->args()) { if (CallArg->getType()->isEmptyTy()) continue; if (Idx == CallArgIdx) { - if (const auto *FormalArg = dyn_cast(CallArg)) - FormalIdx = FormalArg->getArgNo(); + FormalArg = dyn_cast(CallArg); break; } ++Idx; } - ArgValue = RVFI->getIncomingIndirectArg(FormalIdx); - // Skip any split parts of this argument (they are covered by the - // forwarded pointer). + + if (FormalArg) { + // The call arg is a forwarded formal parameter. Forward its + // incoming indirect pointer directly (zero-copy). + ArgValue = RVFI->getIncomingIndirectArg(FormalArg->getArgNo()); + } else { + // The call arg is a computed value. Store it into the incoming + // indirect pointer for the same-position formal parameter + // (musttail guarantees matching prototypes, so types match). + // The pointer survives the tail call since it points to the + // caller's caller's frame. + SDValue IncomingPtr = + RVFI->getIncomingIndirectArg(CallArgIdx); + MemOpChains.push_back( + DAG.getStore(Chain, DL, ArgValue, IncomingPtr, + MachinePointerInfo())); + // Store any split parts at their respective offsets. + unsigned ArgPartOffset = Outs[OutIdx].PartOffset; + while (i + 1 != e && + Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { + SDValue PartValue = OutVals[OutIdx + 1]; + unsigned PartOffset = + Outs[OutIdx + 1].PartOffset - ArgPartOffset; + SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, + DAG.getIntPtrConstant(PartOffset, DL)); + MemOpChains.push_back( + DAG.getStore(Chain, DL, PartValue, Addr, + MachinePointerInfo())); + ++i; + ++OutIdx; + } + ArgValue = IncomingPtr; + } + + // Skip any remaining split parts (for forwarded args, they are + // covered by the forwarded pointer). while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { ++i; ++OutIdx; diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index c6d1743a27d26..ec7eaad2b8924 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -350,3 +350,362 @@ define void @caller_musttail_nine_indirect_swap_first_last(fp128 %a, fp128 %b, f musttail call void @callee_musttail_nine_indirect(fp128 %i, fp128 %b, fp128 %c, fp128 %d, fp128 %e, fp128 %f, fp128 %g, fp128 %h, fp128 %a) ret void } + +; Test musttail where the indirect arg is a computed value, not a forwarded +; formal parameter. The computed value must be stored into the incoming +; indirect pointer before tail calling. +define i32 @caller_musttail_computed(fp128 %a) nounwind { +; RV32-LABEL: caller_musttail_computed: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -64 +; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill +; RV32-NEXT: mv s0, a0 +; RV32-NEXT: lw a3, 0(a0) +; RV32-NEXT: lw a4, 4(a0) +; RV32-NEXT: lw a5, 8(a0) +; RV32-NEXT: lw a6, 12(a0) +; RV32-NEXT: sw a3, 8(sp) +; RV32-NEXT: sw a4, 12(sp) +; RV32-NEXT: sw a5, 16(sp) +; RV32-NEXT: sw a6, 20(sp) +; RV32-NEXT: addi a0, sp, 40 +; RV32-NEXT: addi a1, sp, 24 +; RV32-NEXT: addi a2, sp, 8 +; RV32-NEXT: sw a3, 24(sp) +; RV32-NEXT: sw a4, 28(sp) +; RV32-NEXT: sw a5, 32(sp) +; RV32-NEXT: sw a6, 36(sp) +; RV32-NEXT: call __addtf3 +; RV32-NEXT: lw a0, 40(sp) +; RV32-NEXT: lw a1, 44(sp) +; RV32-NEXT: lw a2, 48(sp) +; RV32-NEXT: lw a3, 52(sp) +; RV32-NEXT: sw a0, 0(s0) +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: mv a0, s0 +; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 64 +; RV32-NEXT: tail callee_musttail_indirect +; +; RV64-LABEL: caller_musttail_computed: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: call __addtf3 +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: tail callee_musttail_indirect + %sum = fadd fp128 %a, %a + %r = musttail call i32 @callee_musttail_indirect(fp128 %sum) + ret i32 %r +} + +; Test musttail with a computed i128 on RV32 (split indirect). The add result +; must be stored back into the incoming pointer. +define i64 @caller_musttail_computed_i128(i128 %a) nounwind { +; RV32-LABEL: caller_musttail_computed_i128: +; RV32: # %bb.0: +; RV32-NEXT: lw a1, 0(a0) +; RV32-NEXT: lw a2, 4(a0) +; RV32-NEXT: lw a3, 8(a0) +; RV32-NEXT: lw a4, 12(a0) +; RV32-NEXT: addi a1, a1, 1 +; RV32-NEXT: seqz a5, a1 +; RV32-NEXT: add a2, a2, a5 +; RV32-NEXT: or a5, a1, a2 +; RV32-NEXT: seqz a5, a5 +; RV32-NEXT: add a5, a3, a5 +; RV32-NEXT: sltu a3, a5, a3 +; RV32-NEXT: add a3, a4, a3 +; RV32-NEXT: sw a1, 0(a0) +; RV32-NEXT: sw a2, 4(a0) +; RV32-NEXT: sw a5, 8(a0) +; RV32-NEXT: sw a3, 12(a0) +; RV32-NEXT: tail callee_musttail_i128 +; +; RV64-LABEL: caller_musttail_computed_i128: +; RV64: # %bb.0: +; RV64-NEXT: addi a0, a0, 1 +; RV64-NEXT: seqz a2, a0 +; RV64-NEXT: add a1, a1, a2 +; RV64-NEXT: tail callee_musttail_i128 + %sum = add i128 %a, 1 + %r = musttail call i64 @callee_musttail_i128(i128 %sum) + ret i64 %r +} + +; Test musttail with one computed and one forwarded indirect arg. +; Position 0 gets the fadd result (stored into %a's incoming pointer), +; position 1 gets %b's incoming pointer forwarded directly. +define i32 @caller_musttail_computed_and_forwarded(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_computed_and_forwarded: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -64 +; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill +; RV32-NEXT: mv s0, a1 +; RV32-NEXT: mv s1, a0 +; RV32-NEXT: lw a3, 0(a0) +; RV32-NEXT: lw a4, 4(a0) +; RV32-NEXT: lw a5, 8(a0) +; RV32-NEXT: lw a6, 12(a0) +; RV32-NEXT: lw a0, 0(a1) +; RV32-NEXT: lw a1, 4(a1) +; RV32-NEXT: lw a2, 8(s0) +; RV32-NEXT: lw a7, 12(s0) +; RV32-NEXT: sw a0, 0(sp) +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: sw a2, 8(sp) +; RV32-NEXT: sw a7, 12(sp) +; RV32-NEXT: addi a0, sp, 32 +; RV32-NEXT: addi a1, sp, 16 +; RV32-NEXT: mv a2, sp +; RV32-NEXT: sw a3, 16(sp) +; RV32-NEXT: sw a4, 20(sp) +; RV32-NEXT: sw a5, 24(sp) +; RV32-NEXT: sw a6, 28(sp) +; RV32-NEXT: call __addtf3 +; RV32-NEXT: lw a0, 32(sp) +; RV32-NEXT: lw a1, 36(sp) +; RV32-NEXT: lw a2, 40(sp) +; RV32-NEXT: lw a3, 44(sp) +; RV32-NEXT: sw a0, 0(s1) +; RV32-NEXT: sw a1, 4(s1) +; RV32-NEXT: sw a2, 8(s1) +; RV32-NEXT: sw a3, 12(s1) +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: mv a1, s0 +; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 52(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 64 +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_computed_and_forwarded: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -32 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: mv s0, a3 +; RV64-NEXT: mv s1, a2 +; RV64-NEXT: call __addtf3 +; RV64-NEXT: mv a2, s1 +; RV64-NEXT: mv a3, s0 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: tail callee_musttail_two_indirect + %sum = fadd fp128 %a, %b + %r = musttail call i32 @callee_musttail_two_indirect(fp128 %sum, fp128 %b) + ret i32 %r +} + +; Test musttail with one forwarded and one computed indirect arg (reversed). +; Position 0 forwards %a, position 1 gets the computed value. +define i32 @caller_musttail_forwarded_and_computed(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_forwarded_and_computed: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -64 +; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill +; RV32-NEXT: mv s0, a1 +; RV32-NEXT: mv s1, a0 +; RV32-NEXT: lw a3, 0(a0) +; RV32-NEXT: lw a4, 4(a0) +; RV32-NEXT: lw a5, 8(a0) +; RV32-NEXT: lw a6, 12(a0) +; RV32-NEXT: lw a0, 0(a1) +; RV32-NEXT: lw a1, 4(a1) +; RV32-NEXT: lw a2, 8(s0) +; RV32-NEXT: lw a7, 12(s0) +; RV32-NEXT: sw a0, 0(sp) +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: sw a2, 8(sp) +; RV32-NEXT: sw a7, 12(sp) +; RV32-NEXT: addi a0, sp, 32 +; RV32-NEXT: addi a1, sp, 16 +; RV32-NEXT: mv a2, sp +; RV32-NEXT: sw a3, 16(sp) +; RV32-NEXT: sw a4, 20(sp) +; RV32-NEXT: sw a5, 24(sp) +; RV32-NEXT: sw a6, 28(sp) +; RV32-NEXT: call __addtf3 +; RV32-NEXT: lw a0, 32(sp) +; RV32-NEXT: lw a1, 36(sp) +; RV32-NEXT: lw a2, 40(sp) +; RV32-NEXT: lw a3, 44(sp) +; RV32-NEXT: sw a0, 0(s0) +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: mv a1, s0 +; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 52(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 64 +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_forwarded_and_computed: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -32 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: mv s0, a1 +; RV64-NEXT: mv s1, a0 +; RV64-NEXT: call __addtf3 +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: mv a0, s1 +; RV64-NEXT: mv a1, s0 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: tail callee_musttail_two_indirect + %sum = fadd fp128 %a, %b + %r = musttail call i32 @callee_musttail_two_indirect(fp128 %a, fp128 %sum) + ret i32 %r +} + +; Test musttail with both args computed. Neither can be zero-copy forwarded. +define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { +; RV32-LABEL: caller_musttail_both_computed: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -160 +; RV32-NEXT: sw ra, 156(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 152(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 148(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s2, 144(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s3, 140(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s4, 136(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s5, 132(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s6, 128(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s7, 124(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s8, 120(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s9, 116(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s10, 112(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s11, 108(sp) # 4-byte Folded Spill +; RV32-NEXT: mv s0, a1 +; RV32-NEXT: mv s1, a0 +; RV32-NEXT: lw s2, 0(a0) +; RV32-NEXT: lw s3, 4(a0) +; RV32-NEXT: lw s4, 8(a0) +; RV32-NEXT: lw s5, 12(a0) +; RV32-NEXT: lw s6, 0(a1) +; RV32-NEXT: lw s7, 4(a1) +; RV32-NEXT: lw s8, 8(a1) +; RV32-NEXT: lw s9, 12(a1) +; RV32-NEXT: sw s6, 56(sp) +; RV32-NEXT: sw s7, 60(sp) +; RV32-NEXT: sw s8, 64(sp) +; RV32-NEXT: sw s9, 68(sp) +; RV32-NEXT: addi a0, sp, 88 +; RV32-NEXT: addi a1, sp, 72 +; RV32-NEXT: addi a2, sp, 56 +; RV32-NEXT: sw s2, 72(sp) +; RV32-NEXT: sw s3, 76(sp) +; RV32-NEXT: sw s4, 80(sp) +; RV32-NEXT: sw s5, 84(sp) +; RV32-NEXT: call __addtf3 +; RV32-NEXT: lw s10, 88(sp) +; RV32-NEXT: lw s11, 92(sp) +; RV32-NEXT: lw a0, 96(sp) +; RV32-NEXT: sw a0, 4(sp) # 4-byte Folded Spill +; RV32-NEXT: lw a0, 100(sp) +; RV32-NEXT: sw a0, 0(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s6, 8(sp) +; RV32-NEXT: sw s7, 12(sp) +; RV32-NEXT: sw s8, 16(sp) +; RV32-NEXT: sw s9, 20(sp) +; RV32-NEXT: addi a0, sp, 40 +; RV32-NEXT: addi a1, sp, 24 +; RV32-NEXT: addi a2, sp, 8 +; RV32-NEXT: sw s2, 24(sp) +; RV32-NEXT: sw s3, 28(sp) +; RV32-NEXT: sw s4, 32(sp) +; RV32-NEXT: sw s5, 36(sp) +; RV32-NEXT: call __subtf3 +; RV32-NEXT: lw a0, 40(sp) +; RV32-NEXT: lw a1, 44(sp) +; RV32-NEXT: lw a2, 48(sp) +; RV32-NEXT: lw a3, 52(sp) +; RV32-NEXT: sw a0, 0(s0) +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: sw s10, 0(s1) +; RV32-NEXT: sw s11, 4(s1) +; RV32-NEXT: lw a0, 4(sp) # 4-byte Folded Reload +; RV32-NEXT: sw a0, 8(s1) +; RV32-NEXT: lw a0, 0(sp) # 4-byte Folded Reload +; RV32-NEXT: sw a0, 12(s1) +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: mv a1, s0 +; RV32-NEXT: lw ra, 156(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 152(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 148(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s2, 144(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s3, 140(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s4, 136(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s5, 132(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s6, 128(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s7, 124(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s8, 120(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s9, 116(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s10, 112(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s11, 108(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 160 +; RV32-NEXT: tail callee_musttail_two_indirect +; +; RV64-LABEL: caller_musttail_both_computed: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -64 +; RV64-NEXT: sd ra, 56(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 48(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 40(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s2, 32(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s3, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s4, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s5, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: mv s0, a3 +; RV64-NEXT: mv s1, a2 +; RV64-NEXT: mv s2, a1 +; RV64-NEXT: mv s3, a0 +; RV64-NEXT: call __addtf3 +; RV64-NEXT: mv s4, a0 +; RV64-NEXT: mv s5, a1 +; RV64-NEXT: mv a0, s3 +; RV64-NEXT: mv a1, s2 +; RV64-NEXT: mv a2, s1 +; RV64-NEXT: mv a3, s0 +; RV64-NEXT: call __subtf3 +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: mv a0, s4 +; RV64-NEXT: mv a1, s5 +; RV64-NEXT: ld ra, 56(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 48(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 40(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s2, 32(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s3, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s4, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s5, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 64 +; RV64-NEXT: tail callee_musttail_two_indirect + %sum = fadd fp128 %a, %b + %diff = fsub fp128 %a, %b + %r = musttail call i32 @callee_musttail_two_indirect(fp128 %sum, fp128 %diff) + ret i32 %r +} From 484f135f9dfdbd97360bde0a7d29a1b46f4ab762 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Mon, 9 Mar 2026 13:30:39 +0100 Subject: [PATCH 05/15] [RISCV][NFC] Apply clang-format to musttail indirect arg forwarding Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index fb0babc6b9dba..545c2a397d707 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24812,23 +24812,18 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // (musttail guarantees matching prototypes, so types match). // The pointer survives the tail call since it points to the // caller's caller's frame. - SDValue IncomingPtr = - RVFI->getIncomingIndirectArg(CallArgIdx); - MemOpChains.push_back( - DAG.getStore(Chain, DL, ArgValue, IncomingPtr, - MachinePointerInfo())); + SDValue IncomingPtr = RVFI->getIncomingIndirectArg(CallArgIdx); + MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, IncomingPtr, + MachinePointerInfo())); // Store any split parts at their respective offsets. unsigned ArgPartOffset = Outs[OutIdx].PartOffset; - while (i + 1 != e && - Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { SDValue PartValue = OutVals[OutIdx + 1]; - unsigned PartOffset = - Outs[OutIdx + 1].PartOffset - ArgPartOffset; + unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, DAG.getIntPtrConstant(PartOffset, DL)); MemOpChains.push_back( - DAG.getStore(Chain, DL, PartValue, Addr, - MachinePointerInfo())); + DAG.getStore(Chain, DL, PartValue, Addr, MachinePointerInfo())); ++i; ++OutIdx; } From 76df154f8c1944f49d64e4590d3d87a558e6aa37 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Sun, 19 Apr 2026 17:38:09 +0200 Subject: [PATCH 06/15] [RISCV] Fix SDValue lifetime for musttail indirect arg forwarding The SDValue stored during LowerFormalArguments points into the entry block's DAG, which is cleared at the end of CodeGenAndEmitDAG(). If the musttail call is in a non-entry basic block, the stored SDValue is stale. Fix by saving the indirect pointer in a virtual register (CopyToReg in LowerFormalArguments, CopyFromReg in LowerCall). The vreg survives across basic blocks. The DenseMap in RVFI now stores Register instead of SDValue, and the header does not include SelectionDAGNodes.h. The vreg save is guarded behind a HasMusttail check so functions without musttail calls see no codegen change. Also adds musttail early return in isEligibleForTailCallOptimization (musttail must always be tail-called), cross-BB test cases, and converts musttail-call.ll from a crash test to a positive test. Fixes #185089. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 202 ++++++++--- .../Target/RISCV/RISCVMachineFunctionInfo.h | 18 + llvm/test/CodeGen/RISCV/musttail-call.ll | 17 +- .../CodeGen/RISCV/musttail-indirect-args.ll | 327 ++++++++++++------ 4 files changed, 404 insertions(+), 160 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4dc2e1c03b7f3..b5b36ed0282ae 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24352,6 +24352,18 @@ SDValue RISCVTargetLowering::LowerFormalArguments( EVT PtrVT = getPointerTy(DAG.getDataLayout()); MVT XLenVT = Subtarget.getXLenVT(); unsigned XLenInBytes = Subtarget.getXLen() / 8; + + // Check if this function has any musttail calls. If so, incoming indirect + // arg pointers must be saved in virtual registers so they survive across + // basic blocks (the SelectionDAG is cleared between BBs). Only do this + // when needed to avoid adding register pressure to non-musttail functions. + bool HasMusttail = llvm::any_of(Func, [](const BasicBlock &BB) { + return llvm::any_of(BB, [](const Instruction &I) { + if (const auto *CI = dyn_cast(&I)) + return CI->isMustTailCall(); + return false; + }); + }); // Used with vargs to accumulate store chains. std::vector OutChains; @@ -24383,6 +24395,13 @@ SDValue RISCVTargetLowering::LowerFormalArguments( InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, MachinePointerInfo())); unsigned ArgIndex = Ins[InsIdx].OrigArgIndex; + if (HasMusttail) { + RISCVMachineFunctionInfo *RVFI = MF.getInfo(); + Register VReg = + MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); + Chain = DAG.getCopyToReg(Chain, DL, VReg, ArgValue); + RVFI->setIncomingIndirectArg(ArgIndex, VReg); + } unsigned ArgPartOffset = Ins[InsIdx].PartOffset; assert(VA.getValVT().isVector() || ArgPartOffset == 0); while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) { @@ -24491,18 +24510,20 @@ bool RISCVTargetLowering::isEligibleForTailCallOptimization( if (Caller.hasFnAttribute("interrupt")) return false; + // musttail calls must always be tail-called. The remaining checks are + // for optional tail call optimization of regular (non-musttail) calls. + if (CLI.CB && CLI.CB->isMustTailCall()) + return true; + // Do not tail call opt if the stack is used to pass parameters. if (CCInfo.getStackSize() != 0) return false; // Do not tail call opt if any parameters need to be passed indirectly. // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are - // passed indirectly. So the address of the value will be passed in a - // register, or if not available, then the address is put on the stack. In - // order to pass indirectly, space on the stack often needs to be allocated - // in order to store the value. In this case the CCInfo.getNextStackOffset() - // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs - // are passed CCValAssign::Indirect. + // passed indirectly. The caller allocates stack space for the value and + // passes a pointer. On a tail call the caller's frame is deallocated + // before the callee executes, leaving the pointer dangling. for (auto &VA : ArgLocs) if (VA.getLocInfo() == CCValAssign::Indirect) return false; @@ -24665,51 +24686,132 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Promote the value if needed. // For now, only handle fully promoted and indirect arguments. if (VA.getLocInfo() == CCValAssign::Indirect) { - // Store the argument in a stack slot and pass its address. - Align StackAlign = - std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG), - getPrefTypeAlign(ArgValue.getValueType(), DAG)); - TypeSize StoredSize = ArgValue.getValueType().getStoreSize(); - // If the original argument was split (e.g. i128), we need - // to store the required parts of it here (and pass just one address). - // Vectors may be partly split to registers and partly to the stack, in - // which case the base address is partly offset and subsequent stores are - // relative to that. - unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; - unsigned ArgPartOffset = Outs[OutIdx].PartOffset; - assert(VA.getValVT().isVector() || ArgPartOffset == 0); - // Calculate the total size to store. We don't have access to what we're - // actually storing other than performing the loop and collecting the - // info. - SmallVector> Parts; - while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { - SDValue PartValue = OutVals[OutIdx + 1]; - unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; - SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); - EVT PartVT = PartValue.getValueType(); - if (PartVT.isScalableVector()) - Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); - StoredSize += PartVT.getStoreSize(); - StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG)); - Parts.push_back(std::make_pair(PartValue, Offset)); - ++i; - ++OutIdx; - } - SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign); - int FI = cast(SpillSlot)->getIndex(); - MemOpChains.push_back( - DAG.getStore(Chain, DL, ArgValue, SpillSlot, - MachinePointerInfo::getFixedStack(MF, FI))); - for (const auto &Part : Parts) { - SDValue PartValue = Part.first; - SDValue PartOffset = Part.second; - SDValue Address = - DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); + // For musttail calls, reuse incoming indirect pointers instead of + // creating new stack temporaries. The incoming pointers point to the + // caller's caller's frame, which remains valid after a tail call. + if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) { + RISCVMachineFunctionInfo *RVFI = MF.getInfo(); + unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex; + + // Resolve which formal parameter is being passed at this call + // position. Outs[].OrigArgIndex indexes the filtered arg list + // (empty types removed), but IncomingIndirectArgs is keyed by + // Argument::getArgNo() (unfiltered position). We need to: + // 1. Find the call operand at filtered position CallArgIdx + // 2. Check if it's a forwarded formal param (dyn_cast) + // 3. Resolve the unfiltered formal param index for the map lookup + const Argument *FormalArg = nullptr; + unsigned FilteredIdx = 0; + for (const auto &CallArg : CLI.CB->args()) { + if (CallArg->getType()->isEmptyTy()) + continue; + if (FilteredIdx == CallArgIdx) { + FormalArg = dyn_cast(CallArg); + break; + } + ++FilteredIdx; + } + + // For forwarded args, getArgNo() gives the unfiltered index directly. + // For computed args, walk the caller's formals to resolve it. + unsigned FormalArgIdx = CallArgIdx; + if (FormalArg) { + FormalArgIdx = FormalArg->getArgNo(); + } else { + const Function *CallerFn = CLI.CB->getFunction(); + FilteredIdx = 0; + for (const auto &Arg : CallerFn->args()) { + if (Arg.getType()->isEmptyTy()) + continue; + if (FilteredIdx == CallArgIdx) { + FormalArgIdx = Arg.getArgNo(); + break; + } + ++FilteredIdx; + } + } + + if (FormalArg) { + // Forwarded formal parameter: reuse its incoming indirect pointer. + Register VReg = RVFI->getIncomingIndirectArg(FormalArgIdx); + ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT); + } else { + // Computed value: store into the incoming indirect pointer for the + // same-position formal parameter (musttail guarantees matching + // prototypes, so types match). The pointer survives the tail call + // since it points to the caller's caller's frame. + Register VReg = RVFI->getIncomingIndirectArg(FormalArgIdx); + SDValue IncomingPtr = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT); + MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, IncomingPtr, + MachinePointerInfo())); + // Store any split parts at their respective offsets. + unsigned ArgPartOffset = Outs[OutIdx].PartOffset; + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { + SDValue PartValue = OutVals[OutIdx + 1]; + unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; + SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, + DAG.getIntPtrConstant(PartOffset, DL)); + MemOpChains.push_back( + DAG.getStore(Chain, DL, PartValue, Addr, MachinePointerInfo())); + ++i; + ++OutIdx; + } + ArgValue = IncomingPtr; + } + + // Skip any remaining split parts (for forwarded args, they are + // covered by the forwarded pointer). + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { + ++i; + ++OutIdx; + } + } else { + // Store the argument in a stack slot and pass its address. + Align StackAlign = + std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG), + getPrefTypeAlign(ArgValue.getValueType(), DAG)); + TypeSize StoredSize = ArgValue.getValueType().getStoreSize(); + // If the original argument was split (e.g. i128), we need + // to store the required parts of it here (and pass just one address). + // Vectors may be partly split to registers and partly to the stack, in + // which case the base address is partly offset and subsequent stores + // are relative to that. + unsigned ArgIndex = Outs[OutIdx].OrigArgIndex; + unsigned ArgPartOffset = Outs[OutIdx].PartOffset; + assert(VA.getValVT().isVector() || ArgPartOffset == 0); + // Calculate the total size to store. We don't have access to what + // we're actually storing other than performing the loop and collecting + // the info. + SmallVector> Parts; + while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) { + SDValue PartValue = OutVals[OutIdx + 1]; + unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; + SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); + EVT PartVT = PartValue.getValueType(); + if (PartVT.isScalableVector()) + Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); + StoredSize += PartVT.getStoreSize(); + StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG)); + Parts.push_back(std::make_pair(PartValue, Offset)); + ++i; + ++OutIdx; + } + SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign); + int FI = cast(SpillSlot)->getIndex(); MemOpChains.push_back( - DAG.getStore(Chain, DL, PartValue, Address, + DAG.getStore(Chain, DL, ArgValue, SpillSlot, MachinePointerInfo::getFixedStack(MF, FI))); + for (const auto &Part : Parts) { + SDValue PartValue = Part.first; + SDValue PartOffset = Part.second; + SDValue Address = + DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset); + MemOpChains.push_back( + DAG.getStore(Chain, DL, PartValue, Address, + MachinePointerInfo::getFixedStack(MF, FI))); + } + ArgValue = SpillSlot; } - ArgValue = SpillSlot; } else { ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget); } @@ -24727,8 +24829,8 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), i); } else { assert(VA.isMemLoc() && "Argument not register or memory"); - assert(!IsTailCall && "Tail call not allowed if stack is used " - "for passing parameters"); + assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) && + "Tail call not allowed if stack is used for passing parameters"); // Work out the address of the stack slot. if (!StackPtr.getNode()) diff --git a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h index 854f2714d9599..06ef7661385c2 100644 --- a/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h +++ b/llvm/lib/Target/RISCV/RISCVMachineFunctionInfo.h @@ -14,6 +14,7 @@ #define LLVM_LIB_TARGET_RISCV_RISCVMACHINEFUNCTIONINFO_H #include "RISCVSubtarget.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MIRYamlMapping.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -65,6 +66,14 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { uint64_t RVVPadding = 0; /// Size of stack frame to save callee saved registers unsigned CalleeSavedStackSize = 0; + + /// Incoming indirect argument pointers saved as virtual registers, keyed by + /// formal parameter index. Used for musttail forwarding of indirect args. + /// Virtual registers (not SDValues) are used because the SelectionDAG is + /// cleared between basic blocks, and musttail calls may be in non-entry + /// blocks. + DenseMap IncomingIndirectArgs; + /// Is there any vector argument or return? bool IsVectorCall = false; @@ -145,6 +154,15 @@ class RISCVMachineFunctionInfo : public MachineFunctionInfo { unsigned getCalleeSavedStackSize() const { return CalleeSavedStackSize; } void setCalleeSavedStackSize(unsigned Size) { CalleeSavedStackSize = Size; } + void setIncomingIndirectArg(unsigned ArgIndex, Register Reg) { + IncomingIndirectArgs[ArgIndex] = Reg; + } + Register getIncomingIndirectArg(unsigned ArgIndex) const { + auto It = IncomingIndirectArgs.find(ArgIndex); + assert(It != IncomingIndirectArgs.end() && "No incoming indirect arg"); + return It->second; + } + enum class PushPopKind { None = 0, StdExtZcmp, VendorXqccmp }; PushPopKind getPushPopKind(const MachineFunction &MF) const; diff --git a/llvm/test/CodeGen/RISCV/musttail-call.ll b/llvm/test/CodeGen/RISCV/musttail-call.ll index f6ec5307b8bad..ee58a1943c655 100644 --- a/llvm/test/CodeGen/RISCV/musttail-call.ll +++ b/llvm/test/CodeGen/RISCV/musttail-call.ll @@ -1,19 +1,16 @@ -; Check that we error out if tail is not possible but call is marked as mustail. +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; Check that musttail with sret generates a tail call (not an error). -; RUN: not --crash llc -mtriple riscv32-unknown-linux-gnu -o - %s \ -; RUN: 2>&1 | FileCheck %s -; RUN: not --crash llc -mtriple riscv32-unknown-elf -o - %s \ -; RUN: 2>&1 | FileCheck %s -; RUN: not --crash llc -mtriple riscv64-unknown-linux-gnu -o - %s \ -; RUN: 2>&1 | FileCheck %s -; RUN: not --crash llc -mtriple riscv64-unknown-elf -o - %s \ -; RUN: 2>&1 | FileCheck %s +; RUN: llc -mtriple riscv32-unknown-linux-gnu -o - %s | FileCheck %s +; RUN: llc -mtriple riscv64-unknown-linux-gnu -o - %s | FileCheck %s %struct.A = type { i32 } declare void @callee_musttail(ptr sret(%struct.A) %a) define void @caller_musttail(ptr sret(%struct.A) %a) { -; CHECK: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail +; CHECK-LABEL: caller_musttail: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: tail callee_musttail entry: musttail call void @callee_musttail(ptr sret(%struct.A) %a) ret void diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index ec7eaad2b8924..0fc4ab6b91093 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -276,26 +276,26 @@ define void @caller_musttail_nine_indirect(fp128 %a, fp128 %b, fp128 %c, fp128 % ; RV64-NEXT: sd s0, 24(sp) # 8-byte Folded Spill ; RV64-NEXT: sd s1, 16(sp) # 8-byte Folded Spill ; RV64-NEXT: sd s2, 8(sp) # 8-byte Folded Spill -; RV64-NEXT: ld t0, 104(sp) -; RV64-NEXT: ld t1, 96(sp) -; RV64-NEXT: ld t2, 88(sp) -; RV64-NEXT: ld t3, 80(sp) -; RV64-NEXT: ld t4, 72(sp) -; RV64-NEXT: ld t5, 64(sp) -; RV64-NEXT: ld t6, 32(sp) -; RV64-NEXT: ld s0, 40(sp) -; RV64-NEXT: ld s1, 48(sp) -; RV64-NEXT: ld s2, 56(sp) -; RV64-NEXT: sd t6, 32(sp) -; RV64-NEXT: sd s0, 40(sp) +; RV64-NEXT: ld t0, 32(sp) +; RV64-NEXT: ld t1, 40(sp) +; RV64-NEXT: ld t2, 48(sp) +; RV64-NEXT: ld t3, 56(sp) +; RV64-NEXT: ld t4, 64(sp) +; RV64-NEXT: ld t5, 72(sp) +; RV64-NEXT: ld t6, 96(sp) +; RV64-NEXT: ld s0, 104(sp) +; RV64-NEXT: ld s1, 80(sp) +; RV64-NEXT: ld s2, 88(sp) +; RV64-NEXT: sd t6, 64(sp) +; RV64-NEXT: sd s0, 72(sp) +; RV64-NEXT: sd t4, 32(sp) +; RV64-NEXT: sd t5, 40(sp) ; RV64-NEXT: sd s1, 48(sp) ; RV64-NEXT: sd s2, 56(sp) -; RV64-NEXT: sd t5, 64(sp) -; RV64-NEXT: sd t4, 72(sp) -; RV64-NEXT: sd t3, 80(sp) -; RV64-NEXT: sd t2, 88(sp) -; RV64-NEXT: sd t1, 96(sp) -; RV64-NEXT: sd t0, 104(sp) +; RV64-NEXT: sd t0, 0(sp) +; RV64-NEXT: sd t1, 8(sp) +; RV64-NEXT: sd t2, 16(sp) +; RV64-NEXT: sd t3, 24(sp) ; RV64-NEXT: ld s0, 24(sp) # 8-byte Folded Reload ; RV64-NEXT: ld s1, 16(sp) # 8-byte Folded Reload ; RV64-NEXT: ld s2, 8(sp) # 8-byte Folded Reload @@ -322,24 +322,24 @@ define void @caller_musttail_nine_indirect_swap_first_last(fp128 %a, fp128 %b, f ; RV64-NEXT: sd s2, 8(sp) # 8-byte Folded Spill ; RV64-NEXT: ld t0, 96(sp) ; RV64-NEXT: ld t1, 104(sp) -; RV64-NEXT: ld t2, 88(sp) -; RV64-NEXT: ld t3, 80(sp) -; RV64-NEXT: ld t4, 72(sp) -; RV64-NEXT: ld t5, 64(sp) -; RV64-NEXT: ld t6, 32(sp) -; RV64-NEXT: ld s0, 40(sp) -; RV64-NEXT: ld s1, 48(sp) -; RV64-NEXT: ld s2, 56(sp) +; RV64-NEXT: ld t2, 32(sp) +; RV64-NEXT: ld t3, 40(sp) +; RV64-NEXT: ld t4, 48(sp) +; RV64-NEXT: ld t5, 56(sp) +; RV64-NEXT: ld t6, 64(sp) +; RV64-NEXT: ld s0, 72(sp) +; RV64-NEXT: ld s1, 80(sp) +; RV64-NEXT: ld s2, 88(sp) +; RV64-NEXT: sd a0, 64(sp) +; RV64-NEXT: sd a1, 72(sp) ; RV64-NEXT: sd t6, 32(sp) ; RV64-NEXT: sd s0, 40(sp) ; RV64-NEXT: sd s1, 48(sp) ; RV64-NEXT: sd s2, 56(sp) -; RV64-NEXT: sd t5, 64(sp) -; RV64-NEXT: sd t4, 72(sp) -; RV64-NEXT: sd t3, 80(sp) -; RV64-NEXT: sd t2, 88(sp) -; RV64-NEXT: sd a0, 96(sp) -; RV64-NEXT: sd a1, 104(sp) +; RV64-NEXT: sd t2, 0(sp) +; RV64-NEXT: sd t3, 8(sp) +; RV64-NEXT: sd t4, 16(sp) +; RV64-NEXT: sd t5, 24(sp) ; RV64-NEXT: mv a0, t0 ; RV64-NEXT: mv a1, t1 ; RV64-NEXT: ld s0, 24(sp) # 8-byte Folded Reload @@ -361,21 +361,21 @@ define i32 @caller_musttail_computed(fp128 %a) nounwind { ; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill ; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill ; RV32-NEXT: mv s0, a0 -; RV32-NEXT: lw a3, 0(a0) -; RV32-NEXT: lw a4, 4(a0) -; RV32-NEXT: lw a5, 8(a0) -; RV32-NEXT: lw a6, 12(a0) -; RV32-NEXT: sw a3, 8(sp) -; RV32-NEXT: sw a4, 12(sp) -; RV32-NEXT: sw a5, 16(sp) -; RV32-NEXT: sw a6, 20(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lw a3, 4(s0) +; RV32-NEXT: lw a4, 8(s0) +; RV32-NEXT: lw a5, 12(s0) +; RV32-NEXT: sw a0, 8(sp) +; RV32-NEXT: sw a0, 24(sp) +; RV32-NEXT: sw a3, 12(sp) +; RV32-NEXT: sw a4, 16(sp) +; RV32-NEXT: sw a5, 20(sp) ; RV32-NEXT: addi a0, sp, 40 ; RV32-NEXT: addi a1, sp, 24 ; RV32-NEXT: addi a2, sp, 8 -; RV32-NEXT: sw a3, 24(sp) -; RV32-NEXT: sw a4, 28(sp) -; RV32-NEXT: sw a5, 32(sp) -; RV32-NEXT: sw a6, 36(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) ; RV32-NEXT: call __addtf3 ; RV32-NEXT: lw a0, 40(sp) ; RV32-NEXT: lw a1, 44(sp) @@ -452,25 +452,25 @@ define i32 @caller_musttail_computed_and_forwarded(fp128 %a, fp128 %b) nounwind ; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill ; RV32-NEXT: mv s0, a1 ; RV32-NEXT: mv s1, a0 -; RV32-NEXT: lw a3, 0(a0) -; RV32-NEXT: lw a4, 4(a0) -; RV32-NEXT: lw a5, 8(a0) -; RV32-NEXT: lw a6, 12(a0) -; RV32-NEXT: lw a0, 0(a1) -; RV32-NEXT: lw a1, 4(a1) -; RV32-NEXT: lw a2, 8(s0) -; RV32-NEXT: lw a7, 12(s0) -; RV32-NEXT: sw a0, 0(sp) -; RV32-NEXT: sw a1, 4(sp) -; RV32-NEXT: sw a2, 8(sp) -; RV32-NEXT: sw a7, 12(sp) +; RV32-NEXT: lw a3, 0(a1) +; RV32-NEXT: lw a4, 4(a1) +; RV32-NEXT: lw a5, 8(a1) +; RV32-NEXT: lw a6, 12(a1) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lw a1, 4(s1) +; RV32-NEXT: lw a2, 8(s1) +; RV32-NEXT: lw a7, 12(s1) +; RV32-NEXT: sw a0, 16(sp) +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a7, 28(sp) ; RV32-NEXT: addi a0, sp, 32 ; RV32-NEXT: addi a1, sp, 16 ; RV32-NEXT: mv a2, sp -; RV32-NEXT: sw a3, 16(sp) -; RV32-NEXT: sw a4, 20(sp) -; RV32-NEXT: sw a5, 24(sp) -; RV32-NEXT: sw a6, 28(sp) +; RV32-NEXT: sw a3, 0(sp) +; RV32-NEXT: sw a4, 4(sp) +; RV32-NEXT: sw a5, 8(sp) +; RV32-NEXT: sw a6, 12(sp) ; RV32-NEXT: call __addtf3 ; RV32-NEXT: lw a0, 32(sp) ; RV32-NEXT: lw a1, 36(sp) @@ -520,25 +520,25 @@ define i32 @caller_musttail_forwarded_and_computed(fp128 %a, fp128 %b) nounwind ; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill ; RV32-NEXT: mv s0, a1 ; RV32-NEXT: mv s1, a0 -; RV32-NEXT: lw a3, 0(a0) -; RV32-NEXT: lw a4, 4(a0) -; RV32-NEXT: lw a5, 8(a0) -; RV32-NEXT: lw a6, 12(a0) -; RV32-NEXT: lw a0, 0(a1) -; RV32-NEXT: lw a1, 4(a1) -; RV32-NEXT: lw a2, 8(s0) -; RV32-NEXT: lw a7, 12(s0) -; RV32-NEXT: sw a0, 0(sp) -; RV32-NEXT: sw a1, 4(sp) -; RV32-NEXT: sw a2, 8(sp) -; RV32-NEXT: sw a7, 12(sp) +; RV32-NEXT: lw a3, 0(a1) +; RV32-NEXT: lw a4, 4(a1) +; RV32-NEXT: lw a5, 8(a1) +; RV32-NEXT: lw a6, 12(a1) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lw a1, 4(s1) +; RV32-NEXT: lw a2, 8(s1) +; RV32-NEXT: lw a7, 12(s1) +; RV32-NEXT: sw a0, 16(sp) +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a7, 28(sp) ; RV32-NEXT: addi a0, sp, 32 ; RV32-NEXT: addi a1, sp, 16 ; RV32-NEXT: mv a2, sp -; RV32-NEXT: sw a3, 16(sp) -; RV32-NEXT: sw a4, 20(sp) -; RV32-NEXT: sw a5, 24(sp) -; RV32-NEXT: sw a6, 28(sp) +; RV32-NEXT: sw a3, 0(sp) +; RV32-NEXT: sw a4, 4(sp) +; RV32-NEXT: sw a5, 8(sp) +; RV32-NEXT: sw a6, 12(sp) ; RV32-NEXT: call __addtf3 ; RV32-NEXT: lw a0, 32(sp) ; RV32-NEXT: lw a1, 36(sp) @@ -599,25 +599,25 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { ; RV32-NEXT: sw s11, 108(sp) # 4-byte Folded Spill ; RV32-NEXT: mv s0, a1 ; RV32-NEXT: mv s1, a0 -; RV32-NEXT: lw s2, 0(a0) -; RV32-NEXT: lw s3, 4(a0) -; RV32-NEXT: lw s4, 8(a0) -; RV32-NEXT: lw s5, 12(a0) -; RV32-NEXT: lw s6, 0(a1) -; RV32-NEXT: lw s7, 4(a1) -; RV32-NEXT: lw s8, 8(a1) -; RV32-NEXT: lw s9, 12(a1) -; RV32-NEXT: sw s6, 56(sp) -; RV32-NEXT: sw s7, 60(sp) -; RV32-NEXT: sw s8, 64(sp) -; RV32-NEXT: sw s9, 68(sp) +; RV32-NEXT: lw s2, 0(a1) +; RV32-NEXT: lw s3, 4(a1) +; RV32-NEXT: lw s4, 8(a1) +; RV32-NEXT: lw s5, 12(a1) +; RV32-NEXT: lw s6, 0(a0) +; RV32-NEXT: lw s7, 4(a0) +; RV32-NEXT: lw s8, 8(a0) +; RV32-NEXT: lw s9, 12(a0) +; RV32-NEXT: sw s6, 72(sp) +; RV32-NEXT: sw s7, 76(sp) +; RV32-NEXT: sw s8, 80(sp) +; RV32-NEXT: sw s9, 84(sp) ; RV32-NEXT: addi a0, sp, 88 ; RV32-NEXT: addi a1, sp, 72 ; RV32-NEXT: addi a2, sp, 56 -; RV32-NEXT: sw s2, 72(sp) -; RV32-NEXT: sw s3, 76(sp) -; RV32-NEXT: sw s4, 80(sp) -; RV32-NEXT: sw s5, 84(sp) +; RV32-NEXT: sw s2, 56(sp) +; RV32-NEXT: sw s3, 60(sp) +; RV32-NEXT: sw s4, 64(sp) +; RV32-NEXT: sw s5, 68(sp) ; RV32-NEXT: call __addtf3 ; RV32-NEXT: lw s10, 88(sp) ; RV32-NEXT: lw s11, 92(sp) @@ -625,17 +625,17 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { ; RV32-NEXT: sw a0, 4(sp) # 4-byte Folded Spill ; RV32-NEXT: lw a0, 100(sp) ; RV32-NEXT: sw a0, 0(sp) # 4-byte Folded Spill -; RV32-NEXT: sw s6, 8(sp) -; RV32-NEXT: sw s7, 12(sp) -; RV32-NEXT: sw s8, 16(sp) -; RV32-NEXT: sw s9, 20(sp) +; RV32-NEXT: sw s6, 24(sp) +; RV32-NEXT: sw s7, 28(sp) +; RV32-NEXT: sw s8, 32(sp) +; RV32-NEXT: sw s9, 36(sp) ; RV32-NEXT: addi a0, sp, 40 ; RV32-NEXT: addi a1, sp, 24 ; RV32-NEXT: addi a2, sp, 8 -; RV32-NEXT: sw s2, 24(sp) -; RV32-NEXT: sw s3, 28(sp) -; RV32-NEXT: sw s4, 32(sp) -; RV32-NEXT: sw s5, 36(sp) +; RV32-NEXT: sw s2, 8(sp) +; RV32-NEXT: sw s3, 12(sp) +; RV32-NEXT: sw s4, 16(sp) +; RV32-NEXT: sw s5, 20(sp) ; RV32-NEXT: call __subtf3 ; RV32-NEXT: lw a0, 40(sp) ; RV32-NEXT: lw a1, 44(sp) @@ -709,3 +709,130 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { %r = musttail call i32 @callee_musttail_two_indirect(fp128 %sum, fp128 %diff) ret i32 %r } + +; Test musttail in a non-entry basic block. The indirect pointer must survive +; across basic blocks (the SelectionDAG is cleared between BBs, so the pointer +; must be preserved in a virtual register, not as a raw SDValue). +declare i32 @callee_musttail_cross_bb(fp128 %a, i1 %c) + +define i32 @caller_musttail_cross_bb(fp128 %a, i1 %cond) nounwind { +; RV32-LABEL: caller_musttail_cross_bb: +; RV32: # %bb.0: # %entry +; RV32-NEXT: andi a2, a1, 1 +; RV32-NEXT: beqz a2, .LBB19_2 +; RV32-NEXT: # %bb.1: # %then +; RV32-NEXT: tail callee_musttail_cross_bb +; RV32-NEXT: .LBB19_2: # %else +; RV32-NEXT: li a0, 0 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_musttail_cross_bb: +; RV64: # %bb.0: # %entry +; RV64-NEXT: andi a3, a2, 1 +; RV64-NEXT: beqz a3, .LBB19_2 +; RV64-NEXT: # %bb.1: # %then +; RV64-NEXT: tail callee_musttail_cross_bb +; RV64-NEXT: .LBB19_2: # %else +; RV64-NEXT: li a0, 0 +; RV64-NEXT: ret +entry: + br i1 %cond, label %then, label %else +then: + %r = musttail call i32 @callee_musttail_cross_bb(fp128 %a, i1 %cond) + ret i32 %r +else: + ret i32 0 +} + +; Test musttail with control flow and a computed indirect arg in a non-entry BB. +declare i32 @callee_musttail_cross_bb_computed(fp128 %a, i1 %c) + +define i32 @caller_musttail_cross_bb_computed(fp128 %a, i1 %cond) nounwind { +; RV32-LABEL: caller_musttail_cross_bb_computed: +; RV32: # %bb.0: # %entry +; RV32-NEXT: addi sp, sp, -64 +; RV32-NEXT: sw ra, 60(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 56(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 52(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s2, 48(sp) # 4-byte Folded Spill +; RV32-NEXT: mv s0, a0 +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lw a3, 4(s0) +; RV32-NEXT: lw a4, 8(s0) +; RV32-NEXT: lw a5, 12(s0) +; RV32-NEXT: mv s1, a1 +; RV32-NEXT: andi s2, a1, 1 +; RV32-NEXT: sw a0, 0(sp) +; RV32-NEXT: sw a0, 16(sp) +; RV32-NEXT: sw a3, 4(sp) +; RV32-NEXT: sw a4, 8(sp) +; RV32-NEXT: sw a5, 12(sp) +; RV32-NEXT: addi a0, sp, 32 +; RV32-NEXT: addi a1, sp, 16 +; RV32-NEXT: mv a2, sp +; RV32-NEXT: sw a3, 20(sp) +; RV32-NEXT: sw a4, 24(sp) +; RV32-NEXT: sw a5, 28(sp) +; RV32-NEXT: call __addtf3 +; RV32-NEXT: beqz s2, .LBB20_2 +; RV32-NEXT: # %bb.1: # %then +; RV32-NEXT: lw a0, 32(sp) +; RV32-NEXT: lw a1, 36(sp) +; RV32-NEXT: lw a2, 40(sp) +; RV32-NEXT: lw a3, 44(sp) +; RV32-NEXT: sw a0, 0(s0) +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: mv a0, s0 +; RV32-NEXT: mv a1, s1 +; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 52(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s2, 48(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 64 +; RV32-NEXT: tail callee_musttail_cross_bb_computed +; RV32-NEXT: .LBB20_2: # %else +; RV32-NEXT: li a0, 0 +; RV32-NEXT: lw ra, 60(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 56(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 52(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s2, 48(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 64 +; RV32-NEXT: ret +; +; RV64-LABEL: caller_musttail_cross_bb_computed: +; RV64: # %bb.0: # %entry +; RV64-NEXT: addi sp, sp, -32 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: mv s0, a2 +; RV64-NEXT: andi s1, a2, 1 +; RV64-NEXT: mv a2, a0 +; RV64-NEXT: mv a3, a1 +; RV64-NEXT: call __addtf3 +; RV64-NEXT: beqz s1, .LBB20_2 +; RV64-NEXT: # %bb.1: # %then +; RV64-NEXT: mv a2, s0 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: tail callee_musttail_cross_bb_computed +; RV64-NEXT: .LBB20_2: # %else +; RV64-NEXT: li a0, 0 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 32 +; RV64-NEXT: ret +entry: + %sum = fadd fp128 %a, %a + br i1 %cond, label %then, label %else +then: + %r = musttail call i32 @callee_musttail_cross_bb_computed(fp128 %sum, i1 %cond) + ret i32 %r +else: + ret i32 0 +} From 782cd6b599132a84b48da70fd3c17957472db5b4 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Tue, 21 Apr 2026 15:11:44 +0200 Subject: [PATCH 07/15] [RISCV] Reject byval in musttail to avoid silent miscompile The previous commit in this series made musttail take an early return true from isEligibleForTailCallOptimization so indirect-arg forwarding could kick in. That bypassed the pre-existing byval check as well, and byval tail-call support was reverted in 501417baa60f (#191508) pending a virtual-register-based re-implementation. The result was a silent miscompile: for musttail with a byval arg the caller allocates a local copy, writes a pointer into it, restores sp, and then tail calls with a dangling pointer. Move the byval rejection before the musttail bypass so it applies to both regular and must-tail calls. A musttail site with a byval arg will now hit reportFatalInternalError in LowerCall, matching the behavior on main before the indirect-arg fix landed, until byval tail-call support is reinstated on top of the new vreg-based forwarding path. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 29 ++++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index b5b36ed0282ae..98185477ec17f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24510,9 +24510,25 @@ bool RISCVTargetLowering::isEligibleForTailCallOptimization( if (Caller.hasFnAttribute("interrupt")) return false; - // musttail calls must always be tail-called. The remaining checks are - // for optional tail call optimization of regular (non-musttail) calls. - if (CLI.CB && CLI.CB->isMustTailCall()) + bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall(); + + // Byval parameters hand the function a pointer directly into the stack area + // we want to reuse during a tail call. Working around this *is* possible + // but less efficient and uglier in LowerCall. For musttail, there is no + // workaround today: a byval arg requires a local copy that becomes invalid + // after the tail call deallocates the caller's frame, so rejecting here + // (and triggering reportFatalInternalError in LowerCall) is safer than + // miscompiling. + for (auto &Arg : Outs) + if (Arg.Flags.isByVal()) + return false; + + // musttail bypasses the remaining checks: the checks either reject cases + // we handle specially (indirect args are forwarded via incoming pointers, + // stack-passed args reuse the matching incoming layout, sret is forwarded + // like any other pointer arg) or are optimizations not applicable to + // mandatory tail calls. + if (IsMustTail) return true; // Do not tail call opt if the stack is used to pass parameters. @@ -24544,13 +24560,6 @@ bool RISCVTargetLowering::isEligibleForTailCallOptimization( return false; } - // Byval parameters hand the function a pointer directly into the stack area - // we want to reuse during a tail call. Working around this *is* possible - // but less efficient and uglier in LowerCall. - for (auto &Arg : Outs) - if (Arg.Flags.isByVal()) - return false; - return true; } From 3c3433f31d5f305915e836d8aa56557f0c26f07a Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Tue, 21 Apr 2026 15:13:25 +0200 Subject: [PATCH 08/15] [RISCV] Address musttail review: thread CopyFromReg chain, use getUnknownStack Three small fixes from lenary's review of PR #185094: - Thread the output chain from getCopyFromReg through MemOpChains so the TokenFactor that feeds the call sequences the copy with any stores emitted for the argument. The previous code dropped the chain output, which is belt-and-suspenders safe here thanks to SSA vregs but is inconsistent with how the rest of LowerCall threads chains. - Use MachinePointerInfo::getUnknownStack(MF) for the stores into the forwarded incoming indirect pointer, instead of the default-constructed MachinePointerInfo(). The destination is a stack slot in the caller's caller's frame, so the alloca address space is the right metadata until we can track the actual frame index. - Add a FIXME explaining why LowerCall has to walk the caller's formals again: InputArg::OrigArgIndex is Argument::getArgNo() (unfiltered) but OutputArg::OrigArgIndex is an index into a filtered arg list (empty types removed). The asymmetry is target-independent and should be normalized; for now backends have to re-derive the mapping. Switch the walk to MF.getFunction() since that is the caller's IR Function directly (equivalent to CLI.CB->getFunction() but clearer). Folds the forwarded-formal and computed-value paths together since both need the CopyFromReg now; only the computed-value path still emits stores. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 52 +++++++++++++-------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 98185477ec17f..3036ae82e45f3 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24703,12 +24703,23 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex; // Resolve which formal parameter is being passed at this call - // position. Outs[].OrigArgIndex indexes the filtered arg list - // (empty types removed), but IncomingIndirectArgs is keyed by - // Argument::getArgNo() (unfiltered position). We need to: - // 1. Find the call operand at filtered position CallArgIdx - // 2. Check if it's a forwarded formal param (dyn_cast) - // 3. Resolve the unfiltered formal param index for the map lookup + // position. + // + // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered), + // but Outs[].OrigArgIndex is an index into a filtered arg list + // (empty types removed, via CallLoweringInfo in the target- + // independent layer). IncomingIndirectArgs is keyed by the + // caller's unfiltered Argument::getArgNo(), so we have to walk + // the caller's formals (same filter) to translate the index. + // This target-independent asymmetry should be normalized so + // backends do not need to re-derive the mapping. + // + // Steps: + // 1. Find the call operand at filtered position CallArgIdx. + // 2. If it is an Argument, use getArgNo() directly (same filter + // for caller formals and call operands). + // 3. Otherwise (computed value), walk the caller's formals and + // skip empty types to map the filtered index to getArgNo(). const Argument *FormalArg = nullptr; unsigned FilteredIdx = 0; for (const auto &CallArg : CLI.CB->args()) { @@ -24727,9 +24738,8 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, if (FormalArg) { FormalArgIdx = FormalArg->getArgNo(); } else { - const Function *CallerFn = CLI.CB->getFunction(); FilteredIdx = 0; - for (const auto &Arg : CallerFn->args()) { + for (const auto &Arg : MF.getFunction().args()) { if (Arg.getType()->isEmptyTy()) continue; if (FilteredIdx == CallArgIdx) { @@ -24740,19 +24750,22 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, } } - if (FormalArg) { - // Forwarded formal parameter: reuse its incoming indirect pointer. - Register VReg = RVFI->getIncomingIndirectArg(FormalArgIdx); - ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT); - } else { + Register VReg = RVFI->getIncomingIndirectArg(FormalArgIdx); + SDValue CopyOp = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT); + // Thread the CopyFromReg output chain through MemOpChains so the + // TokenFactor below sequences the copy with any stores we emit + // for this argument. + MemOpChains.push_back(CopyOp.getValue(1)); + SDValue IncomingPtr = CopyOp; + + if (!FormalArg) { // Computed value: store into the incoming indirect pointer for the // same-position formal parameter (musttail guarantees matching // prototypes, so types match). The pointer survives the tail call // since it points to the caller's caller's frame. - Register VReg = RVFI->getIncomingIndirectArg(FormalArgIdx); - SDValue IncomingPtr = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT); - MemOpChains.push_back(DAG.getStore(Chain, DL, ArgValue, IncomingPtr, - MachinePointerInfo())); + MemOpChains.push_back( + DAG.getStore(Chain, DL, ArgValue, IncomingPtr, + MachinePointerInfo::getUnknownStack(MF))); // Store any split parts at their respective offsets. unsigned ArgPartOffset = Outs[OutIdx].PartOffset; while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { @@ -24761,12 +24774,13 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, DAG.getIntPtrConstant(PartOffset, DL)); MemOpChains.push_back( - DAG.getStore(Chain, DL, PartValue, Addr, MachinePointerInfo())); + DAG.getStore(Chain, DL, PartValue, Addr, + MachinePointerInfo::getUnknownStack(MF))); ++i; ++OutIdx; } - ArgValue = IncomingPtr; } + ArgValue = IncomingPtr; // Skip any remaining split parts (for forwarded args, they are // covered by the forwarded pointer). From b2a04bb8c4edeb4b60936f72f0bb2715bae6cb50 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Tue, 21 Apr 2026 15:13:53 +0200 Subject: [PATCH 09/15] [RISCV] Add musttail tests for stack-spilled, sret, and mixed indirect args Cover three cases that the prior tests did not exercise: - caller_musttail_stack_spill: 10 x i32 args on RV32/RV64. Args 9 and 10 spill to the caller's incoming stack slots; musttail reuses the same layout for the outgoing args. This exercises the isEligibleFor TailCallOptimization stack-size bypass that kicks in for musttail (requested by lenary in review of PR #185094). - caller_musttail_sret: sret + musttail. The sret pointer is just a regular pointer arg in a0 and is forwarded unchanged. Confirms that the sret bypass in isEligibleForTailCallOptimization does not miscompile. - caller_musttail_indirect_and_spill: mix of indirect fp128 and stack- spilled i32 args. Exercises the interaction between indirect arg forwarding and the stack-spill re-use path on the same call. Also add a short note about byval + musttail: isEligibleFor TailCallOptimization rejects byval, which causes the musttail site to hit reportFatalInternalError. Tail-call support for byval was reverted in 501417baa60f (RISC-V/LoongArch) pending a vreg-based re-implementation; once that lands, musttail + byval can be added here. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- .../CodeGen/RISCV/musttail-indirect-args.ll | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index 0fc4ab6b91093..60d2b12366b20 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -836,3 +836,79 @@ then: else: ret i32 0 } + +; Non-indirect args that spill to the stack (exercises the isEligibleFor +; TailCallOptimization stack-size bypass for musttail). Both RV32 and RV64 +; use a0..a7 for the first 8 args and spill from the 9th. The spilled args +; live in the caller's incoming stack slots, which musttail can re-use +; because matching prototypes imply a matching layout. +declare void @callee_musttail_stack_spill(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) + +define void @caller_musttail_stack_spill(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %a9) nounwind { +; RV32-LABEL: caller_musttail_stack_spill: +; RV32: # %bb.0: +; RV32-NEXT: lw t0, 0(sp) +; RV32-NEXT: lw t1, 4(sp) +; RV32-NEXT: sw t0, 0(sp) +; RV32-NEXT: sw t1, 4(sp) +; RV32-NEXT: tail callee_musttail_stack_spill +; +; RV64-LABEL: caller_musttail_stack_spill: +; RV64: # %bb.0: +; RV64-NEXT: ld t0, 0(sp) +; RV64-NEXT: ld t1, 8(sp) +; RV64-NEXT: sd t0, 0(sp) +; RV64-NEXT: sd t1, 8(sp) +; RV64-NEXT: tail callee_musttail_stack_spill + musttail call void @callee_musttail_stack_spill(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %a9) + ret void +} + +; sret + musttail: the sret pointer is just a regular pointer arg in a0. +; Tail call forwards it unchanged. +%struct.Large = type { i64, i64, i64, i64 } +declare void @callee_musttail_sret(ptr sret(%struct.Large), i32) + +define void @caller_musttail_sret(ptr sret(%struct.Large) %out, i32 %x) nounwind { +; RV32-LABEL: caller_musttail_sret: +; RV32: # %bb.0: +; RV32-NEXT: tail callee_musttail_sret +; +; RV64-LABEL: caller_musttail_sret: +; RV64: # %bb.0: +; RV64-NEXT: tail callee_musttail_sret + musttail call void @callee_musttail_sret(ptr sret(%struct.Large) %out, i32 %x) + ret void +} + +; Mix of indirect (fp128) and many i32 args spilled to the stack. +declare void @callee_musttail_indirect_and_spill(fp128, i32, i32, i32, i32, i32, i32, i32, i32, i32) + +define void @caller_musttail_indirect_and_spill(fp128 %a, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8) nounwind { +; RV32-LABEL: caller_musttail_indirect_and_spill: +; RV32: # %bb.0: +; RV32-NEXT: lw t0, 0(sp) +; RV32-NEXT: lw t1, 4(sp) +; RV32-NEXT: sw t0, 0(sp) +; RV32-NEXT: sw t1, 4(sp) +; RV32-NEXT: tail callee_musttail_indirect_and_spill +; +; RV64-LABEL: caller_musttail_indirect_and_spill: +; RV64: # %bb.0: +; RV64-NEXT: ld t0, 0(sp) +; RV64-NEXT: ld t1, 8(sp) +; RV64-NEXT: ld t2, 16(sp) +; RV64-NEXT: sd t0, 0(sp) +; RV64-NEXT: sd t1, 8(sp) +; RV64-NEXT: sd t2, 16(sp) +; RV64-NEXT: tail callee_musttail_indirect_and_spill + musttail call void @callee_musttail_indirect_and_spill(fp128 %a, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8) + ret void +} + +; Note: byval + musttail is intentionally NOT tested here. isEligibleFor +; TailCallOptimization rejects byval outright, which causes the musttail +; site to hit reportFatalInternalError. Tail-call support for byval was +; reverted in 501417baa60f (RISC-V/LoongArch) pending a vreg-based +; re-implementation; once that lands, musttail + byval can be tested as +; well. From 960401fec3a71afaeff29fd42a2f9c517f8e7d45 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Tue, 28 Apr 2026 13:22:57 +0200 Subject: [PATCH 10/15] [RISCV] Sequence musttail computed-arg stores after CopyFromReg In the computed-value branch of the musttail indirect-arg path, the two stores into the incoming indirect pointer used the pre-copy Chain as their input chain. Using CopyOp.getValue(1) makes the dependency on the CopyFromReg explicit so the SDAG sequences each store after the copy that produced its destination pointer. Suggested by lenary in review of PR #185094. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 3036ae82e45f3..060e0d33cb3a8 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24762,9 +24762,11 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Computed value: store into the incoming indirect pointer for the // same-position formal parameter (musttail guarantees matching // prototypes, so types match). The pointer survives the tail call - // since it points to the caller's caller's frame. + // since it points to the caller's caller's frame. Use the + // CopyFromReg output chain so the store is sequenced after the + // copy that produced IncomingPtr. MemOpChains.push_back( - DAG.getStore(Chain, DL, ArgValue, IncomingPtr, + DAG.getStore(CopyOp.getValue(1), DL, ArgValue, IncomingPtr, MachinePointerInfo::getUnknownStack(MF))); // Store any split parts at their respective offsets. unsigned ArgPartOffset = Outs[OutIdx].PartOffset; @@ -24774,7 +24776,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, DAG.getIntPtrConstant(PartOffset, DL)); MemOpChains.push_back( - DAG.getStore(Chain, DL, PartValue, Addr, + DAG.getStore(CopyOp.getValue(1), DL, PartValue, Addr, MachinePointerInfo::getUnknownStack(MF))); ++i; ++OutIdx; From d9c9be76af9203037ed13e66833356e66088095e Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Tue, 28 Apr 2026 13:23:02 +0200 Subject: [PATCH 11/15] [RISCV][NFC] Regenerate CHECK lines for musttail computed-args test Side effect of sequencing musttail computed-arg stores after the CopyFromReg in the previous commit: the scheduler emits the stores into the two incoming pointers in the opposite order, and the register allocator picks different callee-saved registers in caller_musttail_both_computed. The generated asm is equivalent. Regenerated with utils/update_llc_test_checks.py. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- .../CodeGen/RISCV/musttail-indirect-args.ll | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index 60d2b12366b20..5982bd6cee4f0 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -599,10 +599,10 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { ; RV32-NEXT: sw s11, 108(sp) # 4-byte Folded Spill ; RV32-NEXT: mv s0, a1 ; RV32-NEXT: mv s1, a0 -; RV32-NEXT: lw s2, 0(a1) -; RV32-NEXT: lw s3, 4(a1) -; RV32-NEXT: lw s4, 8(a1) -; RV32-NEXT: lw s5, 12(a1) +; RV32-NEXT: lw s5, 0(a1) +; RV32-NEXT: lw s2, 4(a1) +; RV32-NEXT: lw s3, 8(a1) +; RV32-NEXT: lw s4, 12(a1) ; RV32-NEXT: lw s6, 0(a0) ; RV32-NEXT: lw s7, 4(a0) ; RV32-NEXT: lw s8, 8(a0) @@ -614,17 +614,17 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { ; RV32-NEXT: addi a0, sp, 88 ; RV32-NEXT: addi a1, sp, 72 ; RV32-NEXT: addi a2, sp, 56 -; RV32-NEXT: sw s2, 56(sp) -; RV32-NEXT: sw s3, 60(sp) -; RV32-NEXT: sw s4, 64(sp) -; RV32-NEXT: sw s5, 68(sp) +; RV32-NEXT: sw s5, 56(sp) +; RV32-NEXT: sw s2, 60(sp) +; RV32-NEXT: sw s3, 64(sp) +; RV32-NEXT: sw s4, 68(sp) ; RV32-NEXT: call __addtf3 ; RV32-NEXT: lw s10, 88(sp) ; RV32-NEXT: lw s11, 92(sp) ; RV32-NEXT: lw a0, 96(sp) -; RV32-NEXT: sw a0, 4(sp) # 4-byte Folded Spill -; RV32-NEXT: lw a0, 100(sp) ; RV32-NEXT: sw a0, 0(sp) # 4-byte Folded Spill +; RV32-NEXT: lw a0, 100(sp) +; RV32-NEXT: sw a0, 4(sp) # 4-byte Folded Spill ; RV32-NEXT: sw s6, 24(sp) ; RV32-NEXT: sw s7, 28(sp) ; RV32-NEXT: sw s8, 32(sp) @@ -632,25 +632,25 @@ define i32 @caller_musttail_both_computed(fp128 %a, fp128 %b) nounwind { ; RV32-NEXT: addi a0, sp, 40 ; RV32-NEXT: addi a1, sp, 24 ; RV32-NEXT: addi a2, sp, 8 -; RV32-NEXT: sw s2, 8(sp) -; RV32-NEXT: sw s3, 12(sp) -; RV32-NEXT: sw s4, 16(sp) -; RV32-NEXT: sw s5, 20(sp) +; RV32-NEXT: sw s5, 8(sp) +; RV32-NEXT: sw s2, 12(sp) +; RV32-NEXT: sw s3, 16(sp) +; RV32-NEXT: sw s4, 20(sp) ; RV32-NEXT: call __subtf3 ; RV32-NEXT: lw a0, 40(sp) ; RV32-NEXT: lw a1, 44(sp) ; RV32-NEXT: lw a2, 48(sp) ; RV32-NEXT: lw a3, 52(sp) +; RV32-NEXT: sw s10, 0(s1) +; RV32-NEXT: sw s11, 4(s1) +; RV32-NEXT: lw a4, 0(sp) # 4-byte Folded Reload +; RV32-NEXT: sw a4, 8(s1) +; RV32-NEXT: lw a4, 4(sp) # 4-byte Folded Reload +; RV32-NEXT: sw a4, 12(s1) ; RV32-NEXT: sw a0, 0(s0) ; RV32-NEXT: sw a1, 4(s0) ; RV32-NEXT: sw a2, 8(s0) ; RV32-NEXT: sw a3, 12(s0) -; RV32-NEXT: sw s10, 0(s1) -; RV32-NEXT: sw s11, 4(s1) -; RV32-NEXT: lw a0, 4(sp) # 4-byte Folded Reload -; RV32-NEXT: sw a0, 8(s1) -; RV32-NEXT: lw a0, 0(sp) # 4-byte Folded Reload -; RV32-NEXT: sw a0, 12(s1) ; RV32-NEXT: mv a0, s1 ; RV32-NEXT: mv a1, s0 ; RV32-NEXT: lw ra, 156(sp) # 4-byte Folded Reload From f888fe489f6d90e75321d0a77983ed08671be649 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 6 May 2026 18:47:03 +0200 Subject: [PATCH 12/15] [RISCV] Handle scalable vectors in musttail computed-arg parts loop The non-musttail spill path multiplies the part offset by VSCALE for scalable vector parts. The musttail computed-value path was missing that, so scalable vector args computed in the caller and forwarded via musttail would land at the wrong offset relative to the incoming indirect pointer. Mirror the non-musttail spelling: build the offset with DAG.getIntPtrConstant, then wrap with ISD::VSCALE when the part type is a scalable vector. Reported by topperc on PR #185094. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 060e0d33cb3a8..7c9c10553feeb 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24768,13 +24768,19 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, MemOpChains.push_back( DAG.getStore(CopyOp.getValue(1), DL, ArgValue, IncomingPtr, MachinePointerInfo::getUnknownStack(MF))); - // Store any split parts at their respective offsets. + // Store any split parts at their respective offsets. Scalable + // vectors need their part offsets multiplied by VSCALE, matching + // the non-musttail spill path below. unsigned ArgPartOffset = Outs[OutIdx].PartOffset; while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) { SDValue PartValue = OutVals[OutIdx + 1]; unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset; - SDValue Addr = DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, - DAG.getIntPtrConstant(PartOffset, DL)); + SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL); + EVT PartVT = PartValue.getValueType(); + if (PartVT.isScalableVector()) + Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset); + SDValue Addr = + DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, Offset); MemOpChains.push_back( DAG.getStore(CopyOp.getValue(1), DL, PartValue, Addr, MachinePointerInfo::getUnknownStack(MF))); From 718f9cf635f3be7ad3aa274cf9cf62ba817fe79a Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 6 May 2026 18:47:18 +0200 Subject: [PATCH 13/15] [RISCV][NFC] Clarify why musttail computed-arg stores chain off CopyFromReg topperc pointed out on PR #185094 that the data-flow edge through IncomingPtr already prevents the store from being scheduled before the CopyFromReg, so the chain edge is not strictly required for ordering. The chain edge is there to make the side-effect dependency explicit on the chain side as well, matching the convention for memory ops chaining off their producers (lenary's earlier review). Rewrite the comment to describe this accurately. Code unchanged. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 7c9c10553feeb..df6f11da4cc0c 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -24762,9 +24762,13 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, // Computed value: store into the incoming indirect pointer for the // same-position formal parameter (musttail guarantees matching // prototypes, so types match). The pointer survives the tail call - // since it points to the caller's caller's frame. Use the - // CopyFromReg output chain so the store is sequenced after the - // copy that produced IncomingPtr. + // since it points to the caller's caller's frame. + // + // The data-flow edge through IncomingPtr already prevents the + // store from being scheduled before the CopyFromReg. Threading + // CopyOp.getValue(1) (the copy's output chain) into the store + // makes that ordering explicit on the chain edge as well, which + // is the convention for memory ops chaining off their producers. MemOpChains.push_back( DAG.getStore(CopyOp.getValue(1), DL, ArgValue, IncomingPtr, MachinePointerInfo::getUnknownStack(MF))); From 63260eadfb6128160acf1e5d8bf15f4595d6b666 Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 6 May 2026 18:47:25 +0200 Subject: [PATCH 14/15] [RISCV][NFC] Keep isEligibleForTailCallOptimization on one line in test comments topperc noted on PR #185094 that the function name was wrapped across two comment lines, which makes it hard to grep. Keep the name on one line; the lines exceed 80 columns by a few chars but that matches the file-wide style for long identifiers. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- .../CodeGen/RISCV/musttail-indirect-args.ll | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll index 5982bd6cee4f0..50289eca58e28 100644 --- a/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll +++ b/llvm/test/CodeGen/RISCV/musttail-indirect-args.ll @@ -837,11 +837,11 @@ else: ret i32 0 } -; Non-indirect args that spill to the stack (exercises the isEligibleFor -; TailCallOptimization stack-size bypass for musttail). Both RV32 and RV64 -; use a0..a7 for the first 8 args and spill from the 9th. The spilled args -; live in the caller's incoming stack slots, which musttail can re-use -; because matching prototypes imply a matching layout. +; Non-indirect args that spill to the stack (exercises the +; isEligibleForTailCallOptimization stack-size bypass for musttail). Both +; RV32 and RV64 use a0..a7 for the first 8 args and spill from the 9th. The +; spilled args live in the caller's incoming stack slots, which musttail can +; re-use because matching prototypes imply a matching layout. declare void @callee_musttail_stack_spill(i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) define void @caller_musttail_stack_spill(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6, i32 %a7, i32 %a8, i32 %a9) nounwind { @@ -906,9 +906,9 @@ define void @caller_musttail_indirect_and_spill(fp128 %a, i32 %i0, i32 %i1, i32 ret void } -; Note: byval + musttail is intentionally NOT tested here. isEligibleFor -; TailCallOptimization rejects byval outright, which causes the musttail -; site to hit reportFatalInternalError. Tail-call support for byval was -; reverted in 501417baa60f (RISC-V/LoongArch) pending a vreg-based +; Note: byval + musttail is intentionally NOT tested here. +; isEligibleForTailCallOptimization rejects byval outright, which causes the +; musttail site to hit reportFatalInternalError. Tail-call support for byval +; was reverted in 501417baa60f (RISC-V/LoongArch) pending a vreg-based ; re-implementation; once that lands, musttail + byval can be tested as ; well. From b88aa6f4a97934c4d545551e51a383d59633a68b Mon Sep 17 00:00:00 2001 From: Xavier Roche Date: Wed, 6 May 2026 21:26:34 +0200 Subject: [PATCH 15/15] [RISCV] Add musttail tests for scalable vector indirect args Add a sibling to musttail-indirect-args.ll under rvv/ for the scalable vector path that topperc demonstrated is exercised by upstream tests (calling-conv.ll's caller_scalable_vector_split_indirect on PR #185094). is too large for v8-v23, so RISCVCallingConv classifies it as CCValAssign::Indirect and the type is split into multiple parts. Three test functions cover the three branches of the musttail indirect-arg path: - caller_musttail_scalable_forwarded: both args are caller formals, zero-copy pointer forwarding, no per-part offset computation. - caller_musttail_scalable_computed: both args are arithmetic on the formals; dyn_cast fails and we store into the incoming pointer with VSCALE-scaled per-part offsets. The asm shows the second store landing at IncomingPtr + VSCALE * 8 (vlenb << 3), confirming the VSCALE multiplication added in f888fe489f6d. - caller_musttail_scalable_mixed: first arg forwarded, second computed. Assisted-by: Claude (Anthropic) Co-Authored-By: Claude Opus 4.6 --- .../RISCV/rvv/musttail-indirect-args.ll | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvv/musttail-indirect-args.ll diff --git a/llvm/test/CodeGen/RISCV/rvv/musttail-indirect-args.ll b/llvm/test/CodeGen/RISCV/rvv/musttail-indirect-args.ll new file mode 100644 index 0000000000000..58463ec1b3274 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/musttail-indirect-args.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=riscv32 -mattr=+m,+v %s -o - | FileCheck %s --check-prefixes=CHECK,RV32 +; RUN: llc -mtriple=riscv64 -mattr=+m,+v %s -o - | FileCheck %s --check-prefixes=CHECK,RV64 + +; Companion to musttail-indirect-args.ll for scalable vector args. +; is too large to fit in v8-v23, so it is split into +; multiple parts and the callee receives a pointer to the parts via +; CCValAssign::Indirect. The musttail computed-arg path must scale per-part +; offsets by VSCALE -- mirrors the non-musttail spill path validated by +; calling-conv.ll's caller_scalable_vector_split_indirect. + +declare @callee_musttail_scalable( %x, %y) + +; Forwarded: both args are caller formals passed straight through. The +; incoming indirect pointers are forwarded zero-copy; no per-part offset +; computation happens in the caller, so VSCALE does not enter here. +define @caller_musttail_scalable_forwarded( %x, %y) nounwind { +; CHECK-LABEL: caller_musttail_scalable_forwarded: +; CHECK: # %bb.0: +; CHECK-NEXT: tail callee_musttail_scalable + %r = musttail call @callee_musttail_scalable( %x, %y) + ret %r +} + +; Computed: both args are arithmetic on the formals, so dyn_cast +; fails and we go through the store-into-incoming-pointer path. The +; second part's store must use VSCALE * PartOffset relative to +; IncomingPtr, otherwise the part lands at a fixed byte offset that +; differs from what the callee expects (vlenb * 8 above). +define @caller_musttail_scalable_computed( %x, %y) nounwind { +; CHECK-LABEL: caller_musttail_scalable_computed: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: vl8re32.v v24, (a0) +; CHECK-NEXT: add a1, a0, a1 +; CHECK-NEXT: vl8re32.v v0, (a1) +; CHECK-NEXT: vsetvli a2, zero, e32, m8, ta, ma +; CHECK-NEXT: vadd.vv v8, v8, v24 +; CHECK-NEXT: vs8r.v v8, (a0) +; CHECK-NEXT: vadd.vv v16, v16, v0 +; CHECK-NEXT: vs8r.v v16, (a1) +; CHECK-NEXT: tail callee_musttail_scalable + %s = add %x, %y + %r = musttail call @callee_musttail_scalable( %s, %s) + ret %r +} + +; Mixed: first arg forwarded, second arg computed. Exercises both branches +; of the formal-arg-vs-computed path on the same call. +define @caller_musttail_scalable_mixed( %x, %y) nounwind { +; CHECK-LABEL: caller_musttail_scalable_mixed: +; CHECK: # %bb.0: +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: vl8re32.v v24, (a0) +; CHECK-NEXT: add a1, a0, a1 +; CHECK-NEXT: vl8re32.v v0, (a1) +; CHECK-NEXT: vsetvli a2, zero, e32, m8, ta, ma +; CHECK-NEXT: vadd.vv v24, v8, v24 +; CHECK-NEXT: vs8r.v v24, (a0) +; CHECK-NEXT: vadd.vv v24, v16, v0 +; CHECK-NEXT: vs8r.v v24, (a1) +; CHECK-NEXT: tail callee_musttail_scalable + %s = add %x, %y + %r = musttail call @callee_musttail_scalable( %x, %s) + ret %r +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; RV32: {{.*}} +; RV64: {{.*}}