[Clang] Forward incoming Indirect parameters across musttail calls

clang/lib/CodeGen/CGCall.cpp

@@ -5139,6 +5139,30 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     return;
   }
 
+  // For musttail calls in C++, skip the agg.tmp copy when the source is a
+  // trivially-copyable parameter of the current function. The copy would
+  // live in our frame, which the tail call deallocates before the callee
+  // dereferences it. The companion fix in EmitCall (getForwardableIncoming
+  // MustTailArg) forwards the incoming Indirect Argument; for that to fire
+  // here we must hand it the original parameter LValue, not a fresh temp.
+  if (HasAggregateEvalKind && MustTailCall && type->isRecordType() &&
+      type.isTriviallyCopyableType(getContext())) {
+    if (const auto *CCE = dyn_cast<CXXConstructExpr>(E)) {
+      if (CCE->getConstructor()->isCopyOrMoveConstructor() &&
+          CCE->getConstructor()->isTrivial() && CCE->getNumArgs() == 1) {
+        const Expr *Source = CCE->getArg(0)->IgnoreParenImpCasts();
+        if (const auto *DRE = dyn_cast<DeclRefExpr>(Source))
+          if (const auto *PVD = dyn_cast<ParmVarDecl>(DRE->getDecl()))
+            if (PVD->getDeclContext() == dyn_cast<DeclContext>(CurCodeDecl)) {
+              LValue L = EmitLValue(DRE);
+              assert(L.isSimple());
+              args.addUncopiedAggregate(L, type);
+              return;
+            }
+      }
+    }
+  }
+
   args.add(EmitAnyExprToTemp(E), type);
 }
 
@@ -5448,6 +5472,45 @@ static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
   return MaxVectorWidth;
 }
 
+/// Returns the incoming llvm::Argument of the current function if this
+/// musttail call argument forwards an incoming Indirect parameter with a
+/// matching ABI shape; nullptr to fall through to the byval-temp path.
+/// Argument-side analog of a96c14eeb8fc (SRet musttail forwarding).
+static llvm::Argument *getForwardableIncomingMustTailArg(
+    CodeGenFunction &CGF, const CallArg &CallArgument,
+    const ABIArgInfo &CallSlotInfo,
+    llvm::SmallPtrSetImpl<llvm::Argument *> &AlreadyForwarded) {
+  Address SrcAddr = Address::invalid();
+  if (CallArgument.hasLValue())
+    SrcAddr = CallArgument.getKnownLValue().getAddress();
+  else if (CallArgument.getKnownRValue().isAggregate())
+    SrcAddr = CallArgument.getKnownRValue().getAggregateAddress();
+  else
+    return nullptr;
+  llvm::Value *SrcPtr = SrcAddr.emitRawPointer(CGF);
+
+  // Peek through one AddrSpaceCastInst (NVPTX / AMDGPU / SPIR wrap incoming
+  // Indirect params via EmitParmDecl). Do not unwrap loads: a load through
+  // a local alloca means the source is a local.
+  if (auto *ASC = llvm::dyn_cast<llvm::AddrSpaceCastInst>(SrcPtr))
+    SrcPtr = ASC->getOperand(0);
+
+  auto *IncomingArg = llvm::dyn_cast<llvm::Argument>(SrcPtr);
+  if (!IncomingArg || IncomingArg->getParent() != CGF.CurFn)
+    return nullptr;
+
+  // Verifier V7 requires matching ABI attributes across musttail.
+  if (IncomingArg->hasByValAttr() != CallSlotInfo.getIndirectByVal())
+    return nullptr;
+
+  // Do not forward the same noalias Argument to two slots in one call.
+  if (IncomingArg->hasNoAliasAttr() &&
+      !AlreadyForwarded.insert(IncomingArg).second)
+    return nullptr;
+
+  return IncomingArg;
+}
+
 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  const CGCallee &Callee,
                                  ReturnValueSlot ReturnValue,
@@ -5571,6 +5634,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // markers that need to be ended right after the call.
   SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
 
+  // Tracks incoming Arguments already forwarded by a musttail Indirect arg,
+  // for noalias deduplication in getForwardableIncomingMustTailArg.
+  llvm::SmallPtrSet<llvm::Argument *, 4> ForwardedMustTailArgs;
+
   // Translate all of the arguments as necessary to match the IR lowering.
   assert(CallInfo.arg_size() == CallArgs.size() &&
          "Mismatch between function signature & arguments.");
@@ -5643,6 +5710,23 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
+
+      // For musttail, forward an incoming Indirect parameter directly. A
+      // local alloca would dangle after the tail call. Mirrors the SRet
+      // forwarding above (a96c14eeb8fc). Limited to Indirect (not
+      // IndirectAliased) so the byval-ness check in the helper does not
+      // assert on getIndirectByVal().
+      if (IsMustTail && ArgInfo.isIndirect()) {
+        if (llvm::Argument *FwdArg = getForwardableIncomingMustTailArg(
+                *this, *I, ArgInfo, ForwardedMustTailArgs)) {
+          llvm::Value *Val = FwdArg;
+          if (ArgHasMaybeUndefAttr)
+            Val = Builder.CreateFreeze(Val);
+          IRCallArgs[FirstIRArg] = Val;
+          break;
+        }
+      }
+
       if (I->isAggregate()) {
         // We want to avoid creating an unnecessary temporary+copy here;
         // however, we need one in three cases:

clang/test/CodeGen/musttail-indirect-arg.c

@@ -0,0 +1,118 @@
+// Test that Clang forwards incoming Indirect parameters across musttail calls
+// instead of creating a byval-temp alloca that would dangle after the tail call
+// deallocates the caller's frame.
+//
+// Companion to musttail-sret.cpp (commit a96c14eeb8fc): same idea, applied to
+// incoming arguments rather than the sret return slot.
+
+// RUN: %clang_cc1 -triple=riscv64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=loongarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=s390x-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+
+// A struct large enough to land on the indirect-arg path on RV64 (>2*XLEN=16
+// bytes), AArch64 (>16 bytes), LoongArch64, SystemZ.
+struct Big {
+  unsigned long long a, b, c, d;
+};
+
+// Plain forward: caller(B) musttails callee(B). The fix should emit no
+// alloca for the forwarded arg; the call should forward the incoming
+// parameter %a.
+struct Big C1(struct Big a);
+struct Big P1(struct Big a) {
+  __attribute__((musttail)) return C1(a);
+}
+// COMMON-LABEL: define {{.*}} @P1(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON-NOT: = alloca [32 x i8]
+// COMMON: musttail call {{.*}} @C1({{.*}} %a)
+
+// Two indirect args, same forwarding: each forwards its own incoming param.
+struct Big C2(struct Big a, struct Big b);
+struct Big P2(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C2(a, b);
+}
+// COMMON-LABEL: define {{.*}} @P2(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C2({{.*}} %a, {{.*}} %b)
+
+// Swapped args: caller(a, b) musttails callee(b, a). Each forwarded slot
+// must resolve to the correct incoming Argument, not by position.
+struct Big C3(struct Big x, struct Big y);
+struct Big P3(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C3(b, a);
+}
+// COMMON-LABEL: define {{.*}} @P3(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C3({{.*}} %b, {{.*}} %a)
+
+// Mixed direct + indirect: only the indirect arg is affected by the fix.
+struct Big C4(int n, struct Big a);
+struct Big P4(int n, struct Big a) {
+  __attribute__((musttail)) return C4(n, a);
+}
+// COMMON-LABEL: define {{.*}} @P4(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C4({{.*}} %n, {{.*}} %a)
+
+// Caller modifies the parameter before the musttail. Clang lowers the
+// write through the incoming pointer, and the fix forwards the same
+// pointer to the callee. No byval-temp.
+struct Big C5(struct Big a);
+struct Big P5(struct Big a) {
+  a.a += 1;
+  __attribute__((musttail)) return C5(a);
+}
+// COMMON-LABEL: define {{.*}} @P5(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C5({{.*}} %a)
+
+// musttail behind a branch: the forwarded pointer must remain live across
+// the basic block transition. Tests that the helper does not assume the
+// musttail is in the entry block.
+struct Big C6(struct Big a, int cond);
+struct Big P6(struct Big a, int cond) {
+  if (cond)
+    __attribute__((musttail)) return C6(a, cond);
+  return a;
+}
+// COMMON-LABEL: define {{.*}} @P6(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C6({{.*}} %a,
+
+// Same Argument forwarded to two slots: the helper engages for both. The
+// noalias deduplication, if it ever fired, would force the second slot
+// back to a byval-temp; but incoming Indirect params under the Linux C
+// ABI are not noalias, so both slots forward %a directly. This pins the
+// behavior so a future change introducing noalias on Indirect params
+// would surface here. (musttail requires matching prototypes, so caller
+// and callee both take two Big args.)
+struct Big C7(struct Big x, struct Big y);
+struct Big P7(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C7(a, a);
+}
+// COMMON-LABEL: define {{.*}} @P7(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C7({{.*}} %a, {{.*}} %a)
+
+// Negative: local source. Caller takes Big a, but musttails with a LOCAL
+// Big initialized in caller's frame. The byval-temp must remain because the
+// source lives in caller's frame and would dangle if forwarded.
+struct Big C8(struct Big a);
+struct Big P8(struct Big a) {
+  struct Big local = {1, 2, 3, 4};
+  __attribute__((musttail)) return C8(local);
+}
+// COMMON-LABEL: define {{.*}} @P8(
+// COMMON: = alloca
+// COMMON: musttail call {{.*}} @C8(
+
+// Non-musttail tail call: the fix must NOT engage. Existing path emits
+// the byval-temp as before, no musttail in the IR.
+struct Big C9(struct Big a);
+struct Big P9(struct Big a) {
+  return C9(a);
+}
+// COMMON-LABEL: define {{.*}} @P9(
+// COMMON-NOT: musttail

clang/test/CodeGen/musttail-indirect-arg.cpp

@@ -0,0 +1,119 @@
+// Test that Clang forwards incoming Indirect parameters across musttail calls
+// for C++ struct-by-value arguments with trivially-copyable types. Companion to
+// musttail-indirect-arg.c (the C side of the same fix) and musttail-sret.cpp
+// (the SRet precedent in a96c14eeb8fc).
+//
+// C++ goes through a different EmitCallArg path than C: the call argument is a
+// CXXConstructExpr invoking the implicit copy constructor, which would
+// otherwise materialize an agg.tmp before EmitCall. For musttail with a
+// trivially-copyable parameter forwarded directly, the copy is elided so the
+// helper in EmitCall can forward the incoming llvm::Argument.
+
+// RUN: %clang_cc1 -triple=riscv64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=loongarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=s390x-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+
+// A trivially-copyable struct large enough to land on the indirect-arg path
+// on RV64, AArch64, LoongArch64, SystemZ.
+struct Big {
+  unsigned long long a, b, c, d;
+};
+
+// Plain forward: caller(B) musttails callee(B). No agg.tmp copy, no
+// byval-temp; the incoming parameter %a is forwarded directly.
+struct Big C1(struct Big a);
+struct Big P1(struct Big a) {
+  [[clang::musttail]] return C1(a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P13Big(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C13Big({{.*}} %a)
+
+// Two args, same forwarding.
+struct Big C2(struct Big a, struct Big b);
+struct Big P2(struct Big a, struct Big b) {
+  [[clang::musttail]] return C2(a, b);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P23BigS_(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C23BigS_({{.*}} %a, {{.*}} %b)
+
+// Swapped args.
+struct Big C3(struct Big x, struct Big y);
+struct Big P3(struct Big a, struct Big b) {
+  [[clang::musttail]] return C3(b, a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P33BigS_(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C33BigS_({{.*}} %b, {{.*}} %a)
+
+// Non-trivial copy constructor: the trivial-copy elision must NOT engage.
+// Existing path materializes the agg.tmp (the user-defined copy ctor has
+// observable behavior).
+struct NonTrivial {
+  unsigned long long parts[4];
+  NonTrivial(const NonTrivial &);
+};
+NonTrivial C4(NonTrivial a);
+NonTrivial P4(NonTrivial a) {
+  [[clang::musttail]] return C4(a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P410NonTrivial(
+// The user-defined copy ctor IS called (the agg.tmp pattern still happens):
+// COMMON: call {{.*}} @_ZN10NonTrivialC1ERKS_
+
+// Caller modifies the parameter before the musttail. The trivial-copy
+// elision still engages because the source LValue is still the parameter;
+// any mutation flowed through the incoming pointer is observed by the
+// forwarded call.
+struct Big C5(struct Big a);
+struct Big P5(struct Big a) {
+  a.a += 1;
+  [[clang::musttail]] return C5(a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P53Big(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C53Big({{.*}} %a)
+
+// musttail behind a branch: trivial-copy elision must work across BBs.
+struct Big C6(struct Big a, int cond);
+struct Big P6(struct Big a, int cond) {
+  if (cond)
+    [[clang::musttail]] return C6(a, cond);
+  return a;
+}
+// COMMON-LABEL: define {{.*}} @_Z2P63Bigi(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C63Bigi({{.*}} %a,
+
+// Same Argument forwarded to two slots: both engage. Incoming Indirect
+// params are not noalias under the Linux C++ ABI so the dedup in the
+// helper does not fire and both slots forward %a directly.
+struct Big C7(struct Big x, struct Big y);
+struct Big P7(struct Big a, struct Big b) {
+  [[clang::musttail]] return C7(a, a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P73BigS_(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @_Z2C73BigS_({{.*}} %a, {{.*}} %a)
+
+// Negative: source is a local, not a parameter. The trivial-copy elision
+// must NOT engage; the byval-temp pattern remains.
+struct Big C8(struct Big a);
+struct Big P8(struct Big a) {
+  struct Big local = {1, 2, 3, 4};
+  [[clang::musttail]] return C8(local);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P83Big(
+// COMMON: = alloca
+// COMMON: musttail call {{.*}} @_Z2C83Big(
+
+// Non-musttail tail call: trivial-copy elision must NOT engage; the regular
+// agg.tmp copy is still emitted.
+struct Big C9(struct Big a);
+struct Big P9(struct Big a) {
+  return C9(a);
+}
+// COMMON-LABEL: define {{.*}} @_Z2P93Big(
+// COMMON-NOT: musttail