[Clang] Forward incoming Indirect parameters across musttail calls

clang/include/clang/Basic/DiagnosticCommonKinds.td

@@ -390,6 +390,9 @@ def err_aix_musttail_unsupported: Error<
   "'musttail' attribute is not supported on AIX">;
 def err_musttail_noexcept_mismatch: Error<
   "'musttail' in a noexcept function requires a noexcept callee">;
+def err_musttail_unsupported_indirect_arg: Error<
+  "'musttail' call requires passing an argument by reference, but the source "
+  "does not have an addressable storage and would alias the caller's frame">;
 
 // Source manager
 def err_cannot_open_file : Error<"cannot open file '%0': %1">, DefaultFatal;

clang/lib/CodeGen/CGCall.cpp

@@ -5255,6 +5255,33 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     }
   }
 
+  // Under musttail, hand a trivially-copyable record source's LValue to
+  // EmitCall rather than materializing an agg.tmp. EmitCall's Indirect path
+  // routes it via the matching incoming parameter, which survives the tail
+  // call. Limited to params and locals: globals and captures don't have the
+  // dangle issue and the existing path may be more efficient for them.
+  if (HasAggregateEvalKind && MustTailCall && type->isRecordType() &&
+      type.isTriviallyCopyableType(getContext())) {
+    if (const auto *CCE = dyn_cast<CXXConstructExpr>(E)) {
+      if (CCE->getConstructor()->isCopyOrMoveConstructor() &&
+          CCE->getConstructor()->isTrivial() && CCE->getNumArgs() == 1) {
+        const Expr *Source = CCE->getArg(0)->IgnoreParenImpCasts();
+        if (const auto *DRE = dyn_cast<DeclRefExpr>(Source)) {
+          if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
+            if (VD->hasLocalStorage() ||
+                (isa<ParmVarDecl>(VD) &&
+                 VD->getDeclContext() == dyn_cast<DeclContext>(CurCodeDecl))) {
+              LValue L = EmitLValue(DRE);
+              assert(L.isSimple());
+              args.addUncopiedAggregate(L, type);
+              return;
+            }
+          }
+        }
+      }
+    }
+  }
+
   args.add(EmitAnyExprToTemp(E), type);
 }
 
@@ -5564,6 +5591,17 @@ static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
   return MaxVectorWidth;
 }
 
+/// Peel one AddrSpaceCastInst from \p SrcPtr. EmitParmDecl wraps incoming
+/// Indirect params via address-space cast on NVPTX/AMDGPU/SPIR, so peeling
+/// exposes the underlying llvm::Argument when the source IS a forwarded
+/// incoming parameter. Loads are NOT unwrapped: a load through a local
+/// alloca means the source is a local.
+static llvm::Value *peelAddrSpaceCast(llvm::Value *SrcPtr) {
+  if (auto *ASC = llvm::dyn_cast<llvm::AddrSpaceCastInst>(SrcPtr))
+    return ASC->getOperand(0);
+  return SrcPtr;
+}
+
 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  const CGCallee &Callee,
                                  ReturnValueSlot ReturnValue,
@@ -5687,6 +5725,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // markers that need to be ended right after the call.
   SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
 
+  // Deferred Phase-2 writes for musttail Indirect args. Splitting reads
+  // (in the per-arg loop) from writes (after the loop) lets permutations
+  // like C(b, a) land correctly: all sources are captured into scratches
+  // before any incoming-param destination is overwritten.
+  struct MustTailIndirectCopy {
+    LValue Scratch;
+    LValue Dst;
+    QualType Ty;
+  };
+  llvm::SmallVector<MustTailIndirectCopy, 4> MustTailIndirectCopies;
+
   // Translate all of the arguments as necessary to match the IR lowering.
   assert(CallInfo.arg_size() == CallArgs.size() &&
          "Mismatch between function signature & arguments.");
@@ -5759,6 +5808,53 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
+
+      // Musttail Indirect: route via the matching incoming parameter.
+      // Prototype-match (Verifier V5/V6/V7) makes CurFn->arg_begin()+
+      // FirstIRArg a distinct destination for this slot that lives in the
+      // caller's caller's frame and survives the tail call. To handle
+      // permutations safely, the source value is captured into a scratch
+      // alloca here (Phase 1); the write to the incoming-param destination
+      // is deferred until after all sources have been read (Phase 2 below).
+      // IndirectAliased uses the existing fallback (different source-AS).
+      if (IsMustTail && ArgInfo.isIndirect()) {
+        llvm::Argument *IncomingArg = CurFn->arg_begin() + FirstIRArg;
+        llvm::Value *Dst = IncomingArg;
+        Address SrcAddr = Address::invalid();
+        if (I->hasLValue())
+          SrcAddr = I->getKnownLValue().getAddress();
+        else if (I->getKnownRValue().isAggregate())
+          SrcAddr = I->getKnownRValue().getAggregateAddress();
+        if (SrcAddr.isValid()) {
+          llvm::Value *Src = peelAddrSpaceCast(SrcAddr.emitRawPointer(*this));
+          if (Src != Dst) {
+            CharUnits Align = ArgInfo.getIndirectAlign();
+            QualType Ty = I->Ty;
+            llvm::Type *ElemTy = ConvertTypeForMem(Ty);
+            RawAddress Scratch =
+                CreateMemTempWithoutCast(Ty, Align, "musttail.copy");
+            LValue ScratchLV = MakeAddrLValue(Scratch, Ty);
+            LValue SrcLV = MakeAddrLValue(SrcAddr, Ty);
+            EmitAggregateCopy(ScratchLV, SrcLV, Ty,
+                              AggValueSlot::DoesNotOverlap);
+            LValue DstLV = MakeAddrLValue(Address(Dst, ElemTy, Align), Ty);
+            MustTailIndirectCopies.push_back({ScratchLV, DstLV, Ty});
+          }
+          llvm::Value *Val = Dst;
+          if (ArgHasMaybeUndefAttr)
+            Val = Builder.CreateFreeze(Val);
+          IRCallArgs[FirstIRArg] = Val;
+          break;
+        }
+        // No addressable source for this Indirect arg (rare; e.g. a scalar
+        // RValue the ABI classifies as Indirect). The fall-through below
+        // would create a current-frame byval-temp that dangles past the
+        // tail-call teardown. Refuse cleanly; codegen continues producing
+        // IR but the error prevents it from reaching the backend.
+        CGM.getDiags().Report(MustTailCall->getBeginLoc(),
+                              diag::err_musttail_unsupported_indirect_arg);
+      }
+
       if (I->isAggregate()) {
         // We want to avoid creating an unnecessary temporary+copy here;
         // however, we need one in three cases:
@@ -6085,6 +6181,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     }
   }
 
+  // Phase 2 of the musttail Indirect-arg copy: flush each captured scratch
+  // into its incoming-param destination. Phase 1 has read every source, so
+  // permutations like C(b, a) land the right value in each slot.
+  for (const auto &Copy : MustTailIndirectCopies)
+    EmitAggregateCopy(Copy.Dst, Copy.Scratch, Copy.Ty,
+                      AggValueSlot::DoesNotOverlap);
+
   const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
   llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
 

clang/test/CodeGen/musttail-indirect-arg.c

@@ -0,0 +1,160 @@
+// RUN: %clang_cc1 -triple=riscv64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=loongarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+// RUN: %clang_cc1 -triple=s390x-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON
+
+// Musttail calls with struct-by-value args must route each value through the
+// matching incoming Indirect parameter's storage, not a local alloca that
+// dangles past the tail-call frame teardown. For each Indirect slot i: if
+// the source IS the i-th incoming pointer, pass it; otherwise memcpy the
+// source into the i-th incoming pointer and pass that. Each call slot ends
+// up with a distinct pointer in the caller's caller's frame.
+
+// Plain Indirect-ABI struct on the targets above.
+struct Big {
+  unsigned long long a, b, c, d;
+};
+
+// P1: simple forward.
+struct Big C1(struct Big a);
+struct Big P1(struct Big a) {
+  __attribute__((musttail)) return C1(a);
+}
+// COMMON-LABEL: define {{.*}} @P1(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C1({{.*}}, ptr {{[^,]*}} %a)
+
+// P2: two distinct incoming sources.
+struct Big C2(struct Big a, struct Big b);
+struct Big P2(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C2(a, b);
+}
+// COMMON-LABEL: define {{.*}} @P2(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON-NOT: llvm.memcpy
+// COMMON: musttail call {{.*}} @C2({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b)
+
+// P3: swap. Pin the data flow: %a is captured before %b overwrites it, and
+// the saved %a lands in %b. An in-place memmove(%a,%b);memmove(%b,%a) would
+// drop orig_a, so both slots would read orig_b.
+struct Big C3(struct Big x, struct Big y);
+struct Big P3(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C3(b, a);
+}
+// COMMON-LABEL: define {{.*}} @P3(
+// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a,
+// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} %b,
+// COMMON: store {{.*}} [[SAVED]], ptr %b,
+// COMMON: musttail call {{.*}} @C3({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b)
+
+// P5: caller mutates the parameter before the musttail. The mutation lands
+// at the incoming pointer the callee receives.
+struct Big C5(struct Big a);
+struct Big P5(struct Big a) {
+  a.a += 1;
+  __attribute__((musttail)) return C5(a);
+}
+// COMMON-LABEL: define {{.*}} @P5(
+// COMMON: musttail call {{.*}} @C5({{.*}}, ptr {{[^,]*}} %a)
+
+// P6: musttail in a non-entry block.
+struct Big C6(struct Big a, int cond);
+struct Big P6(struct Big a, int cond) {
+  if (cond)
+    __attribute__((musttail)) return C6(a, cond);
+  return a;
+}
+// COMMON-LABEL: define {{.*}} @P6(
+// COMMON: musttail call {{.*}} @C6({{.*}}, ptr {{[^,]*}} %a,
+
+// P7: same arg to two slots. C ABI requires distinct storage per by-value
+// param, so slot 1 cannot share %a's pointer. Slot 0 forwards %a; slot 1
+// memcpys *%a into the i=1 incoming pointer %b and forwards %b.
+struct Big C7(struct Big x, struct Big y);
+struct Big P7(struct Big a, struct Big b) {
+  __attribute__((musttail)) return C7(a, a);
+}
+// COMMON-LABEL: define {{.*}} @P7(
+// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a,
+// COMMON: musttail call {{.*}} @C7({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b)
+
+// P8: local source. The local lives in our frame; copy it into %a, forward %a.
+struct Big C8(struct Big a);
+struct Big P8(struct Big a) {
+  struct Big local = {1, 2, 3, 4};
+  __attribute__((musttail)) return C8(local);
+}
+// COMMON-LABEL: define {{.*}} @P8(
+// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{.*}}
+// COMMON: musttail call {{.*}} @C8({{.*}}, ptr {{[^,]*}} %a)
+
+// P9: non-musttail tail call (existing path).
+struct Big C9(struct Big a);
+struct Big P9(struct Big a) {
+  return C9(a);
+}
+// COMMON-LABEL: define {{.*}} @P9(
+// COMMON-NOT: musttail
+
+// P10: mixed direct + indirect.
+struct Big C10(int x1, struct Big s1, int x2, struct Big s2);
+struct Big P10(int x1, struct Big s1, int x2, struct Big s2) {
+  __attribute__((musttail)) return C10(x1, s1, x2, s2);
+}
+// COMMON-LABEL: define {{.*}} @P10(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C10({{.*}}, i32 {{.*}} %x1, ptr {{[^,]*}} %s1, i32 {{.*}} %x2, ptr {{[^,]*}} %s2)
+
+// P11: many args, including stack-spilled ones on the target ABIs above.
+struct Big C11(struct Big s1, struct Big s2, struct Big s3, struct Big s4,
+               struct Big s5, struct Big s6, struct Big s7, struct Big s8,
+               struct Big s9, struct Big s10);
+struct Big P11(struct Big a1, struct Big a2, struct Big a3, struct Big a4,
+               struct Big a5, struct Big a6, struct Big a7, struct Big a8,
+               struct Big a9, struct Big a10) {
+  __attribute__((musttail)) return C11(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10);
+}
+// COMMON-LABEL: define {{.*}} @P11(
+// COMMON-NOT: = alloca {{.*}}struct.Big
+// COMMON: musttail call {{.*}} @C11(
+// COMMON-SAME: ptr {{[^,]*}} %a1, ptr {{[^,]*}} %a2, ptr {{[^,]*}} %a3, ptr {{[^,]*}} %a4
+// COMMON-SAME: ptr {{[^,]*}} %a5, ptr {{[^,]*}} %a6, ptr {{[^,]*}} %a7, ptr {{[^,]*}} %a8
+// COMMON-SAME: ptr {{[^,]*}} %a9, ptr {{[^,]*}} %a10
+
+// P12: over-aligned struct.
+struct __attribute__((aligned(32))) AlignedBig {
+  unsigned long long a, b, c, d;
+};
+struct AlignedBig C12(struct AlignedBig a);
+struct AlignedBig P12(struct AlignedBig a) {
+  __attribute__((musttail)) return C12(a);
+}
+// COMMON-LABEL: define {{.*}} @P12(
+// COMMON: musttail call {{.*}} @C12({{.*}}, ptr {{[^,]*}} %a)
+
+// P13: mixed source kinds within Indirect slots. Slot 0's source is a local;
+// slot 1's source is an incoming parameter. Both routes engage in the same
+// call: local-source case (would have dangled under v1's byval-temp path)
+// and forward case must coexist with two-phase ordering.
+struct Big C13(struct Big x, struct Big y);
+struct Big P13(struct Big a, struct Big b) {
+  struct Big local = {1, 2, 3, 4};
+  __attribute__((musttail)) return C13(local, a);
+}
+// COMMON-LABEL: define {{.*}} @P13(
+// COMMON-NOT: byval-temp
+// COMMON: %musttail.copy{{[0-9.a-z]*}} =
+// COMMON: musttail call {{.*}} @C13({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b)
+
+// P17: same arg to three slots (generalization of P7).
+struct Big C17(struct Big x, struct Big y, struct Big z);
+struct Big P17(struct Big a, struct Big b, struct Big c) {
+  __attribute__((musttail)) return C17(a, a, a);
+}
+// COMMON-LABEL: define {{.*}} @P17(
+// Both copied slots take their value from %a: %b via the memmove, %c via the
+// captured load. Neither sources from the other copied slot.
+// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a,
+// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a,
+// COMMON: store {{.*}} [[SAVED]], ptr %c,
+// COMMON: musttail call {{.*}} @C17({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b, ptr {{[^,]*}} %c)