[Clang] Forward incoming Indirect parameters across musttail calls

clang/include/clang/Basic/DiagnosticCommonKinds.td

@@ -390,6 +390,9 @@ def err_aix_musttail_unsupported: Error<
   "'musttail' attribute is not supported on AIX">;
 def err_musttail_noexcept_mismatch: Error<
   "'musttail' in a noexcept function requires a noexcept callee">;
+def err_musttail_unsupported_indirect_arg: Error<
+  "'musttail' call requires passing an argument by reference, but the source "
+  "does not have an addressable storage and would alias the caller's frame">;
 
 // Source manager
 def err_cannot_open_file : Error<"cannot open file '%0': %1">, DefaultFatal;

clang/lib/CodeGen/CGCall.cpp

@@ -5173,6 +5173,26 @@ void CodeGenFunction::EmitWritebacks(const CallArgList &args) {
     emitWriteback(*this, I);
 }
 
+/// Whether emitting this glvalue neither has side effects nor reads mutable
+/// state, so deferring its byte read to the call boundary is equivalent to
+/// initializing the argument last, a sequencing C++17 [expr.call]/8 allows.
+/// A dereference or call in the address computation would instead split the
+/// argument's evaluation around the other arguments'.
+static bool isPureForwardableLValue(const Expr *E) {
+  E = E->IgnoreParens();
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+    return isa<VarDecl>(DRE->getDecl());
+  if (const auto *ME = dyn_cast<MemberExpr>(E))
+    return !ME->isArrow() && isa<FieldDecl>(ME->getMemberDecl()) &&
+           isPureForwardableLValue(ME->getBase());
+  if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E))
+    if (ICE->getCastKind() == CK_DerivedToBase ||
+        ICE->getCastKind() == CK_UncheckedDerivedToBase ||
+        ICE->getCastKind() == CK_NoOp)
+      return isPureForwardableLValue(ICE->getSubExpr());
+  return false;
+}
+
 void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
                                   QualType type) {
   std::optional<DisableDebugLocationUpdates> Dis;
@@ -5253,6 +5273,38 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     }
   }
 
+  // Under musttail, hand a trivially-copyable record source's LValue to
+  // EmitCall rather than materializing an agg.tmp. EmitCall's Indirect path
+  // copies it into the matching incoming parameter, which survives the tail
+  // call. The byte read is deferred to the call boundary, so the source is
+  // restricted to pure lvalue chains (see isPureForwardableLValue); casts
+  // are not stripped, so a derived-to-base source keeps its adjusted
+  // address. On the device side CUDA surface/texture types are excluded:
+  // they classify as Direct and forwarding would load raw record bytes
+  // instead of the handle that EmitAggregateCopy materializes.
+  if (HasAggregateEvalKind && MustTailCall && type->isRecordType() &&
+      type.isTriviallyCopyableType(getContext()) &&
+      !(getLangOpts().CUDAIsDevice &&
+        (type->isCUDADeviceBuiltinSurfaceType() ||
+         type->isCUDADeviceBuiltinTextureType()))) {
+    if (const auto *CCE = dyn_cast<CXXConstructExpr>(E)) {
+      const CXXConstructorDecl *Ctor = CCE->getConstructor();
+      if (Ctor->isCopyOrMoveConstructor() && Ctor->isTrivial() &&
+          CCE->getNumArgs() == 1) {
+        const Expr *Source = CCE->getArg(0);
+        if (Source->isGLValue() && isPureForwardableLValue(Source) &&
+            Source->getType().getAddressSpace() != LangAS::hlsl_constant &&
+            getContext().hasSameUnqualifiedType(Source->getType(), type)) {
+          LValue L = EmitLValue(Source);
+          if (L.isSimple()) {
+            args.addUncopiedAggregate(L, type);
+            return;
+          }
+        }
+      }
+    }
+  }
+
   args.add(EmitAnyExprToTemp(E), type);
 }
 
@@ -5562,6 +5614,17 @@ static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
   return MaxVectorWidth;
 }
 
+/// Peel one AddrSpaceCastInst from \p SrcPtr. EmitParmDecl wraps incoming
+/// Indirect params via address-space cast on NVPTX/AMDGPU/SPIR, so peeling
+/// exposes the underlying llvm::Argument when the source IS a forwarded
+/// incoming parameter. Loads are NOT unwrapped: a load through a local
+/// alloca means the source is a local.
+static llvm::Value *peelAddrSpaceCast(llvm::Value *SrcPtr) {
+  if (auto *ASC = llvm::dyn_cast<llvm::AddrSpaceCastInst>(SrcPtr))
+    return ASC->getOperand(0);
+  return SrcPtr;
+}
+
 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  const CGCallee &Callee,
                                  ReturnValueSlot ReturnValue,
@@ -5685,6 +5748,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // markers that need to be ended right after the call.
   SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
 
+  // Deferred Phase-2 writes for musttail Indirect args. Splitting reads
+  // (in the per-arg loop) from writes (after the loop) lets permutations
+  // like C(b, a) land correctly: all sources are captured into scratches
+  // before any incoming-param destination is overwritten.
+  struct MustTailIndirectCopy {
+    LValue Scratch;
+    LValue Dst;
+    QualType Ty;
+  };
+  llvm::SmallVector<MustTailIndirectCopy, 4> MustTailIndirectCopies;
+
   // Translate all of the arguments as necessary to match the IR lowering.
   assert(CallInfo.arg_size() == CallArgs.size() &&
          "Mismatch between function signature & arguments.");
@@ -5757,6 +5831,51 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
+
+      // Musttail Indirect: route via the matching incoming parameter.
+      // Prototype-match (Verifier V5/V6/V7) makes CurFn->arg_begin()+
+      // FirstIRArg a distinct destination for this slot that lives in the
+      // caller's caller's frame and survives the tail call. To handle
+      // permutations safely, the source value is captured into a scratch
+      // alloca here (Phase 1); the write to the incoming-param destination
+      // is deferred until after all sources have been read (Phase 2 below).
+      // IndirectAliased uses the existing fallback (different source-AS).
+      if (IsMustTail && ArgInfo.isIndirect()) {
+        llvm::Argument *IncomingArg = CurFn->arg_begin() + FirstIRArg;
+        llvm::Value *Dst = IncomingArg;
+        Address SrcAddr = Address::invalid();
+        if (I->hasLValue())
+          SrcAddr = I->getKnownLValue().getAddress();
+        else if (I->getKnownRValue().isAggregate())
+          SrcAddr = I->getKnownRValue().getAggregateAddress();
+        if (SrcAddr.isValid()) {
+          llvm::Value *Src = peelAddrSpaceCast(SrcAddr.emitRawPointer(*this));
+          if (Src != Dst) {
+            CharUnits Align = ArgInfo.getIndirectAlign();
+            QualType Ty = I->Ty;
+            llvm::Type *ElemTy = ConvertTypeForMem(Ty);
+            RawAddress Scratch =
+                CreateMemTempWithoutCast(Ty, Align, "musttail.copy");
+            LValue ScratchLV = MakeAddrLValue(Scratch, Ty);
+            LValue SrcLV = MakeAddrLValue(SrcAddr, Ty);
+            EmitAggregateCopy(ScratchLV, SrcLV, Ty,
+                              AggValueSlot::DoesNotOverlap);
+            LValue DstLV = MakeAddrLValue(Address(Dst, ElemTy, Align), Ty);
+            MustTailIndirectCopies.push_back({ScratchLV, DstLV, Ty});
+          }
+          // No freeze: Dst is an incoming parameter pointer, never poison.
+          IRCallArgs[FirstIRArg] = Dst;
+          break;
+        }
+        // No addressable source for this Indirect arg (rare; e.g. a scalar
+        // RValue the ABI classifies as Indirect). The fall-through below
+        // would create a current-frame byval-temp that dangles past the
+        // tail-call teardown. Refuse cleanly; codegen continues producing
+        // IR but the error prevents it from reaching the backend.
+        CGM.getDiags().Report(MustTailCall->getBeginLoc(),
+                              diag::err_musttail_unsupported_indirect_arg);
+      }
+
       if (I->isAggregate()) {
         // We want to avoid creating an unnecessary temporary+copy here;
         // however, we need one in three cases:
@@ -6083,6 +6202,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     }
   }
 
+  // Phase 2 of the musttail Indirect-arg copy: flush each captured scratch
+  // into its incoming-param destination. Phase 1 has read every source, so
+  // permutations like C(b, a) land the right value in each slot.
+  for (const auto &Copy : MustTailIndirectCopies)
+    EmitAggregateCopy(Copy.Dst, Copy.Scratch, Copy.Ty,
+                      AggValueSlot::DoesNotOverlap);
+
   const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
   llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
 

clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp

@@ -139,7 +139,7 @@ struct Srp {
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 8
 // CHECK-A64-NEXT:    store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8, !nonnull [[META1:![0-9]+]], !align [[META2:![0-9]+]]
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
 // CHECK-A64-NEXT:    store i32 1, ptr [[TMP0]], align 4
 // CHECK-A64-NEXT:    ret void
 //
@@ -149,7 +149,7 @@ struct Srp {
 // CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 4
 // CHECK-A64_32-NEXT:    store i64 [[S_COERCE]], ptr [[S]], align 4
 // CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0
-// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 4, !nonnull [[META1:![0-9]+]], !align [[META2:![0-9]+]]
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[X]], align 4, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
 // CHECK-A64_32-NEXT:    store i32 1, ptr [[TMP0]], align 4
 // CHECK-A64_32-NEXT:    ret void
 //
@@ -740,7 +740,7 @@ struct Sraddrspace {
 // CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100)
 // CHECK-A64-NEXT:    store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
-// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8, !align [[META2]]
+// CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8, !align [[META3]]
 // CHECK-A64-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
 // CHECK-A64-NEXT:    ret void
 //
@@ -752,7 +752,7 @@ struct Sraddrspace {
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32
 // CHECK-A64_32-NEXT:    store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4
 // CHECK-A64_32-NEXT:    [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
-// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4, !align [[META2]]
+// CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4, !align [[META3]]
 // CHECK-A64_32-NEXT:    store i32 1, ptr addrspace(100) [[TMP0]], align 4
 // CHECK-A64_32-NEXT:    ret void
 //
@@ -791,9 +791,9 @@ void Traddrspace(Sraddrspace s) { s.x = 1; }
 void Craddrspace(Sraddrspace s) { Traddrspace(s); }
 
 //.
-// CHECK-A64: [[META1]] = !{}
-// CHECK-A64: [[META2]] = !{i64 4}
+// CHECK-A64: [[META2]] = !{}
+// CHECK-A64: [[META3]] = !{i64 4}
 //.
-// CHECK-A64_32: [[META1]] = !{}
-// CHECK-A64_32: [[META2]] = !{i64 4}
+// CHECK-A64_32: [[META2]] = !{}
+// CHECK-A64_32: [[META3]] = !{i64 4}
 //.

clang/test/CodeGen/musttail-indirect-arg-unsupported.c

@@ -0,0 +1,27 @@
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -verify -emit-llvm-only %s
+// RUN: %clang_cc1 -triple=riscv64-linux-gnu -verify -emit-llvm-only %s
+
+// A musttail Indirect argument is forwarded through the matching incoming
+// parameter, which requires an in-memory source. A wide _BitInt has scalar
+// evaluation kind, so the argument is a scalar value with no source storage
+// to forward, regardless of value category. Such a call is rejected rather
+// than routed through a caller-frame temp that dangles past the tail call.
+// This is an implementation limit, not a fundamental one: it could be lifted
+// by storing the value through the incoming parameter's own slot.
+
+typedef _BitInt(256) BI;
+BI cee(BI x);
+BI pee(BI a) {
+  // expected-error@+1 {{'musttail' call requires passing an argument by reference, but the source does not have an addressable storage and would alias the caller's frame}}
+  __attribute__((musttail)) return cee(a);
+}
+
+// An aggregate lvalue has addressable storage to forward, so it is accepted.
+// Confirms the diagnostic is specific to the no-source case.
+struct Big {
+  unsigned long long a, b, c, d;
+};
+struct Big cee_ok(struct Big x);
+struct Big pee_ok(struct Big a) {
+  __attribute__((musttail)) return cee_ok(a);
+}