[Clang] Forward incoming Indirect parameters across musttail calls

clang/include/clang/Basic/DiagnosticCommonKinds.td

@@ -390,6 +390,9 @@ def err_aix_musttail_unsupported: Error<
   "'musttail' attribute is not supported on AIX">;
 def err_musttail_noexcept_mismatch: Error<
   "'musttail' in a noexcept function requires a noexcept callee">;
+def err_musttail_unsupported_indirect_arg: Error<
+  "'musttail' call requires passing an argument by reference, but the source "
+  "does not have an addressable storage and would alias the caller's frame">;
 
 // Source manager
 def err_cannot_open_file : Error<"cannot open file '%0': %1">, DefaultFatal;

clang/lib/CodeGen/CGCall.cpp

@@ -5253,6 +5253,42 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     }
   }
 
+  // C++ analog of the CK_LValueToRValue case above: a trivial copy/move
+  // constructor from a variable forwards the source LValue instead of
+  // materializing an agg.tmp. EmitCall makes the real copy at the
+  // Indirect/byval boundary. Under musttail this also keeps the value in
+  // storage that survives the tail call. Types whose copy is not a plain
+  // memcpy in EmitAggregateCopy are excluded, since forwarding would bypass
+  // that special copy: CUDA surface/texture (lowered to a handle) and, under
+  // ObjC GC, records with object members (need a write barrier).
+  if (HasAggregateEvalKind && type->isRecordType() &&
+      type.isTriviallyCopyableType(getContext()) &&
+      !type->isCUDADeviceBuiltinSurfaceType() &&
+      !type->isCUDADeviceBuiltinTextureType() &&
+      !(getLangOpts().getGC() != LangOptions::NonGC &&
+        type->getAsRecordDecl()->hasObjectMember())) {
+    if (const auto *CCE = dyn_cast<CXXConstructExpr>(E)) {
+      const CXXConstructorDecl *Ctor = CCE->getConstructor();
+      if (Ctor->isCopyOrMoveConstructor() && Ctor->isTrivial() &&
+          CCE->getNumArgs() == 1) {
+        const Expr *Source = CCE->getArg(0)->IgnoreParenImpCasts();
+        // Same-type guard: a stripped derived-to-base cast would forward a
+        // derived lvalue into a base-typed slot and slice at a wrong offset.
+        // Exclude hlsl_constant sources, as the CK_LValueToRValue path does.
+        if (const auto *DRE = dyn_cast<DeclRefExpr>(Source);
+            DRE && isa<VarDecl>(DRE->getDecl()) &&
+            Source->getType().getAddressSpace() != LangAS::hlsl_constant &&
+            getContext().hasSameUnqualifiedType(Source->getType(), type)) {
+          LValue L = EmitLValue(DRE);
+          if (L.isSimple()) {
+            args.addUncopiedAggregate(L, type);
+            return;
+          }
+        }
+      }
+    }
+  }
+
   args.add(EmitAnyExprToTemp(E), type);
 }
 
@@ -5562,6 +5598,17 @@ static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
   return MaxVectorWidth;
 }
 
+/// Peel one AddrSpaceCastInst from \p SrcPtr. EmitParmDecl wraps incoming
+/// Indirect params via address-space cast on NVPTX/AMDGPU/SPIR, so peeling
+/// exposes the underlying llvm::Argument when the source IS a forwarded
+/// incoming parameter. Loads are NOT unwrapped: a load through a local
+/// alloca means the source is a local.
+static llvm::Value *peelAddrSpaceCast(llvm::Value *SrcPtr) {
+  if (auto *ASC = llvm::dyn_cast<llvm::AddrSpaceCastInst>(SrcPtr))
+    return ASC->getOperand(0);
+  return SrcPtr;
+}
+
 RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                  const CGCallee &Callee,
                                  ReturnValueSlot ReturnValue,
@@ -5685,6 +5732,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // markers that need to be ended right after the call.
   SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
 
+  // Deferred Phase-2 writes for musttail Indirect args. Splitting reads
+  // (in the per-arg loop) from writes (after the loop) lets permutations
+  // like C(b, a) land correctly: all sources are captured into scratches
+  // before any incoming-param destination is overwritten.
+  struct MustTailIndirectCopy {
+    LValue Scratch;
+    LValue Dst;
+    QualType Ty;
+  };
+  llvm::SmallVector<MustTailIndirectCopy, 4> MustTailIndirectCopies;
+
   // Translate all of the arguments as necessary to match the IR lowering.
   assert(CallInfo.arg_size() == CallArgs.size() &&
          "Mismatch between function signature & arguments.");
@@ -5757,6 +5815,51 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     case ABIArgInfo::Indirect:
     case ABIArgInfo::IndirectAliased: {
       assert(NumIRArgs == 1);
+
+      // Musttail Indirect: route via the matching incoming parameter.
+      // Prototype-match (Verifier V5/V6/V7) makes CurFn->arg_begin()+
+      // FirstIRArg a distinct destination for this slot that lives in the
+      // caller's caller's frame and survives the tail call. To handle
+      // permutations safely, the source value is captured into a scratch
+      // alloca here (Phase 1); the write to the incoming-param destination
+      // is deferred until after all sources have been read (Phase 2 below).
+      // IndirectAliased uses the existing fallback (different source-AS).
+      if (IsMustTail && ArgInfo.isIndirect()) {
+        llvm::Argument *IncomingArg = CurFn->arg_begin() + FirstIRArg;
+        llvm::Value *Dst = IncomingArg;
+        Address SrcAddr = Address::invalid();
+        if (I->hasLValue())
+          SrcAddr = I->getKnownLValue().getAddress();
+        else if (I->getKnownRValue().isAggregate())
+          SrcAddr = I->getKnownRValue().getAggregateAddress();
+        if (SrcAddr.isValid()) {
+          llvm::Value *Src = peelAddrSpaceCast(SrcAddr.emitRawPointer(*this));
+          if (Src != Dst) {
+            CharUnits Align = ArgInfo.getIndirectAlign();
+            QualType Ty = I->Ty;
+            llvm::Type *ElemTy = ConvertTypeForMem(Ty);
+            RawAddress Scratch =
+                CreateMemTempWithoutCast(Ty, Align, "musttail.copy");
+            LValue ScratchLV = MakeAddrLValue(Scratch, Ty);
+            LValue SrcLV = MakeAddrLValue(SrcAddr, Ty);
+            EmitAggregateCopy(ScratchLV, SrcLV, Ty,
+                              AggValueSlot::DoesNotOverlap);
+            LValue DstLV = MakeAddrLValue(Address(Dst, ElemTy, Align), Ty);
+            MustTailIndirectCopies.push_back({ScratchLV, DstLV, Ty});
+          }
+          // No freeze: Dst is an incoming parameter pointer, never poison.
+          IRCallArgs[FirstIRArg] = Dst;
+          break;
+        }
+        // No addressable source for this Indirect arg (rare; e.g. a scalar
+        // RValue the ABI classifies as Indirect). The fall-through below
+        // would create a current-frame byval-temp that dangles past the
+        // tail-call teardown. Refuse cleanly; codegen continues producing
+        // IR but the error prevents it from reaching the backend.
+        CGM.getDiags().Report(MustTailCall->getBeginLoc(),
+                              diag::err_musttail_unsupported_indirect_arg);
+      }
+
       if (I->isAggregate()) {
         // We want to avoid creating an unnecessary temporary+copy here;
         // however, we need one in three cases:
@@ -6083,6 +6186,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     }
   }
 
+  // Phase 2 of the musttail Indirect-arg copy: flush each captured scratch
+  // into its incoming-param destination. Phase 1 has read every source, so
+  // permutations like C(b, a) land the right value in each slot.
+  for (const auto &Copy : MustTailIndirectCopies)
+    EmitAggregateCopy(Copy.Dst, Copy.Scratch, Copy.Ty,
+                      AggValueSlot::DoesNotOverlap);
+
   const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
   llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
 

clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp

@@ -654,9 +654,7 @@ void Tpaddrspace(Spaddrspace s) { *s.x = 1; }
 // CHECK-A64-SAME: ) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 8
-// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SPADDRSPACE]], align 8
-// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 8, i1 false)
-// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i64
 // CHECK-A64-NEXT:    call void @_Z11Tpaddrspace11Spaddrspace(i64 [[COERCE_VAL_PI]])
@@ -666,9 +664,7 @@ void Tpaddrspace(Spaddrspace s) { *s.x = 1; }
 // CHECK-A64_32-SAME: ) #[[ATTR0]] {
 // CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SPADDRSPACE:%.*]], align 4
-// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SPADDRSPACE]], align 4
-// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 4, i1 false)
-// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SPADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE]], align 4
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i32
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = zext i32 [[COERCE_VAL_PI]] to i64
@@ -709,9 +705,7 @@ void Tp2addrspace(Sp2addrspace s) { *s.x[0] = 1; }
 // CHECK-A64-SAME: ) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 8
-// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SP2ADDRSPACE]], align 8
-// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 16, i1 false)
-// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load [2 x i64], ptr [[COERCE_DIVE]], align 8
 // CHECK-A64-NEXT:    call void @_Z12Tp2addrspace12Sp2addrspace([2 x i64] [[TMP0]])
 // CHECK-A64-NEXT:    ret void
@@ -720,9 +714,7 @@ void Tp2addrspace(Sp2addrspace s) { *s.x[0] = 1; }
 // CHECK-A64_32-SAME: ) #[[ATTR0]] {
 // CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SP2ADDRSPACE:%.*]], align 4
-// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SP2ADDRSPACE]], align 4
-// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 8, i1 false)
-// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SP2ADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load i64, ptr [[COERCE_DIVE]], align 4
 // CHECK-A64_32-NEXT:    call void @_Z12Tp2addrspace12Sp2addrspace(i64 [[TMP0]])
 // CHECK-A64_32-NEXT:    ret void
@@ -761,12 +753,10 @@ void Traddrspace(Sraddrspace s) { s.x = 1; }
 // CHECK-A64-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 8
-// CHECK-A64-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SRADDRSPACE]], align 8
 // CHECK-A64-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100)
 // CHECK-A64-NEXT:    store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8
-// CHECK-A64-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP]], ptr align 8 [[S]], i64 8, i1 false)
-// CHECK-A64-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE1]], align 8
 // CHECK-A64-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i64
 // CHECK-A64-NEXT:    call void @_Z11Traddrspace11Sraddrspace(i64 [[COERCE_VAL_PI]])
@@ -776,12 +766,10 @@ void Traddrspace(Sraddrspace s) { s.x = 1; }
 // CHECK-A64_32-SAME: i64 [[S_COERCE:%.*]]) #[[ATTR0]] {
 // CHECK-A64_32-NEXT:  [[ENTRY:.*:]]
 // CHECK-A64_32-NEXT:    [[S:%.*]] = alloca [[STRUCT_SRADDRSPACE:%.*]], align 4
-// CHECK-A64_32-NEXT:    [[AGG_TMP:%.*]] = alloca [[STRUCT_SRADDRSPACE]], align 4
 // CHECK-A64_32-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32
 // CHECK-A64_32-NEXT:    store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4
-// CHECK-A64_32-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[AGG_TMP]], ptr align 4 [[S]], i32 4, i1 false)
-// CHECK-A64_32-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[AGG_TMP]], i32 0, i32 0
+// CHECK-A64_32-NEXT:    [[COERCE_DIVE1:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0
 // CHECK-A64_32-NEXT:    [[TMP0:%.*]] = load ptr addrspace(100), ptr [[COERCE_DIVE1]], align 4
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_PI:%.*]] = ptrtoint ptr addrspace(100) [[TMP0]] to i32
 // CHECK-A64_32-NEXT:    [[COERCE_VAL_II2:%.*]] = zext i32 [[COERCE_VAL_PI]] to i64

clang/test/CodeGen/aapcs64-align.cpp

@@ -122,7 +122,7 @@ unsigned sizeof_RidiculouslyOverSizedBitfield = sizeof(RidiculouslyOverSizedBitf
 unsigned alignof_RidiculouslyOverSizedBitfield = alignof(RidiculouslyOverSizedBitfield);
 
 // CHECK: define{{.*}} void @g9
-// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull align 16 dead_on_return %agg.tmp)
+// CHECK: call void @f9(i32 noundef 1, ptr noundef nonnull align 16 dead_on_return %byval-temp)
 // CHECK: declare void @f9(i32 noundef, ptr noundef align 16 dead_on_return)
 void f9(int a, RidiculouslyOverSizedBitfield b);
 void g9() {

clang/test/CodeGen/musttail-indirect-arg-unsupported.c

@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -triple=x86_64-linux-gnu -verify -emit-llvm-only %s
+// RUN: %clang_cc1 -triple=riscv64-linux-gnu -verify -emit-llvm-only %s
+
+// A musttail Indirect argument is forwarded through the matching incoming
+// parameter, which requires an in-memory source. A wide _BitInt has scalar
+// evaluation kind, so the argument is a scalar value with no source storage
+// to forward, regardless of value category. Such a call is rejected rather
+// than routed through a caller-frame temp that dangles past the tail call.
+
+typedef _BitInt(256) BI;
+BI cee(BI x);
+BI pee(BI a) {
+  // expected-error@+1 {{'musttail' call requires passing an argument by reference, but the source does not have an addressable storage and would alias the caller's frame}}
+  __attribute__((musttail)) return cee(a);
+}
+
+// An aggregate lvalue has addressable storage to forward, so it is accepted.
+// Confirms the diagnostic is specific to the no-source case.
+struct Big {
+  unsigned long long a, b, c, d;
+};
+struct Big cee_ok(struct Big x);
+struct Big pee_ok(struct Big a) {
+  __attribute__((musttail)) return cee_ok(a);
+}