diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td index f2ed2f4698b8d..90b035bb91435 100644 --- a/clang/include/clang/Basic/DiagnosticCommonKinds.td +++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td @@ -390,6 +390,9 @@ def err_aix_musttail_unsupported: Error< "'musttail' attribute is not supported on AIX">; def err_musttail_noexcept_mismatch: Error< "'musttail' in a noexcept function requires a noexcept callee">; +def err_musttail_unsupported_indirect_arg: Error< + "'musttail' call requires passing an argument by reference, but the source " + "does not have an addressable storage and would alias the caller's frame">; // Source manager def err_cannot_open_file : Error<"cannot open file '%0': %1">, DefaultFatal; diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 0f272eab4e842..5b23e84fc17ad 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -5173,6 +5173,26 @@ void CodeGenFunction::EmitWritebacks(const CallArgList &args) { emitWriteback(*this, I); } +/// Whether emitting this glvalue neither has side effects nor reads mutable +/// state, so deferring its byte read to the call boundary is equivalent to +/// initializing the argument last, a sequencing C++17 [expr.call]/8 allows. +/// A dereference or call in the address computation would instead split the +/// argument's evaluation around the other arguments'. +static bool isPureForwardableLValue(const Expr *E) { + E = E->IgnoreParens(); + if (const auto *DRE = dyn_cast(E)) + return isa(DRE->getDecl()); + if (const auto *ME = dyn_cast(E)) + return !ME->isArrow() && isa(ME->getMemberDecl()) && + isPureForwardableLValue(ME->getBase()); + if (const auto *ICE = dyn_cast(E)) + if (ICE->getCastKind() == CK_DerivedToBase || + ICE->getCastKind() == CK_UncheckedDerivedToBase || + ICE->getCastKind() == CK_NoOp) + return isPureForwardableLValue(ICE->getSubExpr()); + return false; +} + void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, QualType type) { std::optional Dis; @@ -5253,6 +5273,38 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, } } + // Under musttail, hand a trivially-copyable record source's LValue to + // EmitCall rather than materializing an agg.tmp. EmitCall's Indirect path + // copies it into the matching incoming parameter, which survives the tail + // call. The byte read is deferred to the call boundary, so the source is + // restricted to pure lvalue chains (see isPureForwardableLValue); casts + // are not stripped, so a derived-to-base source keeps its adjusted + // address. On the device side CUDA surface/texture types are excluded: + // they classify as Direct and forwarding would load raw record bytes + // instead of the handle that EmitAggregateCopy materializes. + if (HasAggregateEvalKind && MustTailCall && type->isRecordType() && + type.isTriviallyCopyableType(getContext()) && + !(getLangOpts().CUDAIsDevice && + (type->isCUDADeviceBuiltinSurfaceType() || + type->isCUDADeviceBuiltinTextureType()))) { + if (const auto *CCE = dyn_cast(E)) { + const CXXConstructorDecl *Ctor = CCE->getConstructor(); + if (Ctor->isCopyOrMoveConstructor() && Ctor->isTrivial() && + CCE->getNumArgs() == 1) { + const Expr *Source = CCE->getArg(0); + if (Source->isGLValue() && isPureForwardableLValue(Source) && + Source->getType().getAddressSpace() != LangAS::hlsl_constant && + getContext().hasSameUnqualifiedType(Source->getType(), type)) { + LValue L = EmitLValue(Source); + if (L.isSimple()) { + args.addUncopiedAggregate(L, type); + return; + } + } + } + } + } + args.add(EmitAnyExprToTemp(E), type); } @@ -5562,6 +5614,17 @@ static unsigned getMaxVectorWidth(const llvm::Type *Ty) { return MaxVectorWidth; } +/// Peel one AddrSpaceCastInst from \p SrcPtr. EmitParmDecl wraps incoming +/// Indirect params via address-space cast on NVPTX/AMDGPU/SPIR, so peeling +/// exposes the underlying llvm::Argument when the source IS a forwarded +/// incoming parameter. Loads are NOT unwrapped: a load through a local +/// alloca means the source is a local. +static llvm::Value *peelAddrSpaceCast(llvm::Value *SrcPtr) { + if (auto *ASC = llvm::dyn_cast(SrcPtr)) + return ASC->getOperand(0); + return SrcPtr; +} + RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, @@ -5685,6 +5748,17 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // markers that need to be ended right after the call. SmallVector CallLifetimeEndAfterCall; + // Deferred Phase-2 writes for musttail Indirect args. Splitting reads + // (in the per-arg loop) from writes (after the loop) lets permutations + // like C(b, a) land correctly: all sources are captured into scratches + // before any incoming-param destination is overwritten. + struct MustTailIndirectCopy { + LValue Scratch; + LValue Dst; + QualType Ty; + }; + llvm::SmallVector MustTailIndirectCopies; + // Translate all of the arguments as necessary to match the IR lowering. assert(CallInfo.arg_size() == CallArgs.size() && "Mismatch between function signature & arguments."); @@ -5757,6 +5831,51 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, case ABIArgInfo::Indirect: case ABIArgInfo::IndirectAliased: { assert(NumIRArgs == 1); + + // Musttail Indirect: route via the matching incoming parameter. + // Prototype-match (Verifier V5/V6/V7) makes CurFn->arg_begin()+ + // FirstIRArg a distinct destination for this slot that lives in the + // caller's caller's frame and survives the tail call. To handle + // permutations safely, the source value is captured into a scratch + // alloca here (Phase 1); the write to the incoming-param destination + // is deferred until after all sources have been read (Phase 2 below). + // IndirectAliased uses the existing fallback (different source-AS). + if (IsMustTail && ArgInfo.isIndirect()) { + llvm::Argument *IncomingArg = CurFn->arg_begin() + FirstIRArg; + llvm::Value *Dst = IncomingArg; + Address SrcAddr = Address::invalid(); + if (I->hasLValue()) + SrcAddr = I->getKnownLValue().getAddress(); + else if (I->getKnownRValue().isAggregate()) + SrcAddr = I->getKnownRValue().getAggregateAddress(); + if (SrcAddr.isValid()) { + llvm::Value *Src = peelAddrSpaceCast(SrcAddr.emitRawPointer(*this)); + if (Src != Dst) { + CharUnits Align = ArgInfo.getIndirectAlign(); + QualType Ty = I->Ty; + llvm::Type *ElemTy = ConvertTypeForMem(Ty); + RawAddress Scratch = + CreateMemTempWithoutCast(Ty, Align, "musttail.copy"); + LValue ScratchLV = MakeAddrLValue(Scratch, Ty); + LValue SrcLV = MakeAddrLValue(SrcAddr, Ty); + EmitAggregateCopy(ScratchLV, SrcLV, Ty, + AggValueSlot::DoesNotOverlap); + LValue DstLV = MakeAddrLValue(Address(Dst, ElemTy, Align), Ty); + MustTailIndirectCopies.push_back({ScratchLV, DstLV, Ty}); + } + // No freeze: Dst is an incoming parameter pointer, never poison. + IRCallArgs[FirstIRArg] = Dst; + break; + } + // No addressable source for this Indirect arg (rare; e.g. a scalar + // RValue the ABI classifies as Indirect). The fall-through below + // would create a current-frame byval-temp that dangles past the + // tail-call teardown. Refuse cleanly; codegen continues producing + // IR but the error prevents it from reaching the backend. + CGM.getDiags().Report(MustTailCall->getBeginLoc(), + diag::err_musttail_unsupported_indirect_arg); + } + if (I->isAggregate()) { // We want to avoid creating an unnecessary temporary+copy here; // however, we need one in three cases: @@ -6083,6 +6202,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, } } + // Phase 2 of the musttail Indirect-arg copy: flush each captured scratch + // into its incoming-param destination. Phase 1 has read every source, so + // permutations like C(b, a) land the right value in each slot. + for (const auto &Copy : MustTailIndirectCopies) + EmitAggregateCopy(Copy.Dst, Copy.Scratch, Copy.Ty, + AggValueSlot::DoesNotOverlap); + const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this); llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer(); diff --git a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp index d00d64c6e84d2..a7c4f8e401529 100644 --- a/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp +++ b/clang/test/CodeGen/AArch64/struct-coerce-using-ptr.cpp @@ -139,7 +139,7 @@ struct Srp { // CHECK-A64-NEXT: [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 8 // CHECK-A64-NEXT: store [2 x ptr] [[S_COERCE]], ptr [[S]], align 8 // CHECK-A64-NEXT: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0 -// CHECK-A64-NEXT: [[TMP0:%.*]] = load ptr, ptr [[X]], align 8, !nonnull [[META1:![0-9]+]], !align [[META2:![0-9]+]] +// CHECK-A64-NEXT: [[TMP0:%.*]] = load ptr, ptr [[X]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]] // CHECK-A64-NEXT: store i32 1, ptr [[TMP0]], align 4 // CHECK-A64-NEXT: ret void // @@ -149,7 +149,7 @@ struct Srp { // CHECK-A64_32-NEXT: [[S:%.*]] = alloca [[STRUCT_SRP:%.*]], align 4 // CHECK-A64_32-NEXT: store i64 [[S_COERCE]], ptr [[S]], align 4 // CHECK-A64_32-NEXT: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRP]], ptr [[S]], i32 0, i32 0 -// CHECK-A64_32-NEXT: [[TMP0:%.*]] = load ptr, ptr [[X]], align 4, !nonnull [[META1:![0-9]+]], !align [[META2:![0-9]+]] +// CHECK-A64_32-NEXT: [[TMP0:%.*]] = load ptr, ptr [[X]], align 4, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]] // CHECK-A64_32-NEXT: store i32 1, ptr [[TMP0]], align 4 // CHECK-A64_32-NEXT: ret void // @@ -740,7 +740,7 @@ struct Sraddrspace { // CHECK-A64-NEXT: [[COERCE_VAL_IP:%.*]] = inttoptr i64 [[S_COERCE]] to ptr addrspace(100) // CHECK-A64-NEXT: store ptr addrspace(100) [[COERCE_VAL_IP]], ptr [[COERCE_DIVE]], align 8 // CHECK-A64-NEXT: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0 -// CHECK-A64-NEXT: [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8, !align [[META2]] +// CHECK-A64-NEXT: [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 8, !align [[META3]] // CHECK-A64-NEXT: store i32 1, ptr addrspace(100) [[TMP0]], align 4 // CHECK-A64-NEXT: ret void // @@ -752,7 +752,7 @@ struct Sraddrspace { // CHECK-A64_32-NEXT: [[COERCE_VAL_II:%.*]] = trunc i64 [[S_COERCE]] to i32 // CHECK-A64_32-NEXT: store i32 [[COERCE_VAL_II]], ptr [[COERCE_DIVE]], align 4 // CHECK-A64_32-NEXT: [[X:%.*]] = getelementptr inbounds nuw [[STRUCT_SRADDRSPACE]], ptr [[S]], i32 0, i32 0 -// CHECK-A64_32-NEXT: [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4, !align [[META2]] +// CHECK-A64_32-NEXT: [[TMP0:%.*]] = load ptr addrspace(100), ptr [[X]], align 4, !align [[META3]] // CHECK-A64_32-NEXT: store i32 1, ptr addrspace(100) [[TMP0]], align 4 // CHECK-A64_32-NEXT: ret void // @@ -791,9 +791,9 @@ void Traddrspace(Sraddrspace s) { s.x = 1; } void Craddrspace(Sraddrspace s) { Traddrspace(s); } //. -// CHECK-A64: [[META1]] = !{} -// CHECK-A64: [[META2]] = !{i64 4} +// CHECK-A64: [[META2]] = !{} +// CHECK-A64: [[META3]] = !{i64 4} //. -// CHECK-A64_32: [[META1]] = !{} -// CHECK-A64_32: [[META2]] = !{i64 4} +// CHECK-A64_32: [[META2]] = !{} +// CHECK-A64_32: [[META3]] = !{i64 4} //. diff --git a/clang/test/CodeGen/musttail-indirect-arg-unsupported.c b/clang/test/CodeGen/musttail-indirect-arg-unsupported.c new file mode 100644 index 0000000000000..108968ffe7784 --- /dev/null +++ b/clang/test/CodeGen/musttail-indirect-arg-unsupported.c @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -triple=x86_64-linux-gnu -verify -emit-llvm-only %s +// RUN: %clang_cc1 -triple=riscv64-linux-gnu -verify -emit-llvm-only %s + +// A musttail Indirect argument is forwarded through the matching incoming +// parameter, which requires an in-memory source. A wide _BitInt has scalar +// evaluation kind, so the argument is a scalar value with no source storage +// to forward, regardless of value category. Such a call is rejected rather +// than routed through a caller-frame temp that dangles past the tail call. +// This is an implementation limit, not a fundamental one: it could be lifted +// by storing the value through the incoming parameter's own slot. + +typedef _BitInt(256) BI; +BI cee(BI x); +BI pee(BI a) { + // expected-error@+1 {{'musttail' call requires passing an argument by reference, but the source does not have an addressable storage and would alias the caller's frame}} + __attribute__((musttail)) return cee(a); +} + +// An aggregate lvalue has addressable storage to forward, so it is accepted. +// Confirms the diagnostic is specific to the no-source case. +struct Big { + unsigned long long a, b, c, d; +}; +struct Big cee_ok(struct Big x); +struct Big pee_ok(struct Big a) { + __attribute__((musttail)) return cee_ok(a); +} diff --git a/clang/test/CodeGen/musttail-indirect-arg.c b/clang/test/CodeGen/musttail-indirect-arg.c new file mode 100644 index 0000000000000..b6073bd956ba0 --- /dev/null +++ b/clang/test/CodeGen/musttail-indirect-arg.c @@ -0,0 +1,160 @@ +// RUN: %clang_cc1 -triple=riscv64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=loongarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=s390x-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON + +// Musttail calls with struct-by-value args must route each value through the +// matching incoming Indirect parameter's storage, not a local alloca that +// dangles past the tail-call frame teardown. For each Indirect slot i: if +// the source IS the i-th incoming pointer, pass it; otherwise memcpy the +// source into the i-th incoming pointer and pass that. Each call slot ends +// up with a distinct pointer in the caller's caller's frame. + +// Plain Indirect-ABI struct on the targets above. +struct Big { + unsigned long long a, b, c, d; +}; + +// P1: simple forward. +struct Big C1(struct Big a); +struct Big P1(struct Big a) { + __attribute__((musttail)) return C1(a); +} +// COMMON-LABEL: define {{.*}} @P1( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @C1({{.*}}, ptr {{[^,]*}} %a) + +// P2: two distinct incoming sources. +struct Big C2(struct Big a, struct Big b); +struct Big P2(struct Big a, struct Big b) { + __attribute__((musttail)) return C2(a, b); +} +// COMMON-LABEL: define {{.*}} @P2( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON-NOT: llvm.memcpy +// COMMON: musttail call {{.*}} @C2({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P3: swap. Pin the data flow: %a is captured before %b overwrites it, and +// the saved %a lands in %b. An in-place memmove(%a,%b);memmove(%b,%a) would +// drop orig_a, so both slots would read orig_b. +struct Big C3(struct Big x, struct Big y); +struct Big P3(struct Big a, struct Big b) { + __attribute__((musttail)) return C3(b, a); +} +// COMMON-LABEL: define {{.*}} @P3( +// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a, +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} %b, +// COMMON: store {{.*}} [[SAVED]], ptr %b, +// COMMON: musttail call {{.*}} @C3({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P5: caller mutates the parameter before the musttail. The mutation lands +// at the incoming pointer the callee receives. +struct Big C5(struct Big a); +struct Big P5(struct Big a) { + a.a += 1; + __attribute__((musttail)) return C5(a); +} +// COMMON-LABEL: define {{.*}} @P5( +// COMMON: musttail call {{.*}} @C5({{.*}}, ptr {{[^,]*}} %a) + +// P6: musttail in a non-entry block. +struct Big C6(struct Big a, int cond); +struct Big P6(struct Big a, int cond) { + if (cond) + __attribute__((musttail)) return C6(a, cond); + return a; +} +// COMMON-LABEL: define {{.*}} @P6( +// COMMON: musttail call {{.*}} @C6({{.*}}, ptr {{[^,]*}} %a, + +// P7: same arg to two slots. C ABI requires distinct storage per by-value +// param, so slot 1 cannot share %a's pointer. Slot 0 forwards %a; slot 1 +// memcpys *%a into the i=1 incoming pointer %b and forwards %b. +struct Big C7(struct Big x, struct Big y); +struct Big P7(struct Big a, struct Big b) { + __attribute__((musttail)) return C7(a, a); +} +// COMMON-LABEL: define {{.*}} @P7( +// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a, +// COMMON: musttail call {{.*}} @C7({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P8: local source. The local lives in our frame; copy it into %a, forward %a. +struct Big C8(struct Big a); +struct Big P8(struct Big a) { + struct Big local = {1, 2, 3, 4}; + __attribute__((musttail)) return C8(local); +} +// COMMON-LABEL: define {{.*}} @P8( +// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{.*}} +// COMMON: musttail call {{.*}} @C8({{.*}}, ptr {{[^,]*}} %a) + +// P9: non-musttail tail call (existing path). +struct Big C9(struct Big a); +struct Big P9(struct Big a) { + return C9(a); +} +// COMMON-LABEL: define {{.*}} @P9( +// COMMON-NOT: musttail + +// P10: mixed direct + indirect. +struct Big C10(int x1, struct Big s1, int x2, struct Big s2); +struct Big P10(int x1, struct Big s1, int x2, struct Big s2) { + __attribute__((musttail)) return C10(x1, s1, x2, s2); +} +// COMMON-LABEL: define {{.*}} @P10( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @C10({{.*}}, i32 {{.*}} %x1, ptr {{[^,]*}} %s1, i32 {{.*}} %x2, ptr {{[^,]*}} %s2) + +// P11: many args, including stack-spilled ones on the target ABIs above. +struct Big C11(struct Big s1, struct Big s2, struct Big s3, struct Big s4, + struct Big s5, struct Big s6, struct Big s7, struct Big s8, + struct Big s9, struct Big s10); +struct Big P11(struct Big a1, struct Big a2, struct Big a3, struct Big a4, + struct Big a5, struct Big a6, struct Big a7, struct Big a8, + struct Big a9, struct Big a10) { + __attribute__((musttail)) return C11(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); +} +// COMMON-LABEL: define {{.*}} @P11( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @C11( +// COMMON-SAME: ptr {{[^,]*}} %a1, ptr {{[^,]*}} %a2, ptr {{[^,]*}} %a3, ptr {{[^,]*}} %a4 +// COMMON-SAME: ptr {{[^,]*}} %a5, ptr {{[^,]*}} %a6, ptr {{[^,]*}} %a7, ptr {{[^,]*}} %a8 +// COMMON-SAME: ptr {{[^,]*}} %a9, ptr {{[^,]*}} %a10 + +// P12: over-aligned struct. +struct __attribute__((aligned(32))) AlignedBig { + unsigned long long a, b, c, d; +}; +struct AlignedBig C12(struct AlignedBig a); +struct AlignedBig P12(struct AlignedBig a) { + __attribute__((musttail)) return C12(a); +} +// COMMON-LABEL: define {{.*}} @P12( +// COMMON: musttail call {{.*}} @C12({{.*}}, ptr {{[^,]*}} %a) + +// P13: mixed source kinds within Indirect slots. Slot 0's source is a local; +// slot 1's source is an incoming parameter. Both routes engage in the same +// call: local-source case (would have dangled under v1's byval-temp path) +// and forward case must coexist with two-phase ordering. +struct Big C13(struct Big x, struct Big y); +struct Big P13(struct Big a, struct Big b) { + struct Big local = {1, 2, 3, 4}; + __attribute__((musttail)) return C13(local, a); +} +// COMMON-LABEL: define {{.*}} @P13( +// COMMON-NOT: byval-temp +// COMMON: %musttail.copy{{[0-9.a-z]*}} = +// COMMON: musttail call {{.*}} @C13({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P17: same arg to three slots (generalization of P7). +struct Big C17(struct Big x, struct Big y, struct Big z); +struct Big P17(struct Big a, struct Big b, struct Big c) { + __attribute__((musttail)) return C17(a, a, a); +} +// COMMON-LABEL: define {{.*}} @P17( +// Both copied slots take their value from %a: %b via the memmove, %c via the +// captured load. Neither sources from the other copied slot. +// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a, +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a, +// COMMON: store {{.*}} [[SAVED]], ptr %c, +// COMMON: musttail call {{.*}} @C17({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b, ptr {{[^,]*}} %c) diff --git a/clang/test/CodeGen/musttail-indirect-arg.cpp b/clang/test/CodeGen/musttail-indirect-arg.cpp new file mode 100644 index 0000000000000..dd05eab868843 --- /dev/null +++ b/clang/test/CodeGen/musttail-indirect-arg.cpp @@ -0,0 +1,224 @@ +// RUN: %clang_cc1 -triple=riscv64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=aarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=loongarch64-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON +// RUN: %clang_cc1 -triple=s390x-linux-gnu %s -emit-llvm -O1 -o - | FileCheck %s --check-prefix=COMMON + +// C++ side of the musttail Indirect-arg fix. The call argument is typically +// a CXXConstructExpr invoking the trivial copy constructor; EmitCallArg +// hands the same-type glvalue source's LValue to EmitCall so the general +// path engages. Non-trivial copy or move constructors keep the existing +// agg.tmp path. + +struct Big { + unsigned long long a, b, c, d; +}; + +// P1: simple forward. +struct Big C1(struct Big a); +struct Big P1(struct Big a) { + [[clang::musttail]] return C1(a); +} +// COMMON-LABEL: define {{.*}} @_Z2P13Big( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @_Z2C13Big({{.*}}, ptr {{[^,]*}} %a) + +// P2: two distinct args. +struct Big C2(struct Big a, struct Big b); +struct Big P2(struct Big a, struct Big b) { + [[clang::musttail]] return C2(a, b); +} +// COMMON-LABEL: define {{.*}} @_Z2P23BigS_( +// COMMON-NOT: llvm.memcpy +// COMMON: musttail call {{.*}} @_Z2C23BigS_({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P3: swap. Pin the data flow (see musttail-indirect-arg.c): %a is captured +// before %b overwrites it, and the saved %a lands in %b. +struct Big C3(struct Big x, struct Big y); +struct Big P3(struct Big a, struct Big b) { + [[clang::musttail]] return C3(b, a); +} +// COMMON-LABEL: define {{.*}} @_Z2P33BigS_( +// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a, +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} %b, +// COMMON: store {{.*}} [[SAVED]], ptr %b, +// COMMON: musttail call {{.*}} @_Z2C33BigS_({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P4: non-trivial copy constructor. The trivial-copy gate must NOT engage; +// the user-defined copy ctor IS called. Dangling-stack bug in this corner +// remains (out of scope). +struct NonTrivial { + unsigned long long parts[4]; + NonTrivial(const NonTrivial &); +}; +NonTrivial C4(NonTrivial a); +NonTrivial P4(NonTrivial a) { + [[clang::musttail]] return C4(a); +} +// COMMON-LABEL: define {{.*}} @_Z2P410NonTrivial( +// COMMON: call {{.*}} @_ZN10NonTrivialC1ERKS_ + +// P5: modify-then-forward. +struct Big C5(struct Big a); +struct Big P5(struct Big a) { + a.a += 1; + [[clang::musttail]] return C5(a); +} +// COMMON-LABEL: define {{.*}} @_Z2P53Big( +// COMMON: musttail call {{.*}} @_Z2C53Big({{.*}}, ptr {{[^,]*}} %a) + +// P6: musttail behind a branch. +struct Big C6(struct Big a, int cond); +struct Big P6(struct Big a, int cond) { + if (cond) + [[clang::musttail]] return C6(a, cond); + return a; +} +// COMMON-LABEL: define {{.*}} @_Z2P63Bigi( +// COMMON: musttail call {{.*}} @_Z2C63Bigi({{.*}}, ptr {{[^,]*}} %a, + +// P7: same arg to two slots. Slot 0 forwards %a; slot 1 memcpys *%a into the +// i=1 incoming pointer %b and forwards %b. +struct Big C7(struct Big x, struct Big y); +struct Big P7(struct Big a, struct Big b) { + [[clang::musttail]] return C7(a, a); +} +// COMMON-LABEL: define {{.*}} @_Z2P73BigS_( +// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a, +// COMMON: musttail call {{.*}} @_Z2C73BigS_({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P8: local source. Copied into the incoming %a, then %a forwarded. +struct Big C8(struct Big a); +struct Big P8(struct Big a) { + struct Big local = {1, 2, 3, 4}; + [[clang::musttail]] return C8(local); +} +// COMMON-LABEL: define {{.*}} @_Z2P83Big( +// COMMON: llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{.*}} +// COMMON: musttail call {{.*}} @_Z2C83Big({{.*}}, ptr {{[^,]*}} %a) + +// P9: non-musttail tail call (existing path). +struct Big C9(struct Big a); +struct Big P9(struct Big a) { + return C9(a); +} +// COMMON-LABEL: define {{.*}} @_Z2P93Big( +// COMMON-NOT: musttail + +// P10: mixed direct + indirect. +struct Big C10(int x1, struct Big s1, int x2, struct Big s2); +struct Big P10(int x1, struct Big s1, int x2, struct Big s2) { + [[clang::musttail]] return C10(x1, s1, x2, s2); +} +// COMMON-LABEL: define {{.*}} @_Z3P10i3BigiS_( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @_Z3C10i3BigiS_({{.*}}, i32 {{.*}} %x1, ptr {{[^,]*}} %s1, i32 {{.*}} %x2, ptr {{[^,]*}} %s2) + +// P11: many args (stack spill on the target ABIs above). +struct Big C11(struct Big s1, struct Big s2, struct Big s3, struct Big s4, + struct Big s5, struct Big s6, struct Big s7, struct Big s8, + struct Big s9, struct Big s10); +struct Big P11(struct Big a1, struct Big a2, struct Big a3, struct Big a4, + struct Big a5, struct Big a6, struct Big a7, struct Big a8, + struct Big a9, struct Big a10) { + [[clang::musttail]] return C11(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10); +} +// COMMON-LABEL: define {{.*}} @_Z3P113BigS_S_S_S_S_S_S_S_S_( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @_Z3C113BigS_S_S_S_S_S_S_S_S_( + +// P16: member function. (P15 lambda case skipped: Sema currently rejects +// musttail from a lambda's operator() to a non-member function, #119152.) +struct S { + struct Big f(struct Big a); + struct Big P16(struct Big a); +}; +struct Big S::P16(struct Big a) { + [[clang::musttail]] return f(a); +} +// COMMON-LABEL: define {{.*}} @_ZN1S3P16E3Big( +// COMMON-NOT: = alloca {{.*}}struct.Big +// COMMON: musttail call {{.*}} @_ZN1S1fE3Big({{.*}}, ptr {{.*}}, ptr {{[^,]*}} %a) + +// P13: mixed source kinds (local + incoming parameter). +struct Big C13(struct Big x, struct Big y); +struct Big P13(struct Big a, struct Big b) { + struct Big local = {1, 2, 3, 4}; + [[clang::musttail]] return C13(local, a); +} +// COMMON-LABEL: define {{.*}} @_Z3P133BigS_( +// COMMON-NOT: byval-temp +// COMMON: %musttail.copy{{[0-9.a-z]*}} = +// COMMON: musttail call {{.*}} @_Z3C133BigS_({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b) + +// P17: same arg to three slots (generalization of P7). Both copied slots +// take their value from %a: %b via the memmove, %c via the captured load. +struct Big C17(struct Big x, struct Big y, struct Big z); +struct Big P17(struct Big a, struct Big b, struct Big c) { + [[clang::musttail]] return C17(a, a, a); +} +// COMMON-LABEL: define {{.*}} @_Z3P173BigS_S_( +// COMMON: [[SAVED:%musttail.copy[0-9.a-z]*]] = load {{.*}}, ptr %a, +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %b, ptr {{[^,]*}} %a, +// COMMON: store {{.*}} [[SAVED]], ptr %c, +// COMMON: musttail call {{.*}} @_Z3C173BigS_S_({{.*}}, ptr {{[^,]*}} %a, ptr {{[^,]*}} %b, ptr {{[^,]*}} %c) + +// P18: member of a global as the source. Forwarded with no agg.tmp; the copy +// lands directly in the incoming %a. +struct Wrap { + struct Big inner; +}; +extern Wrap gw; +struct Big C18(struct Big a); +struct Big P18(struct Big a) { + [[clang::musttail]] return C18(gw.inner); +} +// COMMON-LABEL: define {{.*}} @_Z3P183Big( +// COMMON-NOT: %agg.tmp +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} @gw, i64 32 +// COMMON: musttail call {{.*}} @_Z3C183Big({{.*}}, ptr {{[^,]*}} %a) + +// P19: deref of a global pointer. The address computation reads mutable +// state, so the bytes are captured at argument position (no forwarding); +// the temp still routes through the incoming parameter. +extern struct Big *gp; +struct Big C19(struct Big a); +struct Big P19(struct Big a) { + [[clang::musttail]] return C19(*gp); +} +// COMMON-LABEL: define {{.*}} @_Z3P193Big( +// COMMON-NOT: %agg.tmp +// COMMON: [[SRC:%[0-9a-z.]+]] = load ptr, ptr @gp +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} [[SRC]], i64 32 +// COMMON: musttail call {{.*}} @_Z3C193Big({{.*}}, ptr {{[^,]*}} %a) + +// P20: derived-to-base source. The base subobject sits at offset 8 in Der; +// the forwarded address must carry that adjustment. +struct Pad { + unsigned long long p; +}; +struct Der : Pad, Big {}; +extern Der gd; +struct Big C20(struct Big a); +struct Big P20(struct Big a) { + [[clang::musttail]] return C20(gd); +} +// COMMON-LABEL: define {{.*}} @_Z3P203Big( +// COMMON-NOT: %agg.tmp +// COMMON: @llvm.mem{{(cpy|move)}}{{.*}}(ptr {{[^,]*}} %a, ptr {{[^,]*}} getelementptr inbounds {{(nuw )?}}(i8, ptr @gd, i64 8), i64 32 +// COMMON: musttail call {{.*}} @_Z3C203Big({{.*}}, ptr {{[^,]*}} %a) + +// P21: impure source with a side-effecting second argument. The source bytes +// are read at argument position, before bump() runs, so the argument's +// evaluation is not interleaved with the other argument's ([expr.call]/8). +extern int bump(); +struct Big C21(struct Big x, int y); +struct Big P21(struct Big a, int b) { + [[clang::musttail]] return C21(*gp, bump()); +} +// COMMON-LABEL: define {{.*}} @_Z3P213Bigi( +// COMMON: [[SRC:%[0-9a-z.]+]] = load ptr, ptr @gp +// COMMON-NOT: @_Z4bumpv +// COMMON: [[VAL:%[0-9a-z.]+]] = load {{.*}}, ptr [[SRC]] +// COMMON: call {{.*}} @_Z4bumpv() +// COMMON: store {{.*}} [[VAL]], ptr %a +// COMMON: musttail call {{.*}} @_Z3C213Bigi({{.*}}, ptr {{[^,]*}} %a, diff --git a/clang/test/CodeGenCUDA/surface.cu b/clang/test/CodeGenCUDA/surface.cu index 4106673f3138a..883ff78ac5601 100644 --- a/clang/test/CodeGenCUDA/surface.cu +++ b/clang/test/CodeGenCUDA/surface.cu @@ -34,6 +34,17 @@ __attribute__((device)) int foo(int x, int y) { return suld_2d_zero(surf, x, y); } +__attribute__((device)) int musttail_callee(surface s, int x); + +// A surface passed to a musttail call must keep the handle materialization; +// the argument is not forwarded as a raw lvalue. +// DEVICE-LABEL: @_Z15musttail_caller7surfaceIvLi2EEi( +// DEVICE: call i64 @llvm.nvvm.texsurf.handle.internal.p1(ptr addrspace(1) @surf) +// DEVICE: musttail call noundef i32 @_Z15musttail_callee7surfaceIvLi2EEi(i64 +__attribute__((device)) int musttail_caller(surface s, int x) { + [[clang::musttail]] return musttail_callee(surf, x); +} + // HOST: define internal void @[[PREFIX:__cuda]]_register_globals // Texture references need registering with correct arguments. // HOST: call void @[[PREFIX]]RegisterSurface(ptr %0, ptr @surf, ptr @0, ptr @0, i32 2, i32 0)