[X86][Windows] Return fp128 on the stack

clang/lib/CodeGen/Targets/X86.cpp

@@ -3437,8 +3437,6 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
     case BuiltinType::Int128:
     case BuiltinType::UInt128:
     case BuiltinType::Float128:
-      // 128-bit float and integer types share the same ABI.
-
       // If it's a parameter type, the normal ABI rule is that arguments larger
       // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
       // even though it isn't particularly efficient.
@@ -3449,10 +3447,14 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
 
       // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
       // Clang matches them for compatibility.
-      // NOTE: GCC actually returns f128 indirectly but will hopefully change.
-      // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054#c8.
-      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
-          llvm::Type::getInt64Ty(getVMContext()), 2));
+      if (BT->getKind() == BuiltinType::Int128 ||
+          BT->getKind() == BuiltinType::UInt128)
+        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
+            llvm::Type::getInt64Ty(getVMContext()), 2));
+
+      // Mingw64 GCC returns f128 via sret. Clang matches that for
+      // compatibility.
+      break;
 
     default:
       break;

clang/test/CodeGen/win-fp128.c

@@ -3,10 +3,10 @@
 // __float128 is unsupported on MSVC
 
 __float128 fp128_ret(void) { return 0; }
-// CHECK-GNU64: define dso_local <2 x i64>  @fp128_ret()
+// CHECK-GNU64: define dso_local fp128 @fp128_ret()
 
 __float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef dead_on_return %0, ptr noundef dead_on_return %1)
+// CHECK-GNU64: define dso_local fp128 @fp128_args(ptr noundef dead_on_return %0, ptr noundef dead_on_return %1)
 
 void fp128_vararg(int a, ...) {
   // CHECK-GNU64-LABEL: define dso_local void @fp128_vararg

llvm/docs/ReleaseNotes.md

@@ -284,6 +284,11 @@ Makes programs 10x faster by doing Special New Thing.
   two-register push in Windows x64 V3 unwind info. The directive takes two
   register operands: ``.seh_push2regs %r12, %r13``.
 
+* The `fp128` type is now passed via sret instead of XMM0 for some calling
+  conventions to match GCC. The C, Win64 and vectorcall calling conventions
+  now use sret, other calling conventions do not need to be compatible with
+  GCC and still return via XMM0.
+
 ### Changes to the OCaml bindings
 
 ### Changes to the Python bindings

llvm/lib/Target/X86/X86CallingConv.td

@@ -390,8 +390,8 @@ def RetCC_X86_Win64_C : CallingConv<[
 
 // X86-64 vectorcall return-value convention.
 def RetCC_X86_64_Vectorcall : CallingConv<[
-  // Vectorcall calling convention always returns FP values in XMMs.
-  CCIfType<[f32, f64, f128], 
+  // See RetCC_X86_64 for details on f128.
+  CCIfType<[f32, f64],
     CCAssignToReg<[XMM0, XMM1, XMM2, XMM3]>>,
 
   // Otherwise, everything is the same as Windows X86-64 C CC.
@@ -469,6 +469,36 @@ def RetCC_X86_32 : CallingConv<[
 
 // This is the root return-value convention for the X86-64 backend.
 def RetCC_X86_64 : CallingConv<[
+  // Mingw64 GCC returns f128 via sret, and LLVM matches it for compatibility.
+  //
+  // Using sret is a reasonable implementation of the Windows x64 calling
+  // convention:
+  //
+  // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#return-values
+  //
+  // > Otherwise, the caller must allocate memory for the return value and pass
+  // > a pointer to it as the first argument.
+  //
+  // Although it is not the only reasonable interpretation:
+  //
+  // > Nonscalar types including floats, doubles, and vector types such as
+  // > __m128, __m128i, __m128d are returned in XMM0.
+  //
+  // For now, we prefer compatibility with GCC. If official guidelines are ever
+  // published, this can be revisited.
+  //
+  // The alignment of 1 is so the frame's alignment is not bumped.
+  CCIfType<[f128], CCIfCC<"CallingConv::Win64", CCAssignToStack<16, 1>>>,
+  CCIfType<[f128], CCIfSubtarget<"isTargetWin64()",
+    CCIfCC<"CallingConv::C", CCAssignToStack<16, 1>>>>,
+  CCIfType<[f128], CCIfSubtarget<"isTargetWin64()",
+    CCIfCC<"CallingConv::X86_VectorCall", CCAssignToStack<16, 1>>>>,
+  // UEFI also uses the Win64 CC.
+  CCIfType<[f128], CCIfSubtarget<"isTargetUEFI64()",
+    CCIfCC<"CallingConv::C", CCAssignToStack<16, 1>>>>,
+  CCIfType<[f128], CCIfSubtarget<"isTargetUEFI64()",
+    CCIfCC<"CallingConv::X86_VectorCall", CCAssignToStack<16, 1>>>>,
+
   // HiPE uses RetCC_X86_64_HiPE
   CCIfCC<"CallingConv::HiPE", CCDelegateTo<RetCC_X86_64_HiPE>>,
 

llvm/lib/Target/X86/X86ISelLoweringCall.cpp

@@ -672,7 +672,13 @@ bool X86TargetLowering::CanLowerReturn(
     const Type *RetTy) const {
   SmallVector<CCValAssign, 16> RVLocs;
   CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
-  return CCInfo.CheckReturn(Outs, RetCC_X86);
+  if (!CCInfo.CheckReturn(Outs, RetCC_X86))
+    return false;
+
+  // Demotion to sret when the value must be returned via memory. This is the
+  // case for fp128 on windows.
+  return llvm::none_of(RVLocs,
+                       [](const CCValAssign &VA) { return VA.isMemLoc(); });
 }
 
 const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {