[X86][Windows] Return fp128 on the stack

clang/lib/CodeGen/Targets/X86.cpp

@@ -3437,8 +3437,6 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
     case BuiltinType::Int128:
     case BuiltinType::UInt128:
     case BuiltinType::Float128:
-      // 128-bit float and integer types share the same ABI.
-
       // If it's a parameter type, the normal ABI rule is that arguments larger
       // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
       // even though it isn't particularly efficient.
@@ -3449,10 +3447,14 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
 
       // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
       // Clang matches them for compatibility.
-      // NOTE: GCC actually returns f128 indirectly but will hopefully change.
-      // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054#c8.
-      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
-          llvm::Type::getInt64Ty(getVMContext()), 2));
+      if (BT->getKind() == BuiltinType::Int128 ||
+          BT->getKind() == BuiltinType::UInt128)
+        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
+            llvm::Type::getInt64Ty(getVMContext()), 2));
+
+      // Mingw64 GCC returns f128 via sret. Clang matches that for
+      // compatibility.
+      break;
 
     default:
       break;

clang/test/CodeGen/win-fp128.c

@@ -3,10 +3,10 @@
 // __float128 is unsupported on MSVC
 
 __float128 fp128_ret(void) { return 0; }
-// CHECK-GNU64: define dso_local <2 x i64>  @fp128_ret()
+// CHECK-GNU64: define dso_local fp128 @fp128_ret()
 
 __float128 fp128_args(__float128 a, __float128 b) { return a * b; }
-// CHECK-GNU64: define dso_local <2 x i64> @fp128_args(ptr noundef dead_on_return %0, ptr noundef dead_on_return %1)
+// CHECK-GNU64: define dso_local fp128 @fp128_args(ptr noundef dead_on_return %0, ptr noundef dead_on_return %1)
 
 void fp128_vararg(int a, ...) {
   // CHECK-GNU64-LABEL: define dso_local void @fp128_vararg

llvm/lib/Target/X86/X86ISelLoweringCall.cpp

@@ -670,6 +670,20 @@ bool X86TargetLowering::CanLowerReturn(
     CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
     const Type *RetTy) const {
+  // Mingw64 GCC returns f128 via sret, which matches the documentation of the
+  // Windows x64 calling convention:
+  //
+  // https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#return-values
+  //
+  // > Otherwise, the caller must allocate memory for the return value and pass
+  // a pointer to it as the first argument.
+  //
+  // Return false, which will perform sret demotion.
+  if (Subtarget.isCallingConvWin64(CallConv) &&
+      llvm::any_of(
+          Outs, [](const ISD::OutputArg &Out) { return Out.VT == MVT::f128; }))
+    return false;
+
   SmallVector<CCValAssign, 16> RVLocs;
   CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
   return CCInfo.CheckReturn(Outs, RetCC_X86);