From 5c265017a1d892fc8c1b720dfce06ac0cff62175 Mon Sep 17 00:00:00 2001 From: Feng Zou Date: Thu, 4 Jun 2026 19:53:55 +0800 Subject: [PATCH 1/3] [X86] Fix crash in LowerMSTORE for maxnum/minnum store on AVX512F without VLX On AVX512F targets without VLX (e.g. KNL), masked stores are custom-lowered by widening to 512-bit operations. A maxnum/minnum store-back is expanded using a legacy packed (CMPP) vector comparison, and the store-back DAGCombine fuses that vector comparison into the masked store. As a result LowerMSTORE receives a mask with vNi32 elements instead of vNi1 and hits an assertion: Assertion `Mask.getSimpleValueType().getScalarType() == MVT::i1 && "Unexpected mask type"' failed. (reproduces with -mattr=+avx512f or -mcpu=knl; -mcpu=skylake-avx512 and +avx512vl are unaffected because the mask is produced as a k-register.) Accept a wide vNi32 mask in addition to vNi1: widen it in its own element type and truncate to vNi1 before building the masked store. A CMPP result is all-ones/all-zeros per lane, so bit 0 of each lane carries the predicate and the zero-filled widened lanes truncate to 0 (disabled). --- llvm/lib/Target/X86/X86ISelLowering.cpp | 17 +- .../X86/avx512-maxnum-minnum-masked-store.ll | 149 ++++++++++++++++++ 2 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index a8236f02f8df4..8d55beaf264b1 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -34287,14 +34287,23 @@ static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget, unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits(); MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec); - // Mask element has to be i1. - assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 && - "Unexpected mask type"); + // For AVX512F targets without VLX (e.g. KNL), a maxnum/minnum store-back + // fuses a legacy CMPP vector comparison into the masked store, so the mask + // may arrive as vNi32 instead of vNi1. Accept both and convert to i1 below. + MVT MaskEltVT = Mask.getSimpleValueType().getScalarType(); + assert((MaskEltVT == MVT::i1 || MaskEltVT.getSizeInBits() >= 8) && + "Expected an i1 mask or a wide vector mask from a CMPP comparison"); - MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); + MVT WideMaskVT = MVT::getVectorVT(MaskEltVT, NumEltsInWideVec); DataToStore = ExtendToType(DataToStore, WideDataVT, DAG); Mask = ExtendToType(Mask, WideMaskVT, DAG, true); + // Truncate a vNi32 mask down to vNi1 for the masked op. A CMPP result is + // all-ones/all-zeros per lane, so bit 0 of each lane carries the predicate, + // and the zero-filled widened lanes truncate to 0 (disabled). + if (MaskEltVT != MVT::i1) + Mask = DAG.getNode(ISD::TRUNCATE, dl, + MVT::getVectorVT(MVT::i1, NumEltsInWideVec), Mask); return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(), N->getOffset(), Mask, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), diff --git a/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll new file mode 100644 index 0000000000000..caeba9a2f1372 --- /dev/null +++ b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll @@ -0,0 +1,149 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=AVX512VL + +; On AVX512F targets without VLX (e.g. KNL), masked stores are widened to 512 +; bits during custom lowering. A maxnum/minnum store-back fuses the legacy CMPP +; vector comparison into the masked store, so the mask arrives as vNi32 rather +; than vNi1. The lowering must accept and convert the wide mask, not assert. + +define void @maxnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { +; AVX512F-LABEL: maxnum_v4f32_masked_store: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovups (%rdi), %xmm1 +; AVX512F-NEXT: vmaxps %xmm0, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: maxnum_v4f32_masked_store: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovups (%rdi), %xmm1 +; AVX512VL-NEXT: vmaxps %xmm0, %xmm1, %xmm1 +; AVX512VL-NEXT: vcmpordps %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovups %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %b = load <4 x float>, ptr %ptr, align 4 + %m = call <4 x float> @llvm.maxnum.v4f32(<4 x float> %a, <4 x float> %b) + store <4 x float> %m, ptr %ptr, align 4 + ret void +} + +define void @maxnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { +; AVX512F-LABEL: maxnum_v2f64_masked_store: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovupd (%rdi), %xmm1 +; AVX512F-NEXT: vmaxpd %xmm0, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: maxnum_v2f64_masked_store: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovupd (%rdi), %xmm1 +; AVX512VL-NEXT: vmaxpd %xmm0, %xmm1, %xmm1 +; AVX512VL-NEXT: vcmpordpd %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovupd %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %b = load <2 x double>, ptr %ptr, align 8 + %m = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %a, <2 x double> %b) + store <2 x double> %m, ptr %ptr, align 8 + ret void +} + +define void @minnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { +; AVX512F-LABEL: minnum_v4f32_masked_store: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovups (%rdi), %xmm1 +; AVX512F-NEXT: vminps %xmm0, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: minnum_v4f32_masked_store: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovups (%rdi), %xmm1 +; AVX512VL-NEXT: vminps %xmm0, %xmm1, %xmm1 +; AVX512VL-NEXT: vcmpordps %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovups %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %b = load <4 x float>, ptr %ptr, align 4 + %m = call <4 x float> @llvm.minnum.v4f32(<4 x float> %a, <4 x float> %b) + store <4 x float> %m, ptr %ptr, align 4 + ret void +} + +define void @minnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { +; AVX512F-LABEL: minnum_v2f64_masked_store: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmovupd (%rdi), %xmm1 +; AVX512F-NEXT: vminpd %xmm0, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: minnum_v2f64_masked_store: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vmovupd (%rdi), %xmm1 +; AVX512VL-NEXT: vminpd %xmm0, %xmm1, %xmm1 +; AVX512VL-NEXT: vcmpordpd %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovupd %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %b = load <2 x double>, ptr %ptr, align 8 + %m = call <2 x double> @llvm.minnum.v2f64(<2 x double> %a, <2 x double> %b) + store <2 x double> %m, ptr %ptr, align 8 + ret void +} + +; Explicit masked store using a setcc mask (reaches LowerMSTORE with an i1 mask). +define void @explicit_masked_store_setcc(<4 x float> %a, <4 x float> %data, ptr %ptr) { +; AVX512F-LABEL: explicit_masked_store_setcc: +; AVX512F: # %bb.0: +; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512F-NEXT: vcmpordps %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: explicit_masked_store_setcc: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vcmpordps %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovups %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %cmp = fcmp ord <4 x float> %a, %a + call void @llvm.masked.store.v4f32.p0(<4 x float> %data, ptr %ptr, i32 4, <4 x i1> %cmp) + ret void +} + +define void @v4i32_masked_store(<4 x i32> %a, <4 x i32> %data, ptr %ptr) { +; AVX512F-LABEL: v4i32_masked_store: +; AVX512F: # %bb.0: +; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 +; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 +; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 +; AVX512F-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1} +; AVX512F-NEXT: vzeroupper +; AVX512F-NEXT: retq +; +; AVX512VL-LABEL: v4i32_masked_store: +; AVX512VL: # %bb.0: +; AVX512VL-NEXT: vptestnmd %xmm0, %xmm0, %k1 +; AVX512VL-NEXT: vmovdqu32 %xmm1, (%rdi) {%k1} +; AVX512VL-NEXT: retq + %cmp = icmp eq <4 x i32> %a, zeroinitializer + call void @llvm.masked.store.v4i32.p0(<4 x i32> %data, ptr %ptr, i32 4, <4 x i1> %cmp) + ret void +} From a4906251b7951501013f593959137f311c25bccb Mon Sep 17 00:00:00 2001 From: Feng Zou Date: Fri, 5 Jun 2026 15:01:57 +0800 Subject: [PATCH 2/3] [DAG] Narrow vselect mask to vXi1 in foldToMaskedStore foldToMaskedStore (added in 1c0ac80d4a9e) rewrites store(vselect(cond, x, load(ptr)), ptr) -> masked_store(x, ptr, cond) passing the vselect condition straight through as the store mask. A masked store follows the IR convention of a vXi1 mask, but the condition can be a wider boolean vector. On AVX512F targets without VLX, a maxnum/minnum store-back lowers the NaN test with a legacy packed (CMPP) comparison whose result is a vXi32/vXi64 vector, so the masked store is created with a wide mask and LowerMSTORE asserts: Assertion `Mask.getSimpleValueType().getScalarType() == MVT::i1 && "Unexpected mask type"' failed. When the matching vXi1 type is legal, narrow the mask to it before building the masked store. Targets where vXi1 is illegal (e.g. AVX/AVX2) keep the wide mask and continue to lower it as a blend/vmaskmov, and targets whose vselect condition is already vXi1 (e.g. AArch64 SVE, RISC-V RVV) are unaffected. This fixes the crash at the source and lets the X86 LowerMSTORE keep its invariant of only ever seeing a vXi1 mask (no target-specific workaround). Co-Authored-By: Claude Opus 4.8 (1M context) --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 13 ++ llvm/lib/Target/X86/X86ISelLowering.cpp | 17 +-- .../X86/avx512-maxnum-minnum-masked-store.ll | 116 +++++++++--------- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 7b2a35b10c318..38a493676e97d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -23963,6 +23963,19 @@ static SDValue foldToMaskedStore(StoreSDNode *Store, SelectionDAG &DAG, if (LoadPos == 1) Mask = DAG.getNOT(Dl, Mask, Mask.getValueType()); + // A masked store follows the IR convention of a vXi1 mask (one bit per + // element). A vselect condition may instead be a wider boolean vector, e.g. + // a vXi32/vXi64 comparison result produced on AVX512 targets without VLX. + // When the matching vXi1 type is legal, narrow the mask to it so that targets + // expecting a vXi1 mask lower it correctly. Targets where vXi1 is illegal + // (e.g. AVX/AVX2) keep the wide mask and lower it as a blend/vmaskmov. + EVT MaskVT = Mask.getValueType(); + if (MaskVT.getVectorElementType() != MVT::i1) { + EVT BoolVT = MaskVT.changeVectorElementType(*DAG.getContext(), MVT::i1); + if (TLI.isTypeLegal(BoolVT)) + Mask = DAG.getNode(ISD::TRUNCATE, Dl, BoolVT, Mask); + } + return DAG.getMaskedStore(Store->getChain(), Dl, OtherVec, StorePtr, StoreOffset, Mask, VT, Store->getMemOperand(), Store->getAddressingMode()); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 8d55beaf264b1..a8236f02f8df4 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -34287,23 +34287,14 @@ static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget, unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits(); MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec); - // For AVX512F targets without VLX (e.g. KNL), a maxnum/minnum store-back - // fuses a legacy CMPP vector comparison into the masked store, so the mask - // may arrive as vNi32 instead of vNi1. Accept both and convert to i1 below. - MVT MaskEltVT = Mask.getSimpleValueType().getScalarType(); - assert((MaskEltVT == MVT::i1 || MaskEltVT.getSizeInBits() >= 8) && - "Expected an i1 mask or a wide vector mask from a CMPP comparison"); + // Mask element has to be i1. + assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 && + "Unexpected mask type"); - MVT WideMaskVT = MVT::getVectorVT(MaskEltVT, NumEltsInWideVec); + MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); DataToStore = ExtendToType(DataToStore, WideDataVT, DAG); Mask = ExtendToType(Mask, WideMaskVT, DAG, true); - // Truncate a vNi32 mask down to vNi1 for the masked op. A CMPP result is - // all-ones/all-zeros per lane, so bit 0 of each lane carries the predicate, - // and the zero-filled widened lanes truncate to 0 (disabled). - if (MaskEltVT != MVT::i1) - Mask = DAG.getNode(ISD::TRUNCATE, dl, - MVT::getVectorVT(MVT::i1, NumEltsInWideVec), Mask); return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(), N->getOffset(), Mask, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(), diff --git a/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll index caeba9a2f1372..e988507e4bb3c 100644 --- a/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll +++ b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll @@ -1,19 +1,36 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefix=AVX ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=AVX512VL -; On AVX512F targets without VLX (e.g. KNL), masked stores are widened to 512 -; bits during custom lowering. A maxnum/minnum store-back fuses the legacy CMPP -; vector comparison into the masked store, so the mask arrives as vNi32 rather -; than vNi1. The lowering must accept and convert the wide mask, not assert. +; On AVX512F targets without VLX (e.g. KNL), a maxnum/minnum store-back fuses a +; legacy CMPP vector comparison into a masked store via foldToMaskedStore, so +; the vselect mask is a wide vNi32/vNi64 vector rather than vNi1. The combine +; must narrow the mask to vNi1 (legal here) before building the masked store, so +; the widening custom lowering only ever sees an i1 mask and does not assert. +; +; On AVX/AVX2 the matching vNi1 type is illegal, so the combine keeps the wide +; mask and the masked store lowers to vmaskmov (the isTypeLegal guard leaves +; this path unchanged). define void @maxnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { +; AVX-LABEL: maxnum_v4f32_masked_store: +; AVX: # %bb.0: +; AVX-NEXT: vmovups (%rdi), %xmm1 +; AVX-NEXT: vmaxps %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi) +; AVX-NEXT: retq +; ; AVX512F-LABEL: maxnum_v4f32_masked_store: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovups (%rdi), %xmm1 ; AVX512F-NEXT: vmaxps %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 -; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 ; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -32,12 +49,22 @@ define void @maxnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { } define void @maxnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { +; AVX-LABEL: maxnum_v2f64_masked_store: +; AVX: # %bb.0: +; AVX-NEXT: vmovupd (%rdi), %xmm1 +; AVX-NEXT: vmaxpd %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vmaskmovpd %xmm1, %xmm0, (%rdi) +; AVX-NEXT: retq +; ; AVX512F-LABEL: maxnum_v2f64_masked_store: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovupd (%rdi), %xmm1 ; AVX512F-NEXT: vmaxpd %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 -; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestnmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $14, %k0, %k0 +; AVX512F-NEXT: kshiftrw $14, %k0, %k1 ; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1} ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -56,12 +83,22 @@ define void @maxnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { } define void @minnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { +; AVX-LABEL: minnum_v4f32_masked_store: +; AVX: # %bb.0: +; AVX-NEXT: vmovups (%rdi), %xmm1 +; AVX-NEXT: vminps %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi) +; AVX-NEXT: retq +; ; AVX512F-LABEL: minnum_v4f32_masked_store: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovups (%rdi), %xmm1 ; AVX512F-NEXT: vminps %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vcmpordps %xmm0, %xmm0, %xmm0 -; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vcmpunordps %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $12, %k0, %k0 +; AVX512F-NEXT: kshiftrw $12, %k0, %k1 ; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -80,12 +117,22 @@ define void @minnum_v4f32_masked_store(<4 x float> %a, ptr %ptr) { } define void @minnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { +; AVX-LABEL: minnum_v2f64_masked_store: +; AVX: # %bb.0: +; AVX-NEXT: vmovupd (%rdi), %xmm1 +; AVX-NEXT: vminpd %xmm0, %xmm1, %xmm1 +; AVX-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 +; AVX-NEXT: vmaskmovpd %xmm1, %xmm0, (%rdi) +; AVX-NEXT: retq +; ; AVX512F-LABEL: minnum_v2f64_masked_store: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vmovupd (%rdi), %xmm1 ; AVX512F-NEXT: vminpd %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vcmpordpd %xmm0, %xmm0, %xmm0 -; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1 +; AVX512F-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vptestnmq %zmm0, %zmm0, %k0 +; AVX512F-NEXT: kshiftlw $14, %k0, %k0 +; AVX512F-NEXT: kshiftrw $14, %k0, %k1 ; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1} ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -102,48 +149,3 @@ define void @minnum_v2f64_masked_store(<2 x double> %a, ptr %ptr) { store <2 x double> %m, ptr %ptr, align 8 ret void } - -; Explicit masked store using a setcc mask (reaches LowerMSTORE with an i1 mask). -define void @explicit_masked_store_setcc(<4 x float> %a, <4 x float> %data, ptr %ptr) { -; AVX512F-LABEL: explicit_masked_store_setcc: -; AVX512F: # %bb.0: -; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 -; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 -; AVX512F-NEXT: vcmpordps %zmm0, %zmm0, %k0 -; AVX512F-NEXT: kshiftlw $12, %k0, %k0 -; AVX512F-NEXT: kshiftrw $12, %k0, %k1 -; AVX512F-NEXT: vmovups %zmm1, (%rdi) {%k1} -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512VL-LABEL: explicit_masked_store_setcc: -; AVX512VL: # %bb.0: -; AVX512VL-NEXT: vcmpordps %xmm0, %xmm0, %k1 -; AVX512VL-NEXT: vmovups %xmm1, (%rdi) {%k1} -; AVX512VL-NEXT: retq - %cmp = fcmp ord <4 x float> %a, %a - call void @llvm.masked.store.v4f32.p0(<4 x float> %data, ptr %ptr, i32 4, <4 x i1> %cmp) - ret void -} - -define void @v4i32_masked_store(<4 x i32> %a, <4 x i32> %data, ptr %ptr) { -; AVX512F-LABEL: v4i32_masked_store: -; AVX512F: # %bb.0: -; AVX512F-NEXT: # kill: def $xmm1 killed $xmm1 def $zmm1 -; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0 -; AVX512F-NEXT: vptestnmd %zmm0, %zmm0, %k0 -; AVX512F-NEXT: kshiftlw $12, %k0, %k0 -; AVX512F-NEXT: kshiftrw $12, %k0, %k1 -; AVX512F-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1} -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512VL-LABEL: v4i32_masked_store: -; AVX512VL: # %bb.0: -; AVX512VL-NEXT: vptestnmd %xmm0, %xmm0, %k1 -; AVX512VL-NEXT: vmovdqu32 %xmm1, (%rdi) {%k1} -; AVX512VL-NEXT: retq - %cmp = icmp eq <4 x i32> %a, zeroinitializer - call void @llvm.masked.store.v4i32.p0(<4 x i32> %data, ptr %ptr, i32 4, <4 x i1> %cmp) - ret void -} From a9f8d06c40ca6af96103a84e4097d7debacdb9d2 Mon Sep 17 00:00:00 2001 From: Feng Zou Date: Fri, 5 Jun 2026 18:29:57 +0800 Subject: [PATCH 3/3] Drop -gnu from triple --- .../test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll index e988507e4bb3c..f6ca1490b558c 100644 --- a/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll +++ b/llvm/test/CodeGen/X86/avx512-maxnum-minnum-masked-store.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefix=AVX -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=AVX512VL +; RUN: llc < %s -mtriple=x86_64-unknown-linux -mattr=+avx | FileCheck %s --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown-linux -mattr=+avx2 | FileCheck %s --check-prefix=AVX +; RUN: llc < %s -mtriple=x86_64-unknown-linux -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F +; RUN: llc < %s -mtriple=x86_64-unknown-linux -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefix=AVX512VL ; On AVX512F targets without VLX (e.g. KNL), a maxnum/minnum store-back fuses a ; legacy CMPP vector comparison into a masked store via foldToMaskedStore, so