Skip to content

[CostModel][X86] Add more realistic v8i64/v16i32 + v8f64/v16f32 add reduction costs #206124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RKSimon merged 1 commit into from
Jun 26, 2026
Merged

[CostModel][X86] Add more realistic v8i64/v16i32 + v8f64/v16f32 add reduction costs #206124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5671,6 +5671,18 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
{ ISD::ADD, MVT::v32i8, 4 },
};

static const CostTblEntry AVX512FCostTbl[] = {
{ ISD::FADD, MVT::v8f64, 4 },
{ ISD::FADD, MVT::v16f32, 5 },
{ ISD::ADD, MVT::v8i64, 4 },
{ ISD::ADD, MVT::v16i32, 6 },
};

static const CostTblEntry AVX512BWCostTbl[] = {
{ ISD::ADD, MVT::v32i16, 7 },
{ ISD::ADD, MVT::v64i8, 4 },
};

int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");

Expand All @@ -5684,6 +5696,14 @@ X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
return Entry->Cost;

if (ST->hasBWI())
if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
return Entry->Cost;

if (ST->hasAVX512())
if (const auto *Entry = CostTableLookup(AVX512FCostTbl, ISD, MTy))
return Entry->Cost;

if (ST->hasAVX())
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
return Entry->Cost;
Expand Down
16 changes: 8 additions & 8 deletions llvm/test/Analysis/CostModel/X86/reduce-add.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,23 @@ define i32 @reduce_i64(i32 %arg) {
; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:8 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512BW-LABEL: 'reduce_i64'
; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:7 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:8 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512DQ-LABEL: 'reduce_i64'
; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V1 = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 3 for: %V4 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:9 SizeLat:8 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V8 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:10 SizeLat:9 for: %V16 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
Expand Down Expand Up @@ -104,23 +104,23 @@ define i32 @reduce_i32(i32 %arg) {
; AVX512F-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:10 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:11 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512BW-LABEL: 'reduce_i32'
; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:9 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:10 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512DQ-LABEL: 'reduce_i32'
; AVX512DQ-NEXT: Cost Model: Found costs of 2 for: %V2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 3 for: %V4 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 5 for: %V8 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:13 SizeLat:10 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of 6 for: %V16 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:10 CodeSize:10 Lat:14 SizeLat:11 for: %V32 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> undef)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
Expand Down Expand Up @@ -182,7 +182,7 @@ define i32 @reduce_i16(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V4 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V8 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 5 for: %V16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:15 SizeLat:11 for: %V32 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 7 for: %V32 = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:12 CodeSize:12 Lat:16 SizeLat:12 for: %V64 = call i16 @llvm.vector.reduce.add.v64i16(<64 x i16> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
Expand Down Expand Up @@ -260,7 +260,7 @@ define i32 @reduce_i8(i32 %arg) {
; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %V8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 3 for: %V16 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V32 = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:13 CodeSize:13 Lat:17 SizeLat:13 for: %V64 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %V64 = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:14 Lat:18 SizeLat:14 for: %V128 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> undef)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Analysis/CostModel/X86/reduce-fadd.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ define void @reduce_f64_fast(double %arg) {
; AVX512-NEXT: Cost Model: Found costs of 0 for: %V1 = call fast double @llvm.vector.reduce.fadd.v1f64(double %arg, <1 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of 2 for: %V2 = call fast double @llvm.vector.reduce.fadd.v2f64(double %arg, <2 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of 3 for: %V4 = call fast double @llvm.vector.reduce.fadd.v4f64(double %arg, <4 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:17 SizeLat:6 for: %V8 = call fast double @llvm.vector.reduce.fadd.v8f64(double %arg, <8 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of 4 for: %V8 = call fast double @llvm.vector.reduce.fadd.v8f64(double %arg, <8 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:7 CodeSize:7 Lat:21 SizeLat:7 for: %V16 = call fast double @llvm.vector.reduce.fadd.v16f64(double %arg, <16 x double> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
Expand Down Expand Up @@ -274,7 +274,7 @@ define void @reduce_f32_fast(float %arg) {
; AVX512-NEXT: Cost Model: Found costs of 2 for: %V2 = call fast float @llvm.vector.reduce.fadd.v2f32(float %arg, <2 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of 3 for: %V4 = call fast float @llvm.vector.reduce.fadd.v4f32(float %arg, <4 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of 4 for: %V8 = call fast float @llvm.vector.reduce.fadd.v8f32(float %arg, <8 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:8 CodeSize:8 Lat:24 SizeLat:8 for: %V16 = call fast float @llvm.vector.reduce.fadd.v16f32(float %arg, <16 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of 5 for: %V16 = call fast float @llvm.vector.reduce.fadd.v16f32(float %arg, <16 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:9 CodeSize:9 Lat:28 SizeLat:9 for: %V32 = call fast float @llvm.vector.reduce.fadd.v32f32(float %arg, <32 x float> undef)
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
Expand Down
45 changes: 12 additions & 33 deletions llvm/test/Transforms/PhaseOrdering/X86/horizontal-reduce-add.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX2
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v2 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v3 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=i686-- -mcpu=x86-64-v4 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
; RUN: opt < %s -passes="default<O2>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s

; PR37890 - subvector reduction followed by shuffle reduction

Expand Down Expand Up @@ -83,31 +83,10 @@ define i16 @PR37890_v16i16(<16 x i16> %a) {
}

define i32 @PR37890_v16i32(<16 x i32> %a) {
; SSE-LABEL: define i32 @PR37890_v16i32(
; SSE-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
; SSE-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
; SSE-NEXT: ret i32 [[E]]
;
; AVX2-LABEL: define i32 @PR37890_v16i32(
; AVX2-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
; AVX2-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
; AVX2-NEXT: ret i32 [[E]]
;
; AVX512-LABEL: define i32 @PR37890_v16i32(
; AVX512-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
; AVX512-NEXT: [[HI0:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; AVX512-NEXT: [[LO0:%.*]] = shufflevector <16 x i32> [[A]], <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; AVX512-NEXT: [[SUM0:%.*]] = add <8 x i32> [[LO0]], [[HI0]]
; AVX512-NEXT: [[HI1:%.*]] = shufflevector <8 x i32> [[SUM0]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
; AVX512-NEXT: [[LO1:%.*]] = shufflevector <8 x i32> [[SUM0]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; AVX512-NEXT: [[SUM1:%.*]] = add <4 x i32> [[LO1]], [[HI1]]
; AVX512-NEXT: [[HI2:%.*]] = shufflevector <4 x i32> [[SUM1]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
; AVX512-NEXT: [[LO2:%.*]] = shufflevector <4 x i32> [[SUM1]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
; AVX512-NEXT: [[SUM2:%.*]] = add <2 x i32> [[LO2]], [[HI2]]
; AVX512-NEXT: [[HI3:%.*]] = shufflevector <2 x i32> [[SUM2]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
; AVX512-NEXT: [[SUM3:%.*]] = add <2 x i32> [[HI3]], [[SUM2]]
; AVX512-NEXT: [[E:%.*]] = extractelement <2 x i32> [[SUM3]], i64 0
; AVX512-NEXT: ret i32 [[E]]
; CHECK-LABEL: define i32 @PR37890_v16i32(
; CHECK-SAME: <16 x i32> [[A:%.*]]) local_unnamed_addr #[[ATTR0]] {
; CHECK-NEXT: [[E:%.*]] = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> [[A]])
; CHECK-NEXT: ret i32 [[E]]
;
%hi0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%lo0 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
Expand Down
23 changes: 9 additions & 14 deletions llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,17 @@ declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32 immarg
define void @test(ptr nocapture readonly %arg, ptr nocapture readonly %arg1, ptr nocapture %arg2) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[GEP1_0:%.*]] = getelementptr inbounds double, ptr [[ARG:%.*]], i64 1
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x ptr> poison, ptr [[ARG:%.*]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x ptr> [[TMP0]], <8 x ptr> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, <8 x ptr> [[TMP1]], <8 x i64> <i64 1, i64 3, i64 5, i64 7, i64 9, i64 11, i64 13, i64 15>
; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1:%.*]], i64 16
; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[GEP2_0]], align 8
; CHECK-NEXT: [[GEP2_4:%.*]] = getelementptr inbounds double, ptr [[ARG1]], i64 20
; CHECK-NEXT: [[TMP1:%.*]] = call <15 x double> @llvm.masked.load.v15f64.p0(ptr align 8 [[GEP1_0]], <15 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <15 x double> poison)
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <15 x double> [[TMP1]], <15 x double> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP5]], <8 x i1> splat (i1 true), <8 x double> poison)
; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[GEP2_0]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <8 x double> [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP6:%.*]] = fmul fast <4 x double> [[TMP0]], [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
; CHECK-NEXT: [[TMP8:%.*]] = load <4 x double>, ptr [[GEP2_4]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <15 x double> [[TMP1]], <15 x double> poison, <4 x i32> <i32 8, i32 10, i32 12, i32 14>
; CHECK-NEXT: [[TMP10:%.*]] = fmul fast <4 x double> [[TMP8]], [[TMP9]]
; CHECK-NEXT: [[RDX_OP:%.*]] = fadd fast <4 x double> [[TMP6]], [[TMP10]]
; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[RDX_OP]])
; CHECK-NEXT: [[TMP6:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x double> [[TMP6]], [[TMP2]]
; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
; CHECK-NEXT: [[I142:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
; CHECK-NEXT: [[I143:%.*]] = insertelement <2 x double> [[I142]], double [[TMP11]], i64 1
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds double, ptr [[ARG2:%.*]], <2 x i64> <i64 0, i64 16>
Expand Down
21 changes: 8 additions & 13 deletions llvm/test/Transforms/VectorCombine/X86/reduction-of-truncations.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,19 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,X64
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,X64
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,X64
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX512
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
; RUN: opt < %s -S --passes=vector-combine -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s

;
; Fold reduce(trunc(X)) -> trunc(reduce(X)) if more cost efficient
;

; Cheap AVX512 v8i64 -> v8i32 truncation
define i32 @reduce_add_trunc_v8i64_i32(<8 x i64> %a0) {
; X64-LABEL: @reduce_add_trunc_v8i64_i32(
; X64-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
; X64-NEXT: [[RED:%.*]] = trunc i64 [[TMP1]] to i32
; X64-NEXT: ret i32 [[RED]]
;
; AVX512-LABEL: @reduce_add_trunc_v8i64_i32(
; AVX512-NEXT: [[TR:%.*]] = trunc <8 x i64> [[A0:%.*]] to <8 x i32>
; AVX512-NEXT: [[RED:%.*]] = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TR]])
; AVX512-NEXT: ret i32 [[RED]]
; CHECK-LABEL: @reduce_add_trunc_v8i64_i32(
; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> [[A0:%.*]])
; CHECK-NEXT: [[RED:%.*]] = trunc i64 [[TMP1]] to i32
; CHECK-NEXT: ret i32 [[RED]]
;
%tr = trunc <8 x i64> %a0 to <8 x i32>
%red = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %tr)
Expand Down
Loading