[AArch64] Optimize vector slide shuffles with zeros to use shift instructions

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -14583,6 +14583,119 @@ static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
   return true;
 }
 
+/// Flag slide shuffle patterns where one operand is zeros.
+/// Left slide: shufflevector %v, zeros, <1,2,3,...> -> ushr
+/// Right slide: shufflevector zeros, %v, <N-1,N,N+1,...> -> shl
+/// Check if a single 64-bit lane has a valid slide pattern.
+/// LaneStart: first element index of this lane in the full vector
+/// LaneElts: number of elements in the lane
+/// Returns slide amount in elements, or 0 if not a valid slide.
+static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
+                               unsigned LaneElts, unsigned NumElts,
+                               bool &IsLeftSlide) {
+  // Check for left slide: <k, k+1, ..., LaneElts-1, zero, ...>
+  // where k > 0 and elements stay within lane
+  int FirstIdx = Mask[LaneStart];
+  if (FirstIdx > (int)LaneStart && FirstIdx < (int)(LaneStart + LaneElts)) {
+    unsigned SlideAmt = FirstIdx - LaneStart;
+    for (unsigned i = 0; i < LaneElts; ++i) {
+      int MaskIdx = Mask[LaneStart + i];
+      if (MaskIdx < 0)
+        continue;
+      if (i < LaneElts - SlideAmt) {
+        // Data element: must be consecutive within lane
+        if (MaskIdx != (int)(LaneStart + SlideAmt + i))
+          return 0;
+      } else {
+        // Zero element: any index >= NumElts is fine (all from V2 which is
+        // zeros)
+        if (MaskIdx < (int)NumElts)
+          return 0;
+      }
+    }
+    IsLeftSlide = true;
+    return SlideAmt;
+  }
+
+  // Check for right slide: <zero, ..., 0, 1, ...>
+  // where zeros come first, then consecutive from lane start
+  if (Mask[LaneStart] >= (int)NumElts || Mask[LaneStart] < 0) {
+    unsigned ZeroCount = 0;
+    for (unsigned i = 0; i < LaneElts; ++i) {
+      int MaskIdx = Mask[LaneStart + i];
+      if (MaskIdx >= 0 && MaskIdx < (int)NumElts)
+        break;
+      ZeroCount++;
+    }
+    if (ZeroCount > 0 && ZeroCount < LaneElts) {
+      for (unsigned i = ZeroCount; i < LaneElts; ++i) {
+        int MaskIdx = Mask[LaneStart + i];
+        if (MaskIdx < 0)
+          continue;
+        if (MaskIdx != (int)(LaneStart + i - ZeroCount))
+          return 0;
+      }
+      IsLeftSlide = false;
+      return ZeroCount;
+    }
+  }
+
+  return 0;
+}
+
+static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
+                                    SDValue V2, unsigned &ShiftAmount,
+                                    bool &IsRightShift) {
+  unsigned VTSize = VT.getSizeInBits();
+  if (VTSize != 64 && VTSize != 128)
+    return SDValue();
+
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned EltSize = VT.getScalarSizeInBits();
+
+  bool V1IsZeros = ISD::isBuildVectorAllZeros(V1.getNode());
+  bool V2IsZeros = ISD::isBuildVectorAllZeros(V2.getNode());
+
+  // Exactly one operand must be zeros
+  if (V1IsZeros == V2IsZeros)
+    return SDValue();
+
+  // Canonicalize so V2 is zeros
+  SmallVector<int, 16> Mask(M.begin(), M.end());
+  SDValue DataVec = V1;
+  if (V1IsZeros) {
+    ShuffleVectorSDNode::commuteMask(Mask);
+    DataVec = V2;
+  }
+
+  // For 64-bit vectors, check single lane
+  // For 128-bit vectors, check both 64-bit lanes have same slide
+  unsigned LaneElts = 64 / EltSize;
+  unsigned NumLanes = VTSize / 64;
+
+  bool FirstIsLeftSlide;
+  unsigned FirstSlideAmt =
+      checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
+  if (FirstSlideAmt == 0)
+    return SDValue();
+
+  // For 128-bit, verify second lane matches
+  if (NumLanes == 2) {
+    bool SecondIsLeftSlide;
+    unsigned SecondSlideAmt =
+        checkLaneSlide(Mask, LaneElts, LaneElts, NumElts, SecondIsLeftSlide);
+    if (SecondSlideAmt != FirstSlideAmt ||
+        SecondIsLeftSlide != FirstIsLeftSlide)
+      return SDValue();
+  }
+
+  ShiftAmount = FirstSlideAmt * EltSize;
+  IsRightShift = FirstIsLeftSlide; // left slide = right shift in bits
+  if (ShiftAmount > 0 && ShiftAmount < 64)
+    return DataVec;
+  return SDValue();
+}
+
 // Check if an EXT instruction can handle the shuffle mask when one source is a
 // splat. This matches shuffles where the splat occupies either a prefix or a
 // suffix and the remaining lanes are a contiguous slice from the non-splat
@@ -15338,6 +15451,23 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
                        DAG.getConstant(8, DL, MVT::i32));
   }
 
+  // Check for slide-with-zeros pattern before EXT (slide is also valid EXT)
+  {
+    unsigned ShiftAmount;
+    bool IsRightShift;
+    if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
+                                               ShiftAmount, IsRightShift)) {
+      MVT ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+      SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
+
+      SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
+      unsigned Opc = IsRightShift ? AArch64ISD::VLSHR : AArch64ISD::VSHL;
+      SDValue Shifted = DAG.getNode(Opc, DL, ShiftVT, Vec, ShiftAmt);
+
+      return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Shifted);
+    }
+  }
+
   bool IsSplat1 =
       V1.getValueType() == VT && DAG.isSplatValue(V1, /*AllowUndefs=*/false);
   bool IsSplat2 =

llvm/test/CodeGen/AArch64/ext-narrow-index.ll

@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -global-isel -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-GISEL
 
@@ -218,12 +218,18 @@ entry:
 }
 
 define <8 x i8> @i8_zero_off15(<16 x i8> %arg1) {
-; CHECK-LABEL: i8_zero_off15:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #15
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: i8_zero_off15:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-SD-NEXT:    ushr d0, d0, #56
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GISEL-LABEL: i8_zero_off15:
+; CHECK-GISEL:       // %bb.0: // %entry
+; CHECK-GISEL-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT:    ext v0.16b, v0.16b, v1.16b, #15
+; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT:    ret
 entry:
   %shuffle = shufflevector <16 x i8> %arg1, <16 x i8> zeroinitializer, <8 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22>
   ret <8 x i8> %shuffle
@@ -269,12 +275,18 @@ entry:
 }
 
 define <4 x i16> @i16_zero_off7(<8 x i16> %arg1) {
-; CHECK-LABEL: i16_zero_off7:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: i16_zero_off7:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-SD-NEXT:    ushr d0, d0, #48
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GISEL-LABEL: i16_zero_off7:
+; CHECK-GISEL:       // %bb.0: // %entry
+; CHECK-GISEL-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT:    ext v0.16b, v0.16b, v1.16b, #14
+; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT:    ret
 entry:
   %shuffle = shufflevector <8 x i16> %arg1, <8 x i16> zeroinitializer, <4 x i32> <i32 7, i32 8, i32 9, i32 10>
   ret <4 x i16> %shuffle
@@ -313,12 +325,18 @@ entry:
 }
 
 define <2 x i32> @i32_zero_off3(<4 x i32> %arg1) {
-; CHECK-LABEL: i32_zero_off3:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: i32_zero_off3:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-SD-NEXT:    ushr d0, d0, #32
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GISEL-LABEL: i32_zero_off3:
+; CHECK-GISEL:       // %bb.0: // %entry
+; CHECK-GISEL-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
+; CHECK-GISEL-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT:    ret
 entry:
   %shuffle = shufflevector <4 x i32> %arg1, <4 x i32> zeroinitializer, <2 x i32> <i32 3, i32 4>
   ret <2 x i32> %shuffle

llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll

@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
 ; RUN: llc -o - %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
@@ -724,75 +724,69 @@ define void @uitofp_ld4_v32i16_to_v8f64(ptr nocapture noundef readonly %x, ptr n
 ; CHECK-NEXT:  Lloh33:
 ; CHECK-NEXT:    ldr q0, [x8, lCPI11_0@PAGEOFF]
 ; CHECK-NEXT:  Lloh34:
-; CHECK-NEXT:    adrp x8, lCPI11_3@PAGE
-; CHECK-NEXT:  Lloh35:
 ; CHECK-NEXT:    ldr q1, [x9, lCPI11_1@PAGEOFF]
-; CHECK-NEXT:  Lloh36:
+; CHECK-NEXT:  Lloh35:
 ; CHECK-NEXT:    ldr q2, [x10, lCPI11_2@PAGEOFF]
-; CHECK-NEXT:  Lloh37:
-; CHECK-NEXT:    ldr q3, [x8, lCPI11_3@PAGEOFF]
 ; CHECK-NEXT:    mov x8, xzr
 ; CHECK-NEXT:  LBB11_1: ; %vector.body
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add x9, x0, x8
-; CHECK-NEXT:    ldp q5, q4, [x9, #32]
-; CHECK-NEXT:    ldp q7, q6, [x9]
+; CHECK-NEXT:    ldp q3, q4, [x9, #32]
+; CHECK-NEXT:    ldp q5, q6, [x9]
 ; CHECK-NEXT:    add x9, x1, x8
 ; CHECK-NEXT:    add x8, x8, #64
-; CHECK-NEXT:    tbl.16b v16, { v4 }, v0
-; CHECK-NEXT:    tbl.16b v17, { v5 }, v0
+; CHECK-NEXT:    tbl.16b v7, { v4 }, v0
+; CHECK-NEXT:    tbl.16b v16, { v3 }, v0
 ; CHECK-NEXT:    tbl.16b v21, { v4 }, v1
-; CHECK-NEXT:    tbl.16b v18, { v6 }, v0
-; CHECK-NEXT:    tbl.16b v19, { v7 }, v0
-; CHECK-NEXT:    tbl.16b v20, { v7 }, v1
-; CHECK-NEXT:    tbl.16b v22, { v5 }, v1
-; CHECK-NEXT:    tbl.16b v23, { v5 }, v2
+; CHECK-NEXT:    tbl.16b v17, { v6 }, v0
+; CHECK-NEXT:    tbl.16b v18, { v5 }, v0
+; CHECK-NEXT:    tbl.16b v19, { v6 }, v1
+; CHECK-NEXT:    tbl.16b v20, { v5 }, v1
+; CHECK-NEXT:    tbl.16b v22, { v3 }, v1
+; CHECK-NEXT:    tbl.16b v23, { v3 }, v2
 ; CHECK-NEXT:    tbl.16b v24, { v4 }, v2
-; CHECK-NEXT:    tbl.16b v25, { v7 }, v2
-; CHECK-NEXT:    tbl.16b v5, { v5 }, v3
-; CHECK-NEXT:    tbl.16b v4, { v4 }, v3
-; CHECK-NEXT:    tbl.16b v7, { v7 }, v3
-; CHECK-NEXT:    tbl.16b v26, { v6 }, v1
-; CHECK-NEXT:    tbl.16b v27, { v6 }, v2
-; CHECK-NEXT:    tbl.16b v6, { v6 }, v3
-; CHECK-NEXT:    ucvtf.2d v17, v17
+; CHECK-NEXT:    tbl.16b v25, { v5 }, v2
+; CHECK-NEXT:    tbl.16b v26, { v6 }, v2
+; CHECK-NEXT:    ushr.2d v3, v3, #48
+; CHECK-NEXT:    ushr.2d v4, v4, #48
+; CHECK-NEXT:    ushr.2d v5, v5, #48
+; CHECK-NEXT:    ushr.2d v6, v6, #48
 ; CHECK-NEXT:    ucvtf.2d v16, v16
-; CHECK-NEXT:    ucvtf.2d v19, v19
+; CHECK-NEXT:    ucvtf.2d v7, v7
 ; CHECK-NEXT:    ucvtf.2d v18, v18
+; CHECK-NEXT:    ucvtf.2d v17, v17
 ; CHECK-NEXT:    ucvtf.2d v22, v22
 ; CHECK-NEXT:    ucvtf.2d v23, v23
-; CHECK-NEXT:    ucvtf.2d v5, v5
+; CHECK-NEXT:    ucvtf.2d v3, v3
 ; CHECK-NEXT:    ucvtf.2d v21, v21
 ; CHECK-NEXT:    ucvtf.2d v24, v24
 ; CHECK-NEXT:    ucvtf.2d v4, v4
 ; CHECK-NEXT:    cmp x8, #2, lsl #12 ; =8192
 ; CHECK-NEXT:    ucvtf.2d v20, v20
 ; CHECK-NEXT:    ucvtf.2d v25, v25
-; CHECK-NEXT:    ucvtf.2d v7, v7
+; CHECK-NEXT:    ucvtf.2d v5, v5
+; CHECK-NEXT:    ucvtf.2d v19, v19
 ; CHECK-NEXT:    ucvtf.2d v26, v26
-; CHECK-NEXT:    ucvtf.2d v27, v27
 ; CHECK-NEXT:    ucvtf.2d v6, v6
-; CHECK-NEXT:    fadd.2d v17, v22, v17
-; CHECK-NEXT:    fadd.2d v5, v23, v5
-; CHECK-NEXT:    fadd.2d v16, v21, v16
+; CHECK-NEXT:    fadd.2d v16, v22, v16
+; CHECK-NEXT:    fadd.2d v3, v23, v3
+; CHECK-NEXT:    fadd.2d v7, v21, v7
 ; CHECK-NEXT:    fadd.2d v4, v24, v4
-; CHECK-NEXT:    fadd.2d v19, v20, v19
-; CHECK-NEXT:    fadd.2d v7, v25, v7
-; CHECK-NEXT:    fadd.2d v18, v26, v18
-; CHECK-NEXT:    fadd.2d v6, v27, v6
-; CHECK-NEXT:    fadd.2d v5, v17, v5
-; CHECK-NEXT:    fadd.2d v4, v16, v4
-; CHECK-NEXT:    fadd.2d v7, v19, v7
-; CHECK-NEXT:    fadd.2d v6, v18, v6
-; CHECK-NEXT:    stp q5, q4, [x9, #32]
-; CHECK-NEXT:    stp q7, q6, [x9]
+; CHECK-NEXT:    fadd.2d v18, v20, v18
+; CHECK-NEXT:    fadd.2d v5, v25, v5
+; CHECK-NEXT:    fadd.2d v17, v19, v17
+; CHECK-NEXT:    fadd.2d v6, v26, v6
+; CHECK-NEXT:    fadd.2d v3, v16, v3
+; CHECK-NEXT:    fadd.2d v4, v7, v4
+; CHECK-NEXT:    fadd.2d v5, v18, v5
+; CHECK-NEXT:    fadd.2d v6, v17, v6
+; CHECK-NEXT:    stp q3, q4, [x9, #32]
+; CHECK-NEXT:    stp q5, q6, [x9]
 ; CHECK-NEXT:    b.ne LBB11_1
 ; CHECK-NEXT:  ; %bb.2: ; %for.cond.cleanup
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:    .loh AdrpLdr Lloh34, Lloh37
-; CHECK-NEXT:    .loh AdrpLdr Lloh32, Lloh36
-; CHECK-NEXT:    .loh AdrpLdr Lloh31, Lloh35
-; CHECK-NEXT:    .loh AdrpAdrp Lloh30, Lloh34
+; CHECK-NEXT:    .loh AdrpLdr Lloh32, Lloh35
+; CHECK-NEXT:    .loh AdrpLdr Lloh31, Lloh34
 ; CHECK-NEXT:    .loh AdrpLdr Lloh30, Lloh33
 entry:
   br label %vector.body