[AArch64] Generate ADDP from shuffled add

[dmgreen]

Summary

This adds a fold of add(x, shuffle(x, <1,0,3,2,5,4,...>), into shuffle(addp(x), <0,0,1,1,2,2,..>. The ADDP instruction takes two vectors and returns one, adding adjacent pairs. So we match x in a custom combine as it is lowered from a v8i32. The original code would be 2 rev64 and 2 add, with the new code being a single addp with a zip1;zip2 shuffle, producing smaller code.

[ktkachov]

Doesn't have to be part of this patch, but can this be extended to FADDP as well?

[dmgreen]

In D126686#3547216, @ktkachov wrote:

Doesn't have to be part of this patch, but can this be extended to FADDP as well?

Yeah sure that sounds good, but in a separate patch. I was attempting to get sub to work in the same way too, using a mul <0,-1>, but it's much harder for it to be profitable without using demanded elements.

[samtebbs]

LGTM! Thanks

llvm/lib/Target/AArch64/AArch64ISelLowering.h

enum NodeType : unsigned {
   FCMLEz,
   FCMLTz,
 
   // Vector across-lanes addition
   // Only the lower result lane is defined.
   SADDV,
   UADDV,
 
+  // Add Pairwise of two vectors
+  ADDP,
   // Add Long Pairwise
   SADDLP,
   UADDLP,
 
   // udot/sdot instructions
   UDOT,
   SDOT,
 

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasNEON()) {
     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
 
     setLoadExtAction(ISD::EXTLOAD,  MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::EXTLOAD,  MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
+
+    // ADDP custom lowering
+    for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
+      setOperationAction(ISD::ADD, VT, Custom);
   }
 
   if (Subtarget->hasSVE()) {
     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
       setOperationAction(ISD::BITREVERSE, VT, Custom);
       setOperationAction(ISD::BSWAP, VT, Custom);
       setOperationAction(ISD::CTLZ, VT, Custom);
       setOperationAction(ISD::CTPOP, VT, Custom);
case AArch64ISD::FIRST_NUMBER:
     MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::REVH_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::REVW_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
+    MAKE_CASE(AArch64ISD::ADDP)
     MAKE_CASE(AArch64ISD::SADDLP)
     MAKE_CASE(AArch64ISD::UADDLP)
     MAKE_CASE(AArch64ISD::CALL_RVMARKER)
     MAKE_CASE(AArch64ISD::ASSERT_ZEXT_BOOL)
     MAKE_CASE(AArch64ISD::MOPS_MEMSET)
     MAKE_CASE(AArch64ISD::MOPS_MEMSET_TAGGING)
     MAKE_CASE(AArch64ISD::MOPS_MEMCOPY)
     MAKE_CASE(AArch64ISD::MOPS_MEMMOVE)
Op = SDValue(
       DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
                          DAG.getUNDEF(MVT::i32), Op,
                          DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
       0);
   Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
   Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
 }
 
+static void ReplaceAddWithADDP(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                               SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+  if (!VT.is256BitVector())
+    return;
+
+  SDValue X = N->getOperand(0);
+  auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(1));
+  if (!Shuf) {
+    Shuf = dyn_cast<ShuffleVectorSDNode>(N->getOperand(0));
+    X = N->getOperand(1);
+    if (!Shuf)
+      return;
+  }
+
+  if (Shuf->getOperand(0) != X || !Shuf->getOperand(1)->isUndef())
+    return;
+
+  // Check the mask is 1,0,3,2,5,4,...
+  ArrayRef<int> Mask = Shuf->getMask();
+  for (int I = 0, E = Mask.size(); I < E; I++)
+    if (Mask[I] != (I % 2 == 0 ? I + 1 : I - 1))
+      return;
+
+  SDLoc DL(N);
+  auto LoHi = DAG.SplitVector(X, DL);
+  assert(LoHi.first.getValueType() == LoHi.second.getValueType());
+  SDValue Addp = DAG.getNode(AArch64ISD::ADDP, N, LoHi.first.getValueType(),
+                             LoHi.first, LoHi.second);
+
+  // Shuffle the elements back into order.
+  SmallVector<int> NMask;
+  for (unsigned I = 0, E = VT.getVectorNumElements() / 2; I < E; I++) {
+    NMask.push_back(I);
+    NMask.push_back(I);
+  }
+  Results.push_back(
+      DAG.getVectorShuffle(VT, DL,
+                           DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Addp,
+                                       DAG.getUNDEF(LoHi.first.getValueType())),
+                           DAG.getUNDEF(VT), NMask));
+}
+
 static void ReplaceReductionResults(SDNode *N,
                                     SmallVectorImpl<SDValue> &Results,
                                     SelectionDAG &DAG, unsigned InterOp,
                                     unsigned AcrossOp) {
   EVT LoVT, HiVT;
   SDValue Lo, Hi;
   SDLoc dl(N);
   std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
case ISD::BITCAST:
     return;
   case ISD::VECREDUCE_ADD:
   case ISD::VECREDUCE_SMAX:
   case ISD::VECREDUCE_SMIN:
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_UMIN:
     Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
     return;
+  case ISD::ADD:
+    ReplaceAddWithADDP(N, Results, DAG);
+    return;
 
   case ISD::CTPOP:
     if (SDValue Result = LowerCTPOP(SDValue(N, 0), DAG))
       Results.push_back(Result);
     return;
   case AArch64ISD::SADDV:
     ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV);
     return;

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 
 def AArch64uabd     : PatFrags<(ops node:$lhs, node:$rhs),
                                [(abdu node:$lhs, node:$rhs),
                                 (int_aarch64_neon_uabd node:$lhs, node:$rhs)]>;
 def AArch64sabd     : PatFrags<(ops node:$lhs, node:$rhs),
                                [(abds node:$lhs, node:$rhs),
                                 (int_aarch64_neon_sabd node:$lhs, node:$rhs)]>;
 
+def AArch64addp_n   : SDNode<"AArch64ISD::ADDP", SDT_AArch64Zip>;
 def AArch64uaddlp_n : SDNode<"AArch64ISD::UADDLP", SDT_AArch64uaddlp>;
 def AArch64saddlp_n : SDNode<"AArch64ISD::SADDLP", SDT_AArch64uaddlp>;
+def AArch64addp     : PatFrags<(ops node:$Rn, node:$Rm),
+                               [(AArch64addp_n node:$Rn, node:$Rm),
+                                (int_aarch64_neon_addp node:$Rn, node:$Rm)]>;
 def AArch64uaddlp   : PatFrags<(ops node:$src),
                                [(AArch64uaddlp_n node:$src),
                                 (int_aarch64_neon_uaddlp node:$src)]>;
 def AArch64saddlp   : PatFrags<(ops node:$src),
                                [(AArch64saddlp_n node:$src),
                                 (int_aarch64_neon_saddlp node:$src)]>;
 
 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
def : Pat<(v8i16 (concat_vectors
                                            (v4i32 VImm8000)))))),
           (SQXTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Vd, dsub), V128:$Vn)>;
 
 //===----------------------------------------------------------------------===//
 // Advanced SIMD three vector instructions.
 //===----------------------------------------------------------------------===//
 
 defm ADD     : SIMDThreeSameVector<0, 0b10000, "add", add>;
-defm ADDP    : SIMDThreeSameVector<0, 0b10111, "addp", int_aarch64_neon_addp>;
+defm ADDP    : SIMDThreeSameVector<0, 0b10111, "addp", AArch64addp>;
 defm CMEQ    : SIMDThreeSameVector<1, 0b10001, "cmeq", AArch64cmeq>;
 defm CMGE    : SIMDThreeSameVector<0, 0b00111, "cmge", AArch64cmge>;
 defm CMGT    : SIMDThreeSameVector<0, 0b00110, "cmgt", AArch64cmgt>;
 defm CMHI    : SIMDThreeSameVector<1, 0b00110, "cmhi", AArch64cmhi>;
 defm CMHS    : SIMDThreeSameVector<1, 0b00111, "cmhs", AArch64cmhs>;
 defm CMTST   : SIMDThreeSameVector<0, 0b10001, "cmtst", AArch64cmtst>;
 foreach VT = [ v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64 ] in {
 def : Pat<(vnot (AArch64cmeqz VT:$Rn)), (!cast<Instruction>("CMTST"#VT) VT:$Rn, VT:$Rn)>;

llvm/test/CodeGen/AArch64/arm64-addp.ll

entry:
   %s = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
   %b = add <2 x i64> %s, %a
   ret <2 x i64> %b
 }
 
 define <4 x i64> @addp_v4i64(<4 x i64> %a) {
 ; CHECK-LABEL: addp_v4i64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ext.16b v2, v1, v1, #8
-; CHECK-NEXT:    ext.16b v3, v0, v0, #8
-; CHECK-NEXT:    add.2d v0, v3, v0
-; CHECK-NEXT:    add.2d v1, v2, v1
+; CHECK-NEXT:    addp.2d v1, v0, v1
+; CHECK-NEXT:    dup.2d v0, v1[0]
+; CHECK-NEXT:    dup.2d v1, v1[1]
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   %b = add <4 x i64> %s, %a
   ret <4 x i64> %b
 }
 
 define <4 x i32> @addp_v4i32(<4 x i32> %a) {
 ; CHECK-LABEL: addp_v4i32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev64.4s v1, v0
 ; CHECK-NEXT:    add.4s v0, v1, v0
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   %b = add <4 x i32> %s, %a
   ret <4 x i32> %b
 }
 
 define <8 x i32> @addp_v8i32(<8 x i32> %a) {
 ; CHECK-LABEL: addp_v8i32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64.4s v2, v1
-; CHECK-NEXT:    rev64.4s v3, v0
-; CHECK-NEXT:    add.4s v0, v3, v0
-; CHECK-NEXT:    add.4s v1, v2, v1
+; CHECK-NEXT:    addp.4s v1, v0, v1
+; CHECK-NEXT:    zip1.4s v0, v1, v1
+; CHECK-NEXT:    zip2.4s v1, v1, v1
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %b = add <8 x i32> %s, %a
   ret <8 x i32> %b
 }
 
 define <16 x i32> @addp_v16i32(<16 x i32> %a) {
 ; CHECK-LABEL: addp_v16i32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev64.4s v4, v3
-; CHECK-NEXT:    rev64.4s v5, v2
-; CHECK-NEXT:    rev64.4s v6, v1
-; CHECK-NEXT:    rev64.4s v7, v0
-; CHECK-NEXT:    add.4s v0, v7, v0
-; CHECK-NEXT:    add.4s v1, v6, v1
-; CHECK-NEXT:    add.4s v2, v5, v2
-; CHECK-NEXT:    add.4s v3, v4, v3
+; CHECK-NEXT:    addp.4s v1, v0, v1
+; CHECK-NEXT:    zip1.4s v0, v1, v1
+; CHECK-NEXT:    zip2.4s v1, v1, v1
+; CHECK-NEXT:    addp.4s v3, v2, v3
+; CHECK-NEXT:    zip1.4s v2, v3, v3
+; CHECK-NEXT:    zip2.4s v3, v3, v3
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   %b = add <16 x i32> %s, %a
   ret <16 x i32> %b
 }
 
 define <8 x i16> @addp_v8i16(<8 x i16> %a) {
 ; CHECK-LABEL: addp_v8i16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev32.8h v1, v0
 ; CHECK-NEXT:    add.8h v0, v1, v0
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
   %b = add <8 x i16> %s, %a
   ret <8 x i16> %b
 }
 
 define <16 x i16> @addp_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: addp_v16i16:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev32.8h v2, v1
-; CHECK-NEXT:    rev32.8h v3, v0
-; CHECK-NEXT:    add.8h v0, v3, v0
-; CHECK-NEXT:    add.8h v1, v2, v1
+; CHECK-NEXT:    addp.8h v1, v0, v1
+; CHECK-NEXT:    zip1.8h v0, v1, v1
+; CHECK-NEXT:    zip2.8h v1, v1, v1
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   %b = add <16 x i16> %s, %a
   ret <16 x i16> %b
 }
 
 define <16 x i8> @addp_v16i8(<16 x i8> %a) {
 ; CHECK-LABEL: addp_v16i8:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    rev16.16b v1, v0
 ; CHECK-NEXT:    add.16b v0, v1, v0
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
   %b = add <16 x i8> %s, %a
   ret <16 x i8> %b
 }
 
 define <32 x i8> @addp_v32i8(<32 x i8> %a) {
 ; CHECK-LABEL: addp_v32i8:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rev16.16b v2, v1
-; CHECK-NEXT:    rev16.16b v3, v0
-; CHECK-NEXT:    add.16b v0, v3, v0
-; CHECK-NEXT:    add.16b v1, v2, v1
+; CHECK-NEXT:    addp.16b v1, v0, v1
+; CHECK-NEXT:    zip1.16b v0, v1, v1
+; CHECK-NEXT:    zip2.16b v1, v1, v1
 ; CHECK-NEXT:    ret
 entry:
   %s = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30>
   %b = add <32 x i8> %s, %a
   ret <32 x i8> %b
 }

llvm/test/CodeGen/AArch64/insert-extend.ll

 ; CHECK-NEXT:    usubl v4.4s, v4.4h, v6.4h
 ; CHECK-NEXT:    shl v6.4s, v7.4s, #16
 ; CHECK-NEXT:    shl v5.4s, v5.4s, #16
 ; CHECK-NEXT:    shl v3.4s, v3.4s, #16
 ; CHECK-NEXT:    shl v4.4s, v4.4s, #16
 ; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
 ; CHECK-NEXT:    add v2.4s, v4.4s, v2.4s
 ; CHECK-NEXT:    add v0.4s, v6.4s, v0.4s
-; CHECK-NEXT:    rev64 v6.4s, v1.4s
-; CHECK-NEXT:    rev64 v7.4s, v2.4s
+; CHECK-NEXT:    rev64 v6.4s, v2.4s
+; CHECK-NEXT:    rev64 v17.4s, v1.4s
 ; CHECK-NEXT:    add v3.4s, v3.4s, v16.4s
-; CHECK-NEXT:    rev64 v4.4s, v0.4s
-; CHECK-NEXT:    rev64 v5.4s, v3.4s
-; CHECK-NEXT:    add v18.4s, v1.4s, v6.4s
-; CHECK-NEXT:    add v19.4s, v2.4s, v7.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v7.4s
-; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
-; CHECK-NEXT:    zip1 v7.4s, v2.4s, v1.4s
-; CHECK-NEXT:    add v17.4s, v3.4s, v5.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
-; CHECK-NEXT:    uzp2 v6.4s, v17.4s, v16.4s
-; CHECK-NEXT:    zip2 v5.4s, v0.4s, v3.4s
-; CHECK-NEXT:    ext v20.16b, v17.16b, v17.16b, #12
-; CHECK-NEXT:    mov v0.s[1], v3.s[0]
-; CHECK-NEXT:    ext v3.16b, v2.16b, v7.16b, #8
-; CHECK-NEXT:    mov v2.s[3], v1.s[2]
-; CHECK-NEXT:    zip1 v4.4s, v19.4s, v18.4s
-; CHECK-NEXT:    trn2 v21.4s, v17.4s, v16.4s
-; CHECK-NEXT:    uzp2 v6.4s, v6.4s, v17.4s
-; CHECK-NEXT:    mov v17.s[0], v16.s[1]
-; CHECK-NEXT:    zip2 v7.4s, v19.4s, v18.4s
-; CHECK-NEXT:    mov v0.d[1], v3.d[1]
-; CHECK-NEXT:    ext v1.16b, v16.16b, v20.16b, #12
-; CHECK-NEXT:    mov v5.d[1], v2.d[1]
-; CHECK-NEXT:    mov v17.d[1], v4.d[1]
-; CHECK-NEXT:    mov v6.d[1], v7.d[1]
-; CHECK-NEXT:    mov v1.d[1], v7.d[1]
-; CHECK-NEXT:    add v3.4s, v5.4s, v0.4s
-; CHECK-NEXT:    mov v21.d[1], v4.d[1]
+; CHECK-NEXT:    rev64 v5.4s, v0.4s
 ; CHECK-NEXT:    rev64 v4.4s, v3.4s
-; CHECK-NEXT:    sub v1.4s, v17.4s, v1.4s
+; CHECK-NEXT:    addp v16.4s, v2.4s, v1.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v17.4s
+; CHECK-NEXT:    sub v2.4s, v2.4s, v6.4s
+; CHECK-NEXT:    addp v7.4s, v3.4s, v0.4s
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v2.4s, v6.4s, v21.4s
-; CHECK-NEXT:    rev64 v6.4s, v1.4s
-; CHECK-NEXT:    add v7.4s, v3.4s, v4.4s
+; CHECK-NEXT:    zip1 v5.4s, v2.4s, v1.4s
 ; CHECK-NEXT:    sub v3.4s, v3.4s, v4.4s
-; CHECK-NEXT:    rev64 v4.4s, v0.4s
-; CHECK-NEXT:    rev64 v5.4s, v2.4s
-; CHECK-NEXT:    add v17.4s, v1.4s, v6.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
-; CHECK-NEXT:    add v19.4s, v0.4s, v4.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    ext v16.16b, v7.16b, v3.16b, #4
-; CHECK-NEXT:    add v18.4s, v2.4s, v5.4s
-; CHECK-NEXT:    ext v6.16b, v17.16b, v1.16b, #4
-; CHECK-NEXT:    sub v2.4s, v2.4s, v5.4s
-; CHECK-NEXT:    ext v5.16b, v19.16b, v0.16b, #4
-; CHECK-NEXT:    rev64 v16.4s, v16.4s
-; CHECK-NEXT:    rev64 v6.4s, v6.4s
-; CHECK-NEXT:    ext v20.16b, v18.16b, v18.16b, #4
-; CHECK-NEXT:    rev64 v5.4s, v5.4s
-; CHECK-NEXT:    mov v7.s[3], v3.s[3]
-; CHECK-NEXT:    ext v4.16b, v3.16b, v16.16b, #12
-; CHECK-NEXT:    mov v19.s[3], v0.s[3]
-; CHECK-NEXT:    mov v17.s[3], v1.s[3]
-; CHECK-NEXT:    ext v6.16b, v1.16b, v6.16b, #12
+; CHECK-NEXT:    ext v18.16b, v7.16b, v7.16b, #8
+; CHECK-NEXT:    zip2 v4.4s, v0.4s, v3.4s
+; CHECK-NEXT:    mov v0.s[1], v3.s[0]
+; CHECK-NEXT:    ext v3.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT:    mov v2.s[3], v1.s[2]
+; CHECK-NEXT:    uzp2 v19.4s, v7.4s, v16.4s
+; CHECK-NEXT:    uzp1 v6.4s, v7.4s, v16.4s
+; CHECK-NEXT:    uzp1 v7.4s, v18.4s, v16.4s
+; CHECK-NEXT:    uzp2 v1.4s, v18.4s, v16.4s
+; CHECK-NEXT:    mov v0.d[1], v3.d[1]
+; CHECK-NEXT:    mov v4.d[1], v2.d[1]
+; CHECK-NEXT:    add v5.4s, v19.4s, v6.4s
+; CHECK-NEXT:    sub v1.4s, v7.4s, v1.4s
+; CHECK-NEXT:    rev64 v2.4s, v5.4s
+; CHECK-NEXT:    sub v6.4s, v0.4s, v4.4s
+; CHECK-NEXT:    add v0.4s, v4.4s, v0.4s
+; CHECK-NEXT:    rev64 v3.4s, v1.4s
+; CHECK-NEXT:    rev64 v4.4s, v6.4s
+; CHECK-NEXT:    rev64 v7.4s, v0.4s
+; CHECK-NEXT:    addp v16.4s, v1.4s, v6.4s
+; CHECK-NEXT:    addp v17.4s, v5.4s, v0.4s
+; CHECK-NEXT:    sub v4.4s, v6.4s, v4.4s
+; CHECK-NEXT:    sub v0.4s, v0.4s, v7.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    sub v2.4s, v5.4s, v2.4s
+; CHECK-NEXT:    ext v3.16b, v1.16b, v16.16b, #8
+; CHECK-NEXT:    ext v5.16b, v17.16b, v0.16b, #4
+; CHECK-NEXT:    ext v6.16b, v16.16b, v4.16b, #4
+; CHECK-NEXT:    zip1 v7.4s, v17.4s, v17.4s
+; CHECK-NEXT:    ext v18.16b, v3.16b, v1.16b, #4
+; CHECK-NEXT:    zip2 v5.4s, v5.4s, v17.4s
+; CHECK-NEXT:    zip2 v6.4s, v6.4s, v16.4s
+; CHECK-NEXT:    trn2 v7.4s, v7.4s, v2.4s
+; CHECK-NEXT:    ext v2.16b, v2.16b, v17.16b, #4
+; CHECK-NEXT:    mov v1.s[2], v16.s[1]
 ; CHECK-NEXT:    ext v5.16b, v0.16b, v5.16b, #12
-; CHECK-NEXT:    rev64 v18.4s, v18.4s
-; CHECK-NEXT:    trn2 v20.4s, v2.4s, v20.4s
-; CHECK-NEXT:    sub v16.4s, v7.4s, v4.4s
-; CHECK-NEXT:    sub v21.4s, v17.4s, v6.4s
-; CHECK-NEXT:    sub v22.4s, v19.4s, v5.4s
-; CHECK-NEXT:    trn2 v2.4s, v18.4s, v2.4s
-; CHECK-NEXT:    mov v17.s[0], v1.s[0]
-; CHECK-NEXT:    ext v1.16b, v20.16b, v20.16b, #4
-; CHECK-NEXT:    mov v19.s[0], v0.s[0]
-; CHECK-NEXT:    mov v7.s[0], v3.s[0]
-; CHECK-NEXT:    add v0.4s, v17.4s, v6.4s
-; CHECK-NEXT:    add v3.4s, v2.4s, v1.4s
-; CHECK-NEXT:    add v5.4s, v19.4s, v5.4s
-; CHECK-NEXT:    add v4.4s, v7.4s, v4.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    mov v4.d[1], v16.d[1]
-; CHECK-NEXT:    mov v5.d[1], v22.d[1]
-; CHECK-NEXT:    mov v0.d[1], v21.d[1]
-; CHECK-NEXT:    mov v3.d[1], v1.d[1]
-; CHECK-NEXT:    movi v1.8h, #1
+; CHECK-NEXT:    ext v6.16b, v4.16b, v6.16b, #12
+; CHECK-NEXT:    uzp2 v3.4s, v3.4s, v18.4s
+; CHECK-NEXT:    mov v4.s[2], v16.s[3]
+; CHECK-NEXT:    mov v0.s[2], v17.s[3]
+; CHECK-NEXT:    ext v2.16b, v2.16b, v2.16b, #4
+; CHECK-NEXT:    sub v18.4s, v1.4s, v3.4s
+; CHECK-NEXT:    sub v19.4s, v4.4s, v6.4s
+; CHECK-NEXT:    sub v20.4s, v0.4s, v5.4s
+; CHECK-NEXT:    sub v21.4s, v7.4s, v2.4s
+; CHECK-NEXT:    mov v4.s[1], v16.s[2]
+; CHECK-NEXT:    mov v0.s[1], v17.s[2]
+; CHECK-NEXT:    mov v2.s[0], v17.s[1]
+; CHECK-NEXT:    mov v1.s[1], v16.s[0]
+; CHECK-NEXT:    add v4.4s, v4.4s, v6.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    add v2.4s, v7.4s, v2.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    mov v2.d[1], v21.d[1]
+; CHECK-NEXT:    mov v1.d[1], v18.d[1]
+; CHECK-NEXT:    mov v4.d[1], v19.d[1]
+; CHECK-NEXT:    mov v0.d[1], v20.d[1]
+; CHECK-NEXT:    movi v3.8h, #1
 ; CHECK-NEXT:    movi v17.2d, #0x00ffff0000ffff
-; CHECK-NEXT:    ushr v2.4s, v0.4s, #15
-; CHECK-NEXT:    ushr v6.4s, v4.4s, #15
-; CHECK-NEXT:    ushr v7.4s, v3.4s, #15
-; CHECK-NEXT:    ushr v16.4s, v5.4s, #15
-; CHECK-NEXT:    and v6.16b, v6.16b, v1.16b
-; CHECK-NEXT:    and v16.16b, v16.16b, v1.16b
-; CHECK-NEXT:    and v7.16b, v7.16b, v1.16b
-; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    mul v2.4s, v6.4s, v17.4s
+; CHECK-NEXT:    ushr v5.4s, v4.4s, #15
+; CHECK-NEXT:    ushr v6.4s, v2.4s, #15
+; CHECK-NEXT:    ushr v7.4s, v0.4s, #15
+; CHECK-NEXT:    ushr v16.4s, v1.4s, #15
+; CHECK-NEXT:    and v6.16b, v6.16b, v3.16b
+; CHECK-NEXT:    and v16.16b, v16.16b, v3.16b
+; CHECK-NEXT:    and v7.16b, v7.16b, v3.16b
+; CHECK-NEXT:    and v3.16b, v5.16b, v3.16b
+; CHECK-NEXT:    mul v5.4s, v6.4s, v17.4s
 ; CHECK-NEXT:    mul v6.4s, v16.4s, v17.4s
-; CHECK-NEXT:    mul v1.4s, v1.4s, v17.4s
+; CHECK-NEXT:    mul v3.4s, v3.4s, v17.4s
 ; CHECK-NEXT:    mul v7.4s, v7.4s, v17.4s
-; CHECK-NEXT:    add v4.4s, v2.4s, v4.4s
-; CHECK-NEXT:    add v5.4s, v6.4s, v5.4s
-; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v3.4s, v7.4s, v3.4s
-; CHECK-NEXT:    eor v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    eor v1.16b, v3.16b, v7.16b
-; CHECK-NEXT:    eor v3.16b, v5.16b, v6.16b
-; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    add v2.4s, v2.4s, v3.4s
+; CHECK-NEXT:    add v2.4s, v5.4s, v2.4s
+; CHECK-NEXT:    add v1.4s, v6.4s, v1.4s
+; CHECK-NEXT:    add v4.4s, v3.4s, v4.4s
+; CHECK-NEXT:    add v0.4s, v7.4s, v0.4s
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    eor v0.16b, v0.16b, v7.16b
+; CHECK-NEXT:    eor v1.16b, v1.16b, v6.16b
+; CHECK-NEXT:    eor v2.16b, v2.16b, v5.16b
+; CHECK-NEXT:    add v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v0.4s, v3.4s
 ; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    lsr w9, w8, #16
 ; CHECK-NEXT:    add w8, w9, w8, uxth
 ; CHECK-NEXT:    lsr w0, w8, #1
 ; CHECK-NEXT:    ret
 entry:
   %idx.ext = sext i32 %st1 to i64

llvm/test/CodeGen/AArch64/reduce-shuffle.ll

 ; CHECK-NEXT:    usubl v16.4s, v1.4h, v5.4h
 ; CHECK-NEXT:    ld1 { v4.s }[1], [x12]
 ; CHECK-NEXT:    ld1 { v7.s }[1], [x2], #4
 ; CHECK-NEXT:    usubl2 v1.4s, v1.8h, v5.8h
 ; CHECK-NEXT:    ushll v5.8h, v7.8b, #0
 ; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
 ; CHECK-NEXT:    ld1 { v6.s }[1], [x2]
 ; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    usubl v7.4s, v3.4h, v5.4h
-; CHECK-NEXT:    usubl2 v3.4s, v3.8h, v5.8h
-; CHECK-NEXT:    usubl2 v5.4s, v2.8h, v4.8h
+; CHECK-NEXT:    usubl v17.4s, v3.4h, v5.4h
+; CHECK-NEXT:    usubl2 v5.4s, v3.8h, v5.8h
+; CHECK-NEXT:    usubl2 v3.4s, v2.8h, v4.8h
 ; CHECK-NEXT:    usubl v2.4s, v2.4h, v4.4h
 ; CHECK-NEXT:    ushll v4.8h, v6.8b, #0
-; CHECK-NEXT:    shl v5.4s, v5.4s, #16
+; CHECK-NEXT:    shl v3.4s, v3.4s, #16
 ; CHECK-NEXT:    usubl2 v6.4s, v0.8h, v4.8h
 ; CHECK-NEXT:    shl v2.4s, v2.4s, #16
 ; CHECK-NEXT:    usubl v0.4s, v0.4h, v4.4h
-; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
-; CHECK-NEXT:    shl v4.4s, v6.4s, #16
-; CHECK-NEXT:    shl v0.4s, v0.4s, #16
-; CHECK-NEXT:    add v2.4s, v2.4s, v16.4s
-; CHECK-NEXT:    add v3.4s, v4.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v7.4s
-; CHECK-NEXT:    uzp2 v6.4s, v2.4s, v1.4s
-; CHECK-NEXT:    ext v17.16b, v2.16b, v2.16b, #12
-; CHECK-NEXT:    zip1 v4.4s, v0.4s, v3.4s
-; CHECK-NEXT:    mov v16.16b, v2.16b
-; CHECK-NEXT:    mov v19.16b, v1.16b
-; CHECK-NEXT:    zip2 v5.4s, v1.4s, v2.4s
-; CHECK-NEXT:    zip2 v18.4s, v2.4s, v1.4s
-; CHECK-NEXT:    mov v16.s[0], v1.s[1]
-; CHECK-NEXT:    uzp2 v6.4s, v6.4s, v2.4s
-; CHECK-NEXT:    zip2 v7.4s, v0.4s, v3.4s
-; CHECK-NEXT:    ext v1.16b, v1.16b, v17.16b, #12
-; CHECK-NEXT:    ext v17.16b, v0.16b, v4.16b, #8
-; CHECK-NEXT:    mov v19.s[1], v2.s[0]
-; CHECK-NEXT:    mov v0.s[3], v3.s[2]
-; CHECK-NEXT:    mov v6.d[1], v7.d[1]
-; CHECK-NEXT:    mov v16.d[1], v4.d[1]
-; CHECK-NEXT:    mov v19.d[1], v17.d[1]
-; CHECK-NEXT:    mov v18.d[1], v0.d[1]
-; CHECK-NEXT:    mov v1.d[1], v7.d[1]
-; CHECK-NEXT:    mov v5.d[1], v0.d[1]
-; CHECK-NEXT:    add v0.4s, v16.4s, v19.4s
-; CHECK-NEXT:    add v4.4s, v6.4s, v18.4s
-; CHECK-NEXT:    rev64 v3.4s, v0.4s
-; CHECK-NEXT:    sub v1.4s, v5.4s, v1.4s
-; CHECK-NEXT:    rev64 v5.4s, v4.4s
-; CHECK-NEXT:    sub v2.4s, v19.4s, v16.4s
-; CHECK-NEXT:    mov v3.d[1], v0.d[1]
-; CHECK-NEXT:    add v6.4s, v1.4s, v2.4s
-; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    mov v5.d[1], v4.d[1]
-; CHECK-NEXT:    rev64 v2.4s, v1.4s
-; CHECK-NEXT:    rev64 v7.4s, v6.4s
+; CHECK-NEXT:    add v19.4s, v3.4s, v1.4s
+; CHECK-NEXT:    shl v6.4s, v6.4s, #16
+; CHECK-NEXT:    shl v4.4s, v0.4s, #16
+; CHECK-NEXT:    add v1.4s, v2.4s, v16.4s
+; CHECK-NEXT:    add v7.4s, v6.4s, v5.4s
+; CHECK-NEXT:    add v18.4s, v4.4s, v17.4s
+; CHECK-NEXT:    ext v20.16b, v1.16b, v1.16b, #12
+; CHECK-NEXT:    zip1 v17.4s, v18.4s, v7.4s
+; CHECK-NEXT:    uzp2 v16.4s, v1.4s, v19.4s
+; CHECK-NEXT:    mov v2.16b, v1.16b
+; CHECK-NEXT:    mov v6.16b, v19.16b
+; CHECK-NEXT:    mov v2.s[0], v19.s[1]
+; CHECK-NEXT:    ext v5.16b, v19.16b, v20.16b, #12
+; CHECK-NEXT:    ext v20.16b, v18.16b, v17.16b, #8
+; CHECK-NEXT:    mov v6.s[1], v1.s[0]
+; CHECK-NEXT:    zip2 v3.4s, v1.4s, v19.4s
+; CHECK-NEXT:    uzp2 v4.4s, v16.4s, v1.4s
+; CHECK-NEXT:    zip2 v16.4s, v18.4s, v7.4s
+; CHECK-NEXT:    mov v18.s[3], v7.s[2]
+; CHECK-NEXT:    mov v2.d[1], v17.d[1]
+; CHECK-NEXT:    mov v6.d[1], v20.d[1]
+; CHECK-NEXT:    zip2 v0.4s, v19.4s, v1.4s
+; CHECK-NEXT:    mov v4.d[1], v16.d[1]
+; CHECK-NEXT:    mov v3.d[1], v18.d[1]
+; CHECK-NEXT:    add v1.4s, v2.4s, v6.4s
+; CHECK-NEXT:    mov v5.d[1], v16.d[1]
+; CHECK-NEXT:    mov v0.d[1], v18.d[1]
 ; CHECK-NEXT:    add v3.4s, v4.4s, v3.4s
+; CHECK-NEXT:    sub v2.4s, v6.4s, v2.4s
+; CHECK-NEXT:    rev64 v6.4s, v1.4s
+; CHECK-NEXT:    rev64 v4.4s, v3.4s
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v5.4s
-; CHECK-NEXT:    add v4.4s, v1.4s, v2.4s
-; CHECK-NEXT:    add v16.4s, v6.4s, v7.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    sub v2.4s, v6.4s, v7.4s
-; CHECK-NEXT:    rev64 v6.4s, v3.4s
-; CHECK-NEXT:    rev64 v17.4s, v0.4s
-; CHECK-NEXT:    ext v7.16b, v4.16b, v1.16b, #4
-; CHECK-NEXT:    ext v5.16b, v16.16b, v2.16b, #4
-; CHECK-NEXT:    add v18.4s, v3.4s, v6.4s
-; CHECK-NEXT:    add v19.4s, v0.4s, v17.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v17.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v6.4s
-; CHECK-NEXT:    rev64 v6.4s, v7.4s
-; CHECK-NEXT:    rev64 v7.4s, v18.4s
+; CHECK-NEXT:    movi v19.8h, #1
+; CHECK-NEXT:    mov v6.d[1], v1.d[1]
+; CHECK-NEXT:    add v5.4s, v0.4s, v2.4s
+; CHECK-NEXT:    sub v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    mov v4.d[1], v3.d[1]
+; CHECK-NEXT:    rev64 v2.4s, v5.4s
+; CHECK-NEXT:    rev64 v7.4s, v0.4s
+; CHECK-NEXT:    add v3.4s, v3.4s, v6.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    addp v4.4s, v3.4s, v5.4s
+; CHECK-NEXT:    sub v2.4s, v5.4s, v2.4s
+; CHECK-NEXT:    sub v6.4s, v0.4s, v7.4s
+; CHECK-NEXT:    rev64 v7.4s, v3.4s
+; CHECK-NEXT:    rev64 v5.4s, v1.4s
+; CHECK-NEXT:    zip1 v16.4s, v4.4s, v4.4s
+; CHECK-NEXT:    addp v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ext v17.16b, v4.16b, v2.16b, #4
+; CHECK-NEXT:    sub v3.4s, v3.4s, v7.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v5.4s
+; CHECK-NEXT:    ext v7.16b, v0.16b, v6.16b, #4
+; CHECK-NEXT:    ext v18.16b, v3.16b, v4.16b, #4
+; CHECK-NEXT:    trn2 v3.4s, v16.4s, v3.4s
+; CHECK-NEXT:    ext v16.16b, v1.16b, v0.16b, #8
+; CHECK-NEXT:    zip2 v5.4s, v17.4s, v4.4s
+; CHECK-NEXT:    zip2 v7.4s, v7.4s, v0.4s
 ; CHECK-NEXT:    ext v17.16b, v18.16b, v18.16b, #4
-; CHECK-NEXT:    ext v18.16b, v19.16b, v0.16b, #4
-; CHECK-NEXT:    rev64 v5.4s, v5.4s
-; CHECK-NEXT:    mov v16.s[3], v2.s[3]
-; CHECK-NEXT:    mov v4.s[3], v1.s[3]
-; CHECK-NEXT:    rev64 v18.4s, v18.4s
-; CHECK-NEXT:    mov v19.s[3], v0.s[3]
+; CHECK-NEXT:    ext v18.16b, v16.16b, v1.16b, #4
 ; CHECK-NEXT:    ext v5.16b, v2.16b, v5.16b, #12
-; CHECK-NEXT:    ext v6.16b, v1.16b, v6.16b, #12
-; CHECK-NEXT:    trn2 v7.4s, v7.4s, v3.4s
-; CHECK-NEXT:    trn2 v3.4s, v3.4s, v17.4s
-; CHECK-NEXT:    ext v18.16b, v0.16b, v18.16b, #12
-; CHECK-NEXT:    sub v17.4s, v16.4s, v5.4s
-; CHECK-NEXT:    sub v20.4s, v4.4s, v6.4s
-; CHECK-NEXT:    ext v3.16b, v3.16b, v3.16b, #4
-; CHECK-NEXT:    mov v16.s[0], v2.s[0]
-; CHECK-NEXT:    sub v2.4s, v19.4s, v18.4s
-; CHECK-NEXT:    mov v4.s[0], v1.s[0]
-; CHECK-NEXT:    mov v19.s[0], v0.s[0]
-; CHECK-NEXT:    add v1.4s, v7.4s, v3.4s
-; CHECK-NEXT:    sub v0.4s, v7.4s, v3.4s
-; CHECK-NEXT:    add v3.4s, v4.4s, v6.4s
-; CHECK-NEXT:    add v4.4s, v16.4s, v5.4s
-; CHECK-NEXT:    add v5.4s, v19.4s, v18.4s
-; CHECK-NEXT:    mov v4.d[1], v17.d[1]
+; CHECK-NEXT:    mov v2.s[2], v4.s[3]
+; CHECK-NEXT:    ext v7.16b, v6.16b, v7.16b, #12
+; CHECK-NEXT:    mov v1.s[2], v0.s[1]
+; CHECK-NEXT:    mov v6.s[2], v0.s[3]
+; CHECK-NEXT:    uzp2 v16.4s, v16.4s, v18.4s
+; CHECK-NEXT:    sub v18.4s, v2.4s, v5.4s
+; CHECK-NEXT:    mov v2.s[1], v4.s[2]
+; CHECK-NEXT:    sub v20.4s, v3.4s, v17.4s
+; CHECK-NEXT:    mov v17.s[0], v4.s[1]
+; CHECK-NEXT:    sub v21.4s, v6.4s, v7.4s
+; CHECK-NEXT:    sub v4.4s, v1.4s, v16.4s
+; CHECK-NEXT:    mov v6.s[1], v0.s[2]
+; CHECK-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-NEXT:    add v0.4s, v2.4s, v5.4s
+; CHECK-NEXT:    add v3.4s, v3.4s, v17.4s
+; CHECK-NEXT:    mov v0.d[1], v18.d[1]
+; CHECK-NEXT:    add v2.4s, v6.4s, v7.4s
+; CHECK-NEXT:    add v1.4s, v1.4s, v16.4s
 ; CHECK-NEXT:    mov v3.d[1], v20.d[1]
-; CHECK-NEXT:    mov v1.d[1], v0.d[1]
-; CHECK-NEXT:    mov v5.d[1], v2.d[1]
-; CHECK-NEXT:    movi v0.8h, #1
-; CHECK-NEXT:    movi v17.2d, #0x00ffff0000ffff
-; CHECK-NEXT:    ushr v2.4s, v1.4s, #15
-; CHECK-NEXT:    ushr v6.4s, v4.4s, #15
-; CHECK-NEXT:    ushr v7.4s, v5.4s, #15
-; CHECK-NEXT:    ushr v16.4s, v3.4s, #15
-; CHECK-NEXT:    and v6.16b, v6.16b, v0.16b
-; CHECK-NEXT:    and v16.16b, v16.16b, v0.16b
-; CHECK-NEXT:    and v7.16b, v7.16b, v0.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    mul v2.4s, v6.4s, v17.4s
-; CHECK-NEXT:    mul v6.4s, v16.4s, v17.4s
-; CHECK-NEXT:    mul v0.4s, v0.4s, v17.4s
-; CHECK-NEXT:    mul v7.4s, v7.4s, v17.4s
-; CHECK-NEXT:    add v4.4s, v2.4s, v4.4s
-; CHECK-NEXT:    add v3.4s, v6.4s, v3.4s
-; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    add v5.4s, v7.4s, v5.4s
-; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    eor v1.16b, v5.16b, v7.16b
-; CHECK-NEXT:    eor v3.16b, v3.16b, v6.16b
-; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    add v2.4s, v2.4s, v3.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v2.d[1], v21.d[1]
+; CHECK-NEXT:    mov v1.d[1], v4.d[1]
+; CHECK-NEXT:    ushr v5.4s, v0.4s, #15
+; CHECK-NEXT:    ushr v7.4s, v3.4s, #15
+; CHECK-NEXT:    and v4.16b, v5.16b, v19.16b
+; CHECK-NEXT:    ushr v6.4s, v2.4s, #15
+; CHECK-NEXT:    movi v5.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    ushr v16.4s, v1.4s, #15
+; CHECK-NEXT:    and v6.16b, v6.16b, v19.16b
+; CHECK-NEXT:    and v16.16b, v16.16b, v19.16b
+; CHECK-NEXT:    and v7.16b, v7.16b, v19.16b
+; CHECK-NEXT:    mul v6.4s, v6.4s, v5.4s
+; CHECK-NEXT:    mul v4.4s, v4.4s, v5.4s
+; CHECK-NEXT:    mul v7.4s, v7.4s, v5.4s
+; CHECK-NEXT:    mul v5.4s, v16.4s, v5.4s
+; CHECK-NEXT:    add v2.4s, v6.4s, v2.4s
+; CHECK-NEXT:    add v0.4s, v4.4s, v0.4s
+; CHECK-NEXT:    add v3.4s, v7.4s, v3.4s
+; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
+; CHECK-NEXT:    eor v3.16b, v3.16b, v7.16b
+; CHECK-NEXT:    eor v1.16b, v1.16b, v5.16b
+; CHECK-NEXT:    eor v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    eor v2.16b, v2.16b, v6.16b
 ; CHECK-NEXT:    add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    add v1.4s, v3.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    addv s0, v0.4s
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    lsr w9, w8, #16
 ; CHECK-NEXT:    add w8, w9, w8, uxth
 ; CHECK-NEXT:    lsr w0, w8, #1
 ; CHECK-NEXT:    ret
 entry:
   %idx.ext = sext i32 %i1 to i64

 ; CHECK-NEXT:    shl v3.4s, v3.4s, #16
 ; CHECK-NEXT:    shl v6.4s, v7.4s, #16
 ; CHECK-NEXT:    shl v5.4s, v5.4s, #16
 ; CHECK-NEXT:    shl v4.4s, v4.4s, #16
 ; CHECK-NEXT:    add v0.4s, v6.4s, v0.4s
 ; CHECK-NEXT:    add v3.4s, v3.4s, v16.4s
 ; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
 ; CHECK-NEXT:    add v2.4s, v4.4s, v2.4s
-; CHECK-NEXT:    rev64 v4.4s, v0.4s
-; CHECK-NEXT:    rev64 v5.4s, v3.4s
-; CHECK-NEXT:    rev64 v6.4s, v1.4s
-; CHECK-NEXT:    rev64 v7.4s, v2.4s
-; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
-; CHECK-NEXT:    add v17.4s, v3.4s, v5.4s
-; CHECK-NEXT:    add v18.4s, v1.4s, v6.4s
-; CHECK-NEXT:    add v19.4s, v2.4s, v7.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v7.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v4.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
-; CHECK-NEXT:    ext v4.16b, v17.16b, v17.16b, #12
-; CHECK-NEXT:    zip1 v5.4s, v18.4s, v19.4s
-; CHECK-NEXT:    ext v7.16b, v2.16b, v1.16b, #4
-; CHECK-NEXT:    mov v19.s[2], v18.s[3]
-; CHECK-NEXT:    uzp2 v18.4s, v17.4s, v16.4s
+; CHECK-NEXT:    rev64 v4.4s, v3.4s
+; CHECK-NEXT:    rev64 v5.4s, v0.4s
+; CHECK-NEXT:    rev64 v6.4s, v2.4s
+; CHECK-NEXT:    rev64 v17.4s, v1.4s
+; CHECK-NEXT:    addp v7.4s, v3.4s, v0.4s
+; CHECK-NEXT:    addp v16.4s, v2.4s, v1.4s
+; CHECK-NEXT:    sub v0.4s, v0.4s, v5.4s
+; CHECK-NEXT:    sub v3.4s, v3.4s, v4.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v17.4s
+; CHECK-NEXT:    sub v2.4s, v2.4s, v6.4s
 ; CHECK-NEXT:    zip2 v6.4s, v3.4s, v0.4s
-; CHECK-NEXT:    ext v4.16b, v16.16b, v4.16b, #12
-; CHECK-NEXT:    zip1 v0.4s, v3.4s, v0.4s
-; CHECK-NEXT:    trn2 v3.4s, v17.4s, v16.4s
-; CHECK-NEXT:    uzp2 v18.4s, v18.4s, v17.4s
+; CHECK-NEXT:    ext v17.16b, v2.16b, v1.16b, #4
 ; CHECK-NEXT:    mov v1.s[3], v2.s[2]
-; CHECK-NEXT:    ext v2.16b, v7.16b, v2.16b, #4
-; CHECK-NEXT:    mov v17.s[0], v16.s[1]
-; CHECK-NEXT:    mov v4.d[1], v19.d[1]
+; CHECK-NEXT:    uzp2 v5.4s, v7.4s, v16.4s
+; CHECK-NEXT:    ext v4.16b, v16.16b, v16.16b, #8
+; CHECK-NEXT:    uzp1 v16.4s, v7.4s, v16.4s
+; CHECK-NEXT:    zip1 v0.4s, v3.4s, v0.4s
 ; CHECK-NEXT:    mov v6.d[1], v1.d[1]
-; CHECK-NEXT:    mov v0.d[1], v2.d[1]
-; CHECK-NEXT:    mov v17.d[1], v5.d[1]
-; CHECK-NEXT:    mov v3.d[1], v5.d[1]
-; CHECK-NEXT:    mov v18.d[1], v19.d[1]
-; CHECK-NEXT:    add v1.4s, v6.4s, v0.4s
-; CHECK-NEXT:    add v2.4s, v4.4s, v17.4s
+; CHECK-NEXT:    ext v1.16b, v17.16b, v2.16b, #4
+; CHECK-NEXT:    rev64 v3.4s, v5.4s
+; CHECK-NEXT:    uzp1 v2.4s, v7.4s, v4.4s
+; CHECK-NEXT:    rev64 v5.4s, v16.4s
+; CHECK-NEXT:    uzp2 v4.4s, v7.4s, v4.4s
+; CHECK-NEXT:    mov v0.d[1], v1.d[1]
+; CHECK-NEXT:    add v1.4s, v3.4s, v5.4s
+; CHECK-NEXT:    sub v2.4s, v2.4s, v4.4s
+; CHECK-NEXT:    add v3.4s, v6.4s, v0.4s
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v6.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v18.4s
-; CHECK-NEXT:    zip1 v4.4s, v2.4s, v1.4s
-; CHECK-NEXT:    zip1 v5.4s, v3.4s, v0.4s
-; CHECK-NEXT:    uzp2 v6.4s, v2.4s, v1.4s
-; CHECK-NEXT:    mov v7.16b, v2.16b
-; CHECK-NEXT:    trn2 v4.4s, v2.4s, v4.4s
-; CHECK-NEXT:    ext v16.16b, v3.16b, v5.16b, #8
-; CHECK-NEXT:    uzp2 v6.4s, v6.4s, v2.4s
-; CHECK-NEXT:    mov v7.s[1], v1.s[1]
-; CHECK-NEXT:    zip2 v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    zip2 v2.4s, v3.4s, v0.4s
-; CHECK-NEXT:    mov v3.s[3], v0.s[2]
+; CHECK-NEXT:    zip1 v4.4s, v1.4s, v3.4s
+; CHECK-NEXT:    uzp2 v5.4s, v1.4s, v3.4s
+; CHECK-NEXT:    zip1 v6.4s, v2.4s, v0.4s
+; CHECK-NEXT:    zip2 v7.4s, v1.4s, v3.4s
+; CHECK-NEXT:    trn2 v4.4s, v1.4s, v4.4s
+; CHECK-NEXT:    uzp2 v5.4s, v5.4s, v1.4s
+; CHECK-NEXT:    ext v16.16b, v2.16b, v6.16b, #8
+; CHECK-NEXT:    zip2 v17.4s, v2.4s, v0.4s
+; CHECK-NEXT:    mov v1.s[1], v3.s[1]
+; CHECK-NEXT:    mov v2.s[3], v0.s[2]
 ; CHECK-NEXT:    mov v4.d[1], v16.d[1]
-; CHECK-NEXT:    mov v7.d[1], v5.d[1]
-; CHECK-NEXT:    mov v6.d[1], v2.d[1]
-; CHECK-NEXT:    mov v1.d[1], v3.d[1]
-; CHECK-NEXT:    add v0.4s, v4.4s, v7.4s
-; CHECK-NEXT:    sub v3.4s, v7.4s, v4.4s
-; CHECK-NEXT:    add v4.4s, v6.4s, v1.4s
-; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #4
-; CHECK-NEXT:    ext v7.16b, v4.16b, v4.16b, #4
-; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
-; CHECK-NEXT:    ext v5.16b, v2.16b, v3.16b, #8
-; CHECK-NEXT:    ext v6.16b, v7.16b, v1.16b, #8
-; CHECK-NEXT:    zip1 v16.4s, v4.4s, v1.4s
-; CHECK-NEXT:    zip2 v17.4s, v4.4s, v1.4s
-; CHECK-NEXT:    ext v2.16b, v5.16b, v2.16b, #4
-; CHECK-NEXT:    zip2 v5.4s, v3.4s, v0.4s
-; CHECK-NEXT:    zip2 v1.4s, v1.4s, v4.4s
-; CHECK-NEXT:    ext v4.16b, v6.16b, v7.16b, #4
-; CHECK-NEXT:    zip2 v6.4s, v0.4s, v3.4s
-; CHECK-NEXT:    zip1 v0.4s, v0.4s, v3.4s
-; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
-; CHECK-NEXT:    add v2.4s, v2.4s, v4.4s
-; CHECK-NEXT:    sub v3.4s, v17.4s, v6.4s
-; CHECK-NEXT:    sub v0.4s, v16.4s, v0.4s
+; CHECK-NEXT:    mov v5.d[1], v17.d[1]
+; CHECK-NEXT:    mov v1.d[1], v6.d[1]
+; CHECK-NEXT:    mov v7.d[1], v2.d[1]
+; CHECK-NEXT:    add v0.4s, v4.4s, v1.4s
+; CHECK-NEXT:    add v2.4s, v5.4s, v7.4s
+; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #4
+; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
+; CHECK-NEXT:    ext v4.16b, v2.16b, v2.16b, #4
+; CHECK-NEXT:    sub v5.4s, v7.4s, v5.4s
+; CHECK-NEXT:    ext v6.16b, v3.16b, v1.16b, #8
+; CHECK-NEXT:    zip1 v7.4s, v2.4s, v5.4s
+; CHECK-NEXT:    ext v17.16b, v4.16b, v5.16b, #8
+; CHECK-NEXT:    zip2 v16.4s, v2.4s, v5.4s
+; CHECK-NEXT:    ext v3.16b, v6.16b, v3.16b, #4
+; CHECK-NEXT:    zip2 v6.4s, v1.4s, v0.4s
+; CHECK-NEXT:    zip2 v2.4s, v5.4s, v2.4s
+; CHECK-NEXT:    ext v4.16b, v17.16b, v4.16b, #4
+; CHECK-NEXT:    zip2 v5.4s, v0.4s, v1.4s
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    add v1.4s, v6.4s, v2.4s
+; CHECK-NEXT:    add v2.4s, v3.4s, v4.4s
+; CHECK-NEXT:    sub v3.4s, v16.4s, v5.4s
+; CHECK-NEXT:    sub v0.4s, v7.4s, v0.4s
 ; CHECK-NEXT:    movi v4.8h, #1
 ; CHECK-NEXT:    movi v17.2d, #0x00ffff0000ffff
 ; CHECK-NEXT:    ushr v5.4s, v0.4s, #15
 ; CHECK-NEXT:    ushr v6.4s, v1.4s, #15
 ; CHECK-NEXT:    ushr v7.4s, v2.4s, #15
 ; CHECK-NEXT:    ushr v16.4s, v3.4s, #15
 ; CHECK-NEXT:    and v6.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    and v16.16b, v16.16b, v4.16b