Add more efficient vector bitcast for v16i8 on AArch64.

[lawben]

Summary

We previously split the vector into two halves and performed two vector reduce operations followed by bit shifting and bitwise or. Now, we use NEON's zip1 to concatenate
the halves in a smart way and then perform only a single vector reduce. This boosts performance quite a bit for this small routine, as vector reduce is a rather expensive
intruction. Original discussion for this started in: https://reviews.llvm.org/D145301

[dmgreen]

Looks like a nice improvement. LGTM.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) {
   if (VecVT.getSizeInBits() > 128)
     return SDValue();
 
   // Ensure that all elements' bits are either 0s or 1s.
   ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT);
 
   SmallVector<SDValue, 16> MaskConstants;
   if (VecVT == MVT::v16i8) {
-    // v16i8 is a special case, as we need to split it into two halves and
-    // combine, perform the mask+addition twice, and then combine them.
+    // v16i8 is a special case, as we have 16 entries but only 8 positional bits
+    // per entry. We split it into two halves, apply the mask, zip the halves to
+    // create 8x 16-bit values, and the perform the vector reduce.
     for (unsigned Half = 0; Half < 2; ++Half) {
       for (unsigned MaskBit = 1; MaskBit <= 128; MaskBit *= 2) {
         MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i32));
       }
     }
     SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants);
     SDValue RepresentativeBits =
         DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask);
 
-    EVT HalfVT = VecVT.getHalfNumVectorElementsVT(*DAG.getContext());
-    unsigned NumElementsInHalf = HalfVT.getVectorNumElements();
-
-    SDValue LowHalf =
-        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, RepresentativeBits,
-                    DAG.getConstant(0, DL, MVT::i64));
-    SDValue HighHalf =
-        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, RepresentativeBits,
-                    DAG.getConstant(NumElementsInHalf, DL, MVT::i64));
-
-    SDValue ReducedLowBits =
-        DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i16, LowHalf);
-    SDValue ReducedHighBits =
-        DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i16, HighHalf);
-
-    SDValue ShiftedHighBits =
-        DAG.getNode(ISD::SHL, DL, MVT::i16, ReducedHighBits,
-                    DAG.getConstant(NumElementsInHalf, DL, MVT::i32));
-    return DAG.getNode(ISD::OR, DL, MVT::i16, ShiftedHighBits, ReducedLowBits);
+    SDValue UpperRepresentativeBits =
+        DAG.getNode(AArch64ISD::EXT, DL, VecVT, RepresentativeBits,
+                    RepresentativeBits, DAG.getConstant(8, DL, MVT::i32));
+    SDValue Zipped = DAG.getNode(AArch64ISD::ZIP1, DL, VecVT,
+                                 RepresentativeBits, UpperRepresentativeBits);
+    Zipped = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Zipped);
+    return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i16, Zipped);
   }
 
   // All other vector sizes.
   unsigned MaxBitMask = 1u << (VecVT.getVectorNumElements() - 1);
   for (unsigned MaskBit = 1; MaskBit <= MaxBitMask; MaskBit *= 2) {
     MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i64));
   }
 

llvm/test/CodeGen/AArch64/vec-combine-compare-to-bitmask.ll

 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT: Lloh0:
 ; CHECK-NEXT:  adrp     x8, lCPI0_0@PAGE
 ; CHECK-NEXT:  cmeq.16b v0, v0, #0
 ; CHECK-NEXT: Lloh1:
 ; CHECK-NEXT:  ldr      q1, [x8, lCPI0_0@PAGEOFF]
 ; CHECK-NEXT:  bic.16b  v0, v1, v0
 ; CHECK-NEXT:  ext.16b  v1, v0, v0, #8
-; CHECK-NEXT:  addv.8b  b0, v0
-; CHECK-NEXT:  addv.8b  b1, v1
-; CHECK-NEXT:  fmov     w9, s0
-; CHECK-NEXT:  fmov     w8, s1
-; CHECK-NEXT:  orr      w0, w9, w8, lsl #8
+; CHECK-NEXT:  zip1.16b v0, v0, v1
+; CHECK-NEXT:  addv.8h  h0, v0
+; CHECK-NEXT:  fmov     w0, s0
 ; CHECK-NEXT:  ret
 
   %cmp_result = icmp ne <16 x i8> %vec, zeroinitializer
   %bitmask = bitcast <16 x i1> %cmp_result to i16
   ret i16 %bitmask
 }
 
 define i16 @convert_to_bitmask8(<8 x i16> %vec) {

 ; CHECK-NEXT:  Lloh20:
 ; CHECK-NEXT:    adrp x8, lCPI10_0@PAGE
 ; CHECK-NEXT:    shl.16b v0, v0, #7
 ; CHECK-NEXT:    cmlt.16b v0, v0, #0
 ; CHECK-NEXT:  Lloh21:
 ; CHECK-NEXT:    ldr q1, [x8, lCPI10_0@PAGEOFF]
 ; CHECK-NEXT:    and.16b v0, v0, v1
 ; CHECK-NEXT:    ext.16b v1, v0, v0, #8
-; CHECK-NEXT:    addv.8b b0, v0
-; CHECK-NEXT:    addv.8b b1, v1
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    fmov w8, s1
-; CHECK-NEXT:    orr w0, w9, w8, lsl #8
+; CHECK-NEXT:    zip1.16b v0, v0, v1
+; CHECK-NEXT:    addv.8h h0, v0
+; CHECK-NEXT:    fmov w0, s0
 ; CHECK-NEXT:    ret
 
   %bitmask = bitcast <16 x i1> %vec to i16
   ret i16 %bitmask
 }
 
 define i2 @convert_to_bitmask_2xi32(<2 x i32> %vec) {
 ; CHECK-LABEL: convert_to_bitmask_2xi32

llvm/test/CodeGen/AArch64/vec-combine-compare-truncate-store.ll

 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:  Lloh0:
 ; CHECK-NEXT:  adrp     x8, lCPI0_0@PAGE
 ; CHECK-NEXT:  cmeq.16b v0, v0, #0
 ; CHECK-NEXT:  Lloh1:
 ; CHECK-NEXT:  ldr      q1, [x8, lCPI0_0@PAGEOFF]
 ; CHECK-NEXT:  bic.16b  v0, v1, v0
 ; CHECK-NEXT:  ext.16b  v1, v0, v0, #8
-; CHECK-NEXT:  addv.8b  b0, v0
-; CHECK-NEXT:  addv.8b  b1, v1
-; CHECK-NEXT:  fmov     w9, s0
-; CHECK-NEXT:  fmov     w8, s1
-; CHECK-NEXT:  orr      w8, w9, w8, lsl #8
-; CHECK-NEXT:  strh     w8, [x0]
+; CHECK-NEXT:  zip1.16b v0, v0, v1
+; CHECK-NEXT:  addv.8h  h0, v0
+; CHECK-NEXT:  str      h0, [x0]
 ; CHECK-NEXT:  ret
 
   %cmp_result = icmp ne <16 x i8> %vec, zeroinitializer
   store <16 x i1> %cmp_result, ptr %out
   ret void
 }
 
 define void @store_8_elements(<8 x i16> %vec, ptr %out) {