[ARM] Look through concat when lowering in-place shuffles (VZIP, ..)

[ab]

Summary

Currently, we canonicalize shuffles that produce a result larger than
their operands with:

shuffle(concat(v1, undef), concat(v2, undef))

->

shuffle(concat(v1, v2), undef)

because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).

This is useful in the general case, but there are special cases where
native shuffles produce larger results: the two-result ops.

Look through the concat when lowering them:

shuffle(concat(v1, v2), undef)

->

concat(VZIP(v1, v2):0, :1)

This lets us generate the native shuffles instead of scalarizing to
dozens of VMOVs.

I'm a little worried about the disparity between the lowering and
isShuffleMaskLegal, but with the current API we have no way of looking
at the actual operands, and this isn't a problem in practice because
the ARM combine runs last.

The obvious alternative would be to stop doing the combine, but I
think it's useful. We can also avoid doing it for these masks, but
we'll still need to look through concat(v, undef) to avoid
generating needlessly-wide shuffles.

[qcolombet]

Hi Ahmed,

LGTM with some improvements on the tests patterns.

If you really really are motivated, you could fix all the patterns in a subsequent commit :).

Cheers,
-Quentin

test/CodeGen/ARM/vtrn.ll

17 ↗

(On Diff #27613)

I know this is consistent with the surrounding tests, but I would prefer that we check that the arguments are what we expect. In other words, could you check that we are feeding the right arguments here?

[ab]

No need for motivation with Chandler's script ;)

r240114, r240116, r240118.

-Ahmed

llvm/trunk/lib/Target/ARM/ARMISelLowering.cpp

if (EltSize <= 32) {
     if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
             ShuffleMask, VT, WhichResult, isV_UNDEF)) {
       if (isV_UNDEF)
         V2 = V1;
       return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
           .getValue(WhichResult);
     }
 
+    // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
+    // shuffles that produce a result larger than their operands with:
+    //   shuffle(concat(v1, undef), concat(v2, undef))
+    // ->
+    //   shuffle(concat(v1, v2), undef)
+    // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
+    //
+    // This is useful in the general case, but there are special cases where
+    // native shuffles produce larger results: the two-result ops.
+    //
+    // Look through the concat when lowering them:
+    //   shuffle(concat(v1, v2), undef)
+    // ->
+    //   concat(VZIP(v1, v2):0, :1)
+    //
+    if (V1->getOpcode() == ISD::CONCAT_VECTORS &&
+        V2->getOpcode() == ISD::UNDEF) {
+      SDValue SubV1 = V1->getOperand(0);
+      SDValue SubV2 = V1->getOperand(1);
+      EVT SubVT = SubV1.getValueType();
+
+      // We expect these to have been canonicalized to -1.
+      assert(std::all_of(ShuffleMask.begin(), ShuffleMask.end(), [&](int i) {
+        return i < (int)VT.getVectorNumElements();
+      }) && "Unexpected shuffle index into UNDEF operand!");
+
+      if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
+              ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
+        if (isV_UNDEF)
+          SubV2 = SubV1;
+        assert((WhichResult == 0) &&
+               "In-place shuffle of concat can only have one result!");
+        SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
+                                  SubV1, SubV2);
+        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
+                           Res.getValue(1));
+      }
+    }
   }
 
   // If the shuffle is not directly supported and it has 4 elements, use
   // the PerfectShuffle-generated table to synthesize it from other shuffles.
   unsigned NumElts = VT.getVectorNumElements();
   if (NumElts == 4) {
     unsigned PFIndexes[4];
     for (unsigned i = 0; i != 4; ++i) {

llvm/trunk/test/CodeGen/ARM/vtrn.ll

; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vtrni8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vtrni8_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d19, [r0]
-; CHECK-NEXT:    vldr d18, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d19[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[2]
-; CHECK-NEXT:    vmov.8 d16[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d16[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[4]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[6]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[6]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[1]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[3]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[5]
-; CHECK-NEXT:    vmov.8 d17[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14, i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
   ret <16 x i8> %tmp3
 }
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
         %tmp5 = add <4 x i16> %tmp3, %tmp4
   ret <4 x i16> %tmp5
 }
 
 define <8 x i16> @vtrni16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ; CHECK-LABEL: vtrni16_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d16, [r0]
 ; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vmov.u16 r0, d16[0]
-; CHECK-NEXT:    vmov.16 d18[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[0]
-; CHECK-NEXT:    vmov.16 d18[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[2]
-; CHECK-NEXT:    vmov.16 d18[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[2]
-; CHECK-NEXT:    vmov.16 d18[3], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[1]
-; CHECK-NEXT:    vmov.16 d19[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[1]
-; CHECK-NEXT:    vmov.16 d19[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[3]
-; CHECK-NEXT:    vmov.16 d19[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[3]
-; CHECK-NEXT:    vmov.16 d19[3], r0
-; CHECK-NEXT:    vmov r0, r1, d18
-; CHECK-NEXT:    vmov r2, r3, d19
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 4, i32 2, i32 6, i32 1, i32 5, i32 3, i32 7>
   ret <8 x i16> %tmp3
 }
 
 define <2 x i32> @vtrni32(<2 x i32>* %A, <2 x i32>* %B) nounwind {
; CHECK-NEXT:    mov pc, lr
   ret <2 x i32> %tmp5
 }
 
 define <4 x i32> @vtrni32_Qres(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ; CHECK-LABEL: vtrni32_Qres:
 ; CHECK:       @ BB#0:
 ; CHECK-NEXT:    vldr d17, [r1]
 ; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vrev64.32 q9, q8
-; CHECK-NEXT:    vuzp.32 q8, q9
+; CHECK-NEXT:    vtrn.32 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = shufflevector <2 x i32> %tmp1, <2 x i32> %tmp2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x i32> %tmp3
 }
; CHECK-NEXT:    mov pc, lr
   ret <2 x float> %tmp5
 }
 
 define <4 x float> @vtrnf_Qres(<2 x float>* %A, <2 x float>* %B) nounwind {
 ; CHECK-LABEL: vtrnf_Qres:
 ; CHECK:       @ BB#0:
 ; CHECK-NEXT:    vldr d17, [r1]
 ; CHECK-NEXT:    vldr d16, [r0]
-; CHECK-NEXT:    vrev64.32 q9, q8
-; CHECK-NEXT:    vuzp.32 q8, q9
+; CHECK-NEXT:    vtrn.32 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <2 x float>, <2 x float>* %A
   %tmp2 = load <2 x float>, <2 x float>* %B
   %tmp3 = shufflevector <2 x float> %tmp1, <2 x float> %tmp2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
   ret <4 x float> %tmp3
 }
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 undef, i32 undef, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vtrni8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vtrni8_undef_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vldr d19, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[2]
-; CHECK-NEXT:    vmov.8 d16[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[6]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[6]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[1]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[3]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vtrn.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 undef, i32 2, i32 10, i32 undef, i32 12, i32 6, i32 14, i32 1, i32 9, i32 3, i32 11, i32 5, i32 undef, i32 undef, i32 15>
   ret <16 x i8> %tmp3
 }

llvm/trunk/test/CodeGen/ARM/vuzp.ll

; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vuzpi8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vuzpi8_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d19, [r0]
-; CHECK-NEXT:    vldr d18, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d19[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d16[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[6]
-; CHECK-NEXT:    vmov.8 d16[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[2]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[4]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[6]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[1]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[3]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[5]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d17[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d17[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   ret <16 x i8> %tmp3
 }
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
         %tmp5 = add <4 x i16> %tmp3, %tmp4
   ret <4 x i16> %tmp5
 }
 
 define <8 x i16> @vuzpi16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ; CHECK-LABEL: vuzpi16_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d16, [r0]
 ; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vmov.u16 r0, d16[0]
-; CHECK-NEXT:    vmov.16 d18[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[2]
-; CHECK-NEXT:    vmov.16 d18[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[0]
-; CHECK-NEXT:    vmov.16 d18[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[2]
-; CHECK-NEXT:    vmov.16 d18[3], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[1]
-; CHECK-NEXT:    vmov.16 d19[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[3]
-; CHECK-NEXT:    vmov.16 d19[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[1]
-; CHECK-NEXT:    vmov.16 d19[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[3]
-; CHECK-NEXT:    vmov.16 d19[3], r0
-; CHECK-NEXT:    vmov r0, r1, d18
-; CHECK-NEXT:    vmov r2, r3, d19
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
   ret <8 x i16> %tmp3
 }
 
 ; VUZP.32 is equivalent to VTRN.32 for 64-bit vectors.
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 13, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vuzpi8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vuzpi8_undef_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vldr d19, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[2]
-; CHECK-NEXT:    vmov.8 d16[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[0]
-; CHECK-NEXT:    vmov.8 d16[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[6]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[7]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[5]
-; CHECK-NEXT:    vmov.8 d17[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vuzp.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 undef, i32 undef, i32 13, i32 15>
   ret <16 x i8> %tmp3
 }

llvm/trunk/test/CodeGen/ARM/vzip.ll

; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vzipi8_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vzipi8_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d19, [r0]
-; CHECK-NEXT:    vldr d18, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d19[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[1]
-; CHECK-NEXT:    vmov.8 d16[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d16[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[2]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[3]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[4]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[5]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[6]
-; CHECK-NEXT:    vmov.8 d17[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[6]
-; CHECK-NEXT:    vmov.8 d17[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   ret <16 x i8> %tmp3
 }
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
         %tmp5 = add <4 x i16> %tmp3, %tmp4
   ret <4 x i16> %tmp5
 }
 
 define <8 x i16> @vzipi16_Qres(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ; CHECK-LABEL: vzipi16_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d16, [r0]
 ; CHECK-NEXT:    vldr d17, [r1]
-; CHECK-NEXT:    vmov.u16 r0, d16[0]
-; CHECK-NEXT:    vmov.16 d18[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[0]
-; CHECK-NEXT:    vmov.16 d18[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[1]
-; CHECK-NEXT:    vmov.16 d18[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[1]
-; CHECK-NEXT:    vmov.16 d18[3], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[2]
-; CHECK-NEXT:    vmov.16 d19[0], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[2]
-; CHECK-NEXT:    vmov.16 d19[1], r0
-; CHECK-NEXT:    vmov.u16 r0, d16[3]
-; CHECK-NEXT:    vmov.16 d19[2], r0
-; CHECK-NEXT:    vmov.u16 r0, d17[3]
-; CHECK-NEXT:    vmov.16 d19[3], r0
-; CHECK-NEXT:    vmov r0, r1, d18
-; CHECK-NEXT:    vmov r2, r3, d19
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.16 d16, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x i16> %tmp3
 }
 
 ; VZIP.32 is equivalent to VTRN.32 for 64-bit vectors.
; CHECK-NEXT:    mov pc, lr
   %tmp4 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 undef, i32 undef, i32 15>
         %tmp5 = add <8 x i8> %tmp3, %tmp4
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @vzipi8_undef_Qres(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ; CHECK-LABEL: vzipi8_undef_Qres:
 ; CHECK:       @ BB#0:
-; CHECK-NEXT:    vldr d18, [r0]
-; CHECK-NEXT:    vldr d19, [r1]
-; CHECK-NEXT:    vmov.u8 r0, d18[0]
-; CHECK-NEXT:    vmov.8 d16[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[1]
-; CHECK-NEXT:    vmov.8 d16[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[1]
-; CHECK-NEXT:    vmov.8 d16[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[2]
-; CHECK-NEXT:    vmov.8 d16[5], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[3]
-; CHECK-NEXT:    vmov.8 d16[6], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[3]
-; CHECK-NEXT:    vmov.8 d16[7], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[4]
-; CHECK-NEXT:    vmov.8 d17[0], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[4]
-; CHECK-NEXT:    vmov.8 d17[1], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[5]
-; CHECK-NEXT:    vmov.8 d17[2], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[5]
-; CHECK-NEXT:    vmov.8 d17[3], r0
-; CHECK-NEXT:    vmov.u8 r0, d18[6]
-; CHECK-NEXT:    vmov.8 d17[4], r0
-; CHECK-NEXT:    vmov.u8 r0, d19[7]
-; CHECK-NEXT:    vmov.8 d17[7], r0
+; CHECK-NEXT:    vldr d17, [r1]
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vzip.8 d16, d17
 ; CHECK-NEXT:    vmov r0, r1, d16
 ; CHECK-NEXT:    vmov r2, r3, d17
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = shufflevector <8 x i8> %tmp1, <8 x i8> %tmp2, <16 x i32> <i32 0, i32 undef, i32 1, i32 9, i32 undef, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 undef, i32 undef, i32 15>
   ret <16 x i8> %tmp3
 }