[ARM] Handle any extend whilst lowering mull

[dmgreen]

Summary

Similar to D93832 but for ARM, this handles any_extend whilst creating MULL nodes, treating them as zextends.

[RKSimon]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
   if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
     return true;
   if (isExtendedBUILD_VECTOR(N, DAG, true))
     return true;
   return false;
 }
 
-/// isZeroExtended - Check if a node is a vector value that is zero-extended
-/// or a constant BUILD_VECTOR with zero-extended elements.
+/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
+/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
-  if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N))
+  if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
+      ISD::isZEXTLoad(N))
     return true;
   if (isExtendedBUILD_VECTOR(N, DAG, false))
     return true;
   return false;
 }
 
 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
   if (OrigVT.getSizeInBits() >= 64)
static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
   // operation legalization where we can't create illegal types.
   return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
                         LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
                         LD->getMemoryVT(), LD->getAlignment(),
                         LD->getMemOperand()->getFlags());
 }
 
 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
-/// extending load, or BUILD_VECTOR with extended elements, return the
-/// unextended value. The unextended vector should be 64 bits so that it can
+/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
+/// the unextended value. The unextended vector should be 64 bits so that it can
 /// be used as an operand to a VMULL instruction. If the original vector size
 /// before extension is less than 64 bits we add a an extension to resize
 /// the vector to 64 bits.
 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
-  if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
+  if (N->getOpcode() == ISD::SIGN_EXTEND ||
+      N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
     return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
                                         N->getOperand(0)->getValueType(0),
                                         N->getValueType(0),
                                         N->getOpcode());
 
   if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
     assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&
            "Expected extending load");

llvm/test/CodeGen/ARM/vmla.ll

; CHECK-NEXT:    bx lr
   %tmp6 = mul <2 x i64> %tmp4, %tmp5
   %tmp7 = add <2 x i64> %A, %tmp6
   ret <2 x i64> %tmp7
 }
 
 define <8 x i16> @vmlala8(<8 x i16> %A, <8 x i8> %B, <8 x i8> %C) nounwind {
 ; CHECK-LABEL: vmlala8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u8 q8, d3
-; CHECK-NEXT:    vmovl.u8 q9, d2
-; CHECK-NEXT:    vmla.i16 q0, q9, q8
+; CHECK-NEXT:    vmlal.u8 q0, d2, d3
 ; CHECK-NEXT:    vbic.i16 q0, #0xff00
 ; CHECK-NEXT:    bx lr
   %tmp4 = zext <8 x i8> %B to <8 x i16>
   %tmp5 = zext <8 x i8> %C to <8 x i16>
   %tmp6 = mul <8 x i16> %tmp4, %tmp5
   %tmp7 = add <8 x i16> %A, %tmp6
   %and = and <8 x i16> %tmp7, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   ret <8 x i16> %and
 }
 
 define <4 x i32> @vmlala16(<4 x i32> %A, <4 x i16> %B, <4 x i16> %C) nounwind {
 ; CHECK-LABEL: vmlala16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u16 q8, d3
-; CHECK-NEXT:    vmovl.u16 q9, d2
-; CHECK-NEXT:    vmla.i32 q0, q9, q8
+; CHECK-NEXT:    vmlal.u16 q0, d2, d3
 ; CHECK-NEXT:    vmov.i32 q8, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q8
 ; CHECK-NEXT:    bx lr
   %tmp4 = zext <4 x i16> %B to <4 x i32>
   %tmp5 = zext <4 x i16> %C to <4 x i32>
   %tmp6 = mul <4 x i32> %tmp4, %tmp5
   %tmp7 = add <4 x i32> %A, %tmp6
   %and = and <4 x i32> %tmp7, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define <2 x i64> @vmlala32(<2 x i64> %A, <2 x i32> %B, <2 x i32> %C) nounwind {
 ; CHECK-LABEL: vmlala32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vmovl.u32 q8, d3
-; CHECK-NEXT:    vmovl.u32 q9, d2
-; CHECK-NEXT:    vmov.32 r0, d16[0]
-; CHECK-NEXT:    vmov.32 r1, d18[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r3, d17[0]
-; CHECK-NEXT:    vmov.32 r2, d19[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r1, r0
-; CHECK-NEXT:    mla r1, r1, r12, r5
-; CHECK-NEXT:    umull r5, r4, r2, r3
-; CHECK-NEXT:    mla r2, r2, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r3, r2
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r0, r1
-; CHECK-NEXT:    vmov.32 d16[1], r0
-; CHECK-NEXT:    vadd.i64 q8, q0, q8
-; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    vmlal.u32 q0, d2, d3
+; CHECK-NEXT:    vmov.i64 q8, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q8
+; CHECK-NEXT:    bx lr
   %tmp4 = zext <2 x i32> %B to <2 x i64>
   %tmp5 = zext <2 x i32> %C to <2 x i64>
   %tmp6 = mul <2 x i64> %tmp4, %tmp5
   %tmp7 = add <2 x i64> %A, %tmp6
   %and = and <2 x i64> %tmp7, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }
 
entry:
   %3 = mul <2 x i64> %1, %2
   %4 = add <2 x i64> %arg0_uint64x2_t, %3
   ret <2 x i64> %4
 }
 
 define arm_aapcs_vfpcc <4 x i32> @test_vmlal_lanea16(<4 x i32> %arg0_uint32x4_t, <4 x i16> %arg1_uint16x4_t, <4 x i16> %arg2_uint16x4_t) nounwind readnone {
 ; CHECK-LABEL: test_vmlal_lanea16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.16 d16, d3[1]
-; CHECK-NEXT:    vmovl.u16 q9, d2
-; CHECK-NEXT:    vmovl.u16 q8, d16
-; CHECK-NEXT:    vmla.i32 q0, q9, q8
+; CHECK-NEXT:    vmlal.u16 q0, d2, d3[1]
 ; CHECK-NEXT:    vmov.i32 q8, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q8
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <4 x i16> %arg2_uint16x4_t, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1> ; <<4 x i16>> [#uses=1]
   %1 = zext <4 x i16> %arg1_uint16x4_t to <4 x i32>
   %2 = zext <4 x i16> %0 to <4 x i32>
   %3 = mul <4 x i32> %1, %2
   %4 = add <4 x i32> %arg0_uint32x4_t, %3
   %and = and <4 x i32> %4, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define arm_aapcs_vfpcc <2 x i64> @test_vmlal_lanea32(<2 x i64> %arg0_uint64x2_t, <2 x i32> %arg1_uint32x2_t, <2 x i32> %arg2_uint32x2_t) nounwind readnone {
 ; CHECK-LABEL: test_vmlal_lanea32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vdup.32 d16, d3[1]
-; CHECK-NEXT:    vmovl.u32 q9, d2
-; CHECK-NEXT:    vmovl.u32 q8, d16
-; CHECK-NEXT:    vmov.32 r0, d18[0]
-; CHECK-NEXT:    vmov.32 r3, d19[0]
-; CHECK-NEXT:    vmov.32 r1, d16[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r2, d17[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r0, r1
-; CHECK-NEXT:    mla r0, r0, r12, r5
-; CHECK-NEXT:    umull r5, r4, r3, r2
-; CHECK-NEXT:    mla r3, r3, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r2, r3
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r1, r0
-; CHECK-NEXT:    vmov.32 d16[1], r0
-; CHECK-NEXT:    vadd.i64 q8, q0, q8
-; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    vmlal.u32 q0, d2, d3[1]
+; CHECK-NEXT:    vmov.i64 q8, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q8
+; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <2 x i32> %arg2_uint32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = zext <2 x i32> %arg1_uint32x2_t to <2 x i64>
   %2 = zext <2 x i32> %0 to <2 x i64>
   %3 = mul <2 x i64> %1, %2
   %4 = add <2 x i64> %arg0_uint64x2_t, %3
   %and = and <2 x i64> %4, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }

llvm/test/CodeGen/ARM/vmls.ll

; CHECK-NEXT:    bx lr
   %tmp6 = mul <2 x i64> %tmp4, %tmp5
   %tmp7 = sub <2 x i64> %A, %tmp6
   ret <2 x i64> %tmp7
 }
 
 define <8 x i16> @vmlsla8(<8 x i16> %A, <8 x i8> %B, <8 x i8> %C) nounwind {
 ; CHECK-LABEL: vmlsla8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u8 q8, d3
-; CHECK-NEXT:    vmovl.u8 q9, d2
-; CHECK-NEXT:    vmls.i16 q0, q9, q8
+; CHECK-NEXT:    vmlsl.u8 q0, d2, d3
 ; CHECK-NEXT:    vbic.i16 q0, #0xff00
 ; CHECK-NEXT:    bx lr
   %tmp4 = zext <8 x i8> %B to <8 x i16>
   %tmp5 = zext <8 x i8> %C to <8 x i16>
   %tmp6 = mul <8 x i16> %tmp4, %tmp5
   %tmp7 = sub <8 x i16> %A, %tmp6
   %and = and <8 x i16> %tmp7, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   ret <8 x i16> %and
 }
 
 define <4 x i32> @vmlsla16(<4 x i32> %A, <4 x i16> %B, <4 x i16> %C) nounwind {
 ; CHECK-LABEL: vmlsla16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u16 q8, d3
-; CHECK-NEXT:    vmovl.u16 q9, d2
-; CHECK-NEXT:    vmls.i32 q0, q9, q8
+; CHECK-NEXT:    vmlsl.u16 q0, d2, d3
 ; CHECK-NEXT:    vmov.i32 q8, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q8
 ; CHECK-NEXT:    bx lr
   %tmp4 = zext <4 x i16> %B to <4 x i32>
   %tmp5 = zext <4 x i16> %C to <4 x i32>
   %tmp6 = mul <4 x i32> %tmp4, %tmp5
   %tmp7 = sub <4 x i32> %A, %tmp6
   %and = and <4 x i32> %tmp7, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define <2 x i64> @vmlsla32(<2 x i64> %A, <2 x i32> %B, <2 x i32> %C) nounwind {
 ; CHECK-LABEL: vmlsla32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vmovl.u32 q8, d3
-; CHECK-NEXT:    vmovl.u32 q9, d2
-; CHECK-NEXT:    vmov.32 r0, d16[0]
-; CHECK-NEXT:    vmov.32 r1, d18[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r3, d17[0]
-; CHECK-NEXT:    vmov.32 r2, d19[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r1, r0
-; CHECK-NEXT:    mla r1, r1, r12, r5
-; CHECK-NEXT:    umull r5, r4, r2, r3
-; CHECK-NEXT:    mla r2, r2, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r3, r2
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r0, r1
-; CHECK-NEXT:    vmov.32 d16[1], r0
-; CHECK-NEXT:    vsub.i64 q8, q0, q8
-; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    vmlsl.u32 q0, d2, d3
+; CHECK-NEXT:    vmov.i64 q8, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q8
+; CHECK-NEXT:    bx lr
   %tmp4 = zext <2 x i32> %B to <2 x i64>
   %tmp5 = zext <2 x i32> %C to <2 x i64>
   %tmp6 = mul <2 x i64> %tmp4, %tmp5
   %tmp7 = sub <2 x i64> %A, %tmp6
   %and = and <2 x i64> %tmp7, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }
 
entry:
   %3 = mul <2 x i64> %1, %2
   %4 = sub <2 x i64> %arg0_uint64x2_t, %3
   ret <2 x i64> %4
 }
 
 define arm_aapcs_vfpcc <4 x i32> @test_vmlsl_lanea16(<4 x i32> %arg0_uint32x4_t, <4 x i16> %arg1_uint16x4_t, <4 x i16> %arg2_uint16x4_t) nounwind readnone {
 ; CHECK-LABEL: test_vmlsl_lanea16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.16 d16, d3[1]
-; CHECK-NEXT:    vmovl.u16 q9, d2
-; CHECK-NEXT:    vmovl.u16 q8, d16
-; CHECK-NEXT:    vmls.i32 q0, q9, q8
+; CHECK-NEXT:    vmlsl.u16 q0, d2, d3[1]
 ; CHECK-NEXT:    vmov.i32 q8, #0xffff
 ; CHECK-NEXT:    vand q0, q0, q8
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <4 x i16> %arg2_uint16x4_t, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1> ; <<4 x i16>> [#uses=1]
   %1 = zext <4 x i16> %arg1_uint16x4_t to <4 x i32>
   %2 = zext <4 x i16> %0 to <4 x i32>
   %3 = mul <4 x i32> %1, %2
   %4 = sub <4 x i32> %arg0_uint32x4_t, %3
   %and = and <4 x i32> %4, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define arm_aapcs_vfpcc <2 x i64> @test_vmlsl_lanea32(<2 x i64> %arg0_uint64x2_t, <2 x i32> %arg1_uint32x2_t, <2 x i32> %arg2_uint32x2_t) nounwind readnone {
 ; CHECK-LABEL: test_vmlsl_lanea32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vdup.32 d16, d3[1]
-; CHECK-NEXT:    vmovl.u32 q9, d2
-; CHECK-NEXT:    vmovl.u32 q8, d16
-; CHECK-NEXT:    vmov.32 r0, d18[0]
-; CHECK-NEXT:    vmov.32 r3, d19[0]
-; CHECK-NEXT:    vmov.32 r1, d16[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r2, d17[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r0, r1
-; CHECK-NEXT:    mla r0, r0, r12, r5
-; CHECK-NEXT:    umull r5, r4, r3, r2
-; CHECK-NEXT:    mla r3, r3, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r2, r3
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r1, r0
-; CHECK-NEXT:    vmov.32 d16[1], r0
-; CHECK-NEXT:    vsub.i64 q8, q0, q8
-; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    vmlsl.u32 q0, d2, d3[1]
+; CHECK-NEXT:    vmov.i64 q8, #0xffffffff
+; CHECK-NEXT:    vand q0, q0, q8
+; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <2 x i32> %arg2_uint32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = zext <2 x i32> %arg1_uint32x2_t to <2 x i64>
   %2 = zext <2 x i32> %0 to <2 x i64>
   %3 = mul <2 x i64> %1, %2
   %4 = sub <2 x i64> %arg0_uint64x2_t, %3
   %and = and <2 x i64> %4, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }

llvm/test/CodeGen/ARM/vmul.ll

 ; CHECK-NEXT:    bx lr
   %tmp3 = call <2 x i64> @llvm.arm.neon.vmullu.v2i64(<2 x i32> %A, <2 x i32> %B)
   ret <2 x i64> %tmp3
 }
 
 define <8 x i16> @vmulla8(<8 x i8> %A, <8 x i8> %B) nounwind {
 ; CHECK-LABEL: vmulla8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u8 q8, d1
-; CHECK-NEXT:    vmovl.u8 q9, d0
-; CHECK-NEXT:    vmul.i16 q0, q9, q8
+; CHECK-NEXT:    vmull.u8 q0, d0, d1
 ; CHECK-NEXT:    vbic.i16 q0, #0xff00
 ; CHECK-NEXT:    bx lr
   %tmp3 = zext <8 x i8> %A to <8 x i16>
   %tmp4 = zext <8 x i8> %B to <8 x i16>
   %tmp5 = mul <8 x i16> %tmp3, %tmp4
   %and = and <8 x i16> %tmp5, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
   ret <8 x i16> %and
 }
 
 define <4 x i32> @vmulla16(<4 x i16> %A, <4 x i16> %B) nounwind {
 ; CHECK-LABEL: vmulla16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vmovl.u16 q8, d1
-; CHECK-NEXT:    vmovl.u16 q9, d0
-; CHECK-NEXT:    vmul.i32 q8, q9, q8
+; CHECK-NEXT:    vmull.u16 q8, d0, d1
 ; CHECK-NEXT:    vmov.i32 q9, #0xffff
 ; CHECK-NEXT:    vand q0, q8, q9
 ; CHECK-NEXT:    bx lr
   %tmp3 = zext <4 x i16> %A to <4 x i32>
   %tmp4 = zext <4 x i16> %B to <4 x i32>
   %tmp5 = mul <4 x i32> %tmp3, %tmp4
   %and = and <4 x i32> %tmp5, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define <2 x i64> @vmulla32(<2 x i32> %A, <2 x i32> %B) nounwind {
 ; CHECK-LABEL: vmulla32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vmovl.u32 q8, d1
-; CHECK-NEXT:    vmovl.u32 q9, d0
-; CHECK-NEXT:    vmov.32 r0, d16[0]
-; CHECK-NEXT:    vmov.32 r1, d18[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r3, d17[0]
-; CHECK-NEXT:    vmov.32 r2, d19[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r1, r0
-; CHECK-NEXT:    mla r1, r1, r12, r5
-; CHECK-NEXT:    umull r5, r4, r2, r3
-; CHECK-NEXT:    mla r2, r2, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
+; CHECK-NEXT:    vmull.u32 q8, d0, d1
 ; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r3, r2
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r0, r1
-; CHECK-NEXT:    vmov.32 d16[1], r0
 ; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    bx lr
   %tmp3 = zext <2 x i32> %A to <2 x i64>
   %tmp4 = zext <2 x i32> %B to <2 x i64>
   %tmp5 = mul <2 x i64> %tmp3, %tmp4
   %and = and <2 x i64> %tmp5, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }
 
 define <8 x i16> @vmullp8(<8 x i8> %A, <8 x i8> %B) nounwind {
entry:
   %0 = shufflevector <2 x i32> %arg1_uint32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = tail call <2 x i64> @llvm.arm.neon.vmullu.v2i64(<2 x i32> %arg0_uint32x2_t, <2 x i32> %0) ; <<2 x i64>> [#uses=1]
   ret <2 x i64> %1
 }
 
 define arm_aapcs_vfpcc <4 x i32> @test_vmull_lanea16(<4 x i16> %arg0_uint16x4_t, <4 x i16> %arg1_uint16x4_t) nounwind readnone {
 ; CHECK-LABEL: test_vmull_lanea16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vdup.16 d16, d1[1]
-; CHECK-NEXT:    vmovl.u16 q9, d0
-; CHECK-NEXT:    vmovl.u16 q8, d16
-; CHECK-NEXT:    vmul.i32 q8, q9, q8
+; CHECK-NEXT:    vmull.u16 q8, d0, d1[1]
 ; CHECK-NEXT:    vmov.i32 q9, #0xffff
 ; CHECK-NEXT:    vand q0, q8, q9
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <4 x i16> %arg1_uint16x4_t, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1> ; <<4 x i16>> [#uses=1]
   %1 = zext <4 x i16> %arg0_uint16x4_t to <4 x i32>
   %2 = zext <4 x i16> %0 to <4 x i32>
   %3 = mul <4 x i32> %1, %2
   %and = and <4 x i32> %3, <i32 65535, i32 65535, i32 65535, i32 65535>
   ret <4 x i32> %and
 }
 
 define arm_aapcs_vfpcc <2 x i64> @test_vmull_lanea32(<2 x i32> %arg0_uint32x2_t, <2 x i32> %arg1_uint32x2_t) nounwind readnone {
 ; CHECK-LABEL: test_vmull_lanea32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, r11, lr}
-; CHECK-NEXT:    vdup.32 d16, d1[1]
-; CHECK-NEXT:    vmovl.u32 q9, d0
-; CHECK-NEXT:    vmovl.u32 q8, d16
-; CHECK-NEXT:    vmov.32 r0, d18[0]
-; CHECK-NEXT:    vmov.32 r3, d19[0]
-; CHECK-NEXT:    vmov.32 r1, d16[0]
-; CHECK-NEXT:    vmov.32 r12, d16[1]
-; CHECK-NEXT:    vmov.32 r2, d17[0]
-; CHECK-NEXT:    vmov.32 lr, d17[1]
-; CHECK-NEXT:    vmov.32 r6, d19[1]
-; CHECK-NEXT:    umull r7, r5, r0, r1
-; CHECK-NEXT:    mla r0, r0, r12, r5
-; CHECK-NEXT:    umull r5, r4, r3, r2
-; CHECK-NEXT:    mla r3, r3, lr, r4
-; CHECK-NEXT:    vmov.32 r4, d18[1]
+; CHECK-NEXT:    vmull.u32 q8, d0, d1[1]
 ; CHECK-NEXT:    vmov.i64 q9, #0xffffffff
-; CHECK-NEXT:    mla r2, r6, r2, r3
-; CHECK-NEXT:    vmov.32 d17[0], r5
-; CHECK-NEXT:    vmov.32 d16[0], r7
-; CHECK-NEXT:    vmov.32 d17[1], r2
-; CHECK-NEXT:    mla r0, r4, r1, r0
-; CHECK-NEXT:    vmov.32 d16[1], r0
 ; CHECK-NEXT:    vand q0, q8, q9
-; CHECK-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+; CHECK-NEXT:    bx lr
 entry:
   %0 = shufflevector <2 x i32> %arg1_uint32x2_t, <2 x i32> undef, <2 x i32> <i32 1, i32 1> ; <<2 x i32>> [#uses=1]
   %1 = zext <2 x i32> %arg0_uint32x2_t to <2 x i64>
   %2 = zext <2 x i32> %0 to <2 x i64>
   %3 = mul <2 x i64> %1, %2
   %and = and <2 x i64> %3, <i64 4294967295, i64 4294967295>
   ret <2 x i64> %and
 }