[ARM] Better AND's for MVE compares

[dmgreen]

Summary

Add a number of folds to convert and(vcmp, vcmp) into a single VPT block, where the second vcmp becomes predicated on the first.

The VCMP; VPST; VCMP will then be converted to VPT; VCMP in the VPTBlockPass.

[SjoerdMeijer]

Looks like a good optimisation to me.

llvm/trunk/lib/Target/ARM/ARMInstrMVE.td

 
 multiclass unpred_vcmp_z<SDPatternOperator opnode, string suffix, int fc> {
   def i8  : Pat<(v16i1 (opnode (v16i8 MQPR:$v1))),
                 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc))>;
   def i16 : Pat<(v8i1 (opnode (v8i16 MQPR:$v1))),
                 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc))>;
   def i32 : Pat<(v4i1 (opnode (v4i32 MQPR:$v1))),
                 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc))>;
+
+  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (opnode (v16i8 MQPR:$v1))))),
+            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc, 1, VCCR:$p1))>;
+  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (opnode (v8i16 MQPR:$v1))))),
+            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc, 1, VCCR:$p1))>;
+  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (opnode (v4i32 MQPR:$v1))))),
+            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc, 1, VCCR:$p1))>;
 }
 
 multiclass unpred_vcmp_r<SDPatternOperator opnode, string suffix, int fc> {
   def i8  : Pat<(v16i1 (opnode (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
                 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc))>;
   def i16 : Pat<(v8i1 (opnode (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),
                 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc))>;
   def i32 : Pat<(v4i1 (opnode (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
                 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc))>;
+
+  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (opnode (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))))),
+            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc, 1, VCCR:$p1))>;
+  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (opnode (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))))),
+            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc, 1, VCCR:$p1))>;
+  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (opnode (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))))),
+            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc, 1, VCCR:$p1))>;
 }
 
 multiclass unpred_vcmpf_z<SDPatternOperator opnode, int fc> {
   def f16 : Pat<(v8i1 (opnode (v8f16 MQPR:$v1))),
                 (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc))>;
   def f32 : Pat<(v4i1 (opnode (v4f32 MQPR:$v1))),
                 (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc))>;
+
+  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (opnode (v8f16 MQPR:$v1))))),
+            (v8i1 (MVE_VCMPf32r (v8f16 MQPR:$v1), ZR, fc, 1, VCCR:$p1))>;
+  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (opnode (v4f32 MQPR:$v1))))),
+            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc, 1, VCCR:$p1))>;
 }
 
 multiclass unpred_vcmpf_r<SDPatternOperator opnode, int fc> {
   def f16 : Pat<(v8i1 (opnode (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
                 (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc))>;
   def f32 : Pat<(v4i1 (opnode (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
                 (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc))>;
+
+  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (opnode (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))))),
+            (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc, 1, VCCR:$p1))>;
+  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (opnode (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))))),
+            (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc, 1, VCCR:$p1))>;
 }
 
 let Predicates = [HasMVEInt] in {
   defm MVE_VCEQZ  : unpred_vcmp_z<ARMvceqz, "i", 0>;
   defm MVE_VCNEZ  : unpred_vcmp_z<ARMvcnez, "i", 1>;
   defm MVE_VCLEZ  : unpred_vcmp_z<ARMvclez, "s", 13>;
   defm MVE_VCGTZ  : unpred_vcmp_z<ARMvcgtz, "s", 12>;
   defm MVE_VCLTZ  : unpred_vcmp_z<ARMvcltz, "s", 11>;

llvm/trunk/test/CodeGen/Thumb2/mve-pred-and.ll

entry:
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpnez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsltz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 lt, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpslez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 le, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 le, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
entry:
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpugtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, zr
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ugt <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpulez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vmov.i32 q2, #0x0
-; CHECK-NEXT:    vcmp.u32 cs, q2, q1
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ule <4 x i32> %b, zeroinitializer
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
entry:
   ret <4 x i32> %s
 }
 
 
 
 define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpeq_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp eq <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpne_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpslt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q2, q1
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsgt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsle_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q2, q1
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 ge, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 ge, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpult_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.u32 hi, q2, q1
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 hi, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ult <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpugt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.u32 hi, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 hi, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ugt <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpule_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.u32 cs, q2, q1
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ule <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpuge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.u32 cs, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.u32 cs, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp uge <4 x i32> %b, %c
   %o = and <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
entry:
   %o = and <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
 }
 
 define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
 ; CHECK-LABEL: cmpeq_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 eq, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i16 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <8 x i16> %a, zeroinitializer
   %c2 = icmp eq <8 x i16> %b, %c
   %o = and <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
entry:
   %o = and <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %s
 }
 
 define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
 ; CHECK-LABEL: cmpeq_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 eq, q1, q2
-; CHECK-NEXT:    vmrs r0, p0
 ; CHECK-NEXT:    vcmp.i8 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    ands r0, r1
-; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i8 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <16 x i8> %a, zeroinitializer
   %c2 = icmp eq <16 x i8> %b, %c
   %o = and <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %s