[ARM] Better OR's for MVE compares

[dmgreen]

Summary

This adds a DeMorgan combine for OR's of compares to turn them into AND's, helping prevent them from going into and out of gpr registers. It also fills in the VCLE and VCLT nodes that MVE can select, allowing it to invert more compares.

[SjoerdMeijer]

llvm/lib/Target/ARM/ARMISelLowering.cpp

11850

I am wondering if a map is easier as a data structure here so that you can just do a lookup and don't need the for-loop here?

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

11850

I think it may be simpler to just deal with condition codes directly, as in D65072. It allows us to just use the existing methods for getting inverse nodes, for example.

[SjoerdMeijer]

LGTM

[dmgreen]

Changed std::vector to SmallVector

llvm/lib/Target/ARM/ARMISelLowering.h

enum NodeType : unsigned {
       PREDICATE_CAST, // Predicate cast for MVE i1 types
 
       VCEQ,         // Vector compare equal.
       VCEQZ,        // Vector compare equal to zero.
       VCNE,         // Vector compare not equal (MVE)
       VCNEZ,        // Vector compare not equal to zero (MVE)
       VCGE,         // Vector compare greater than or equal.
       VCGEZ,        // Vector compare greater than or equal to zero.
+      VCLE,         // Vector compare less than or equal.
       VCLEZ,        // Vector compare less than or equal to zero.
       VCGEU,        // Vector compare unsigned greater than or equal.
       VCGT,         // Vector compare greater than.
       VCGTZ,        // Vector compare greater than zero.
+      VCLT,         // Vector compare less than.
       VCLTZ,        // Vector compare less than zero.
       VCGTU,        // Vector compare unsigned greater than.
       VTST,         // Vector test bits.
 
       // Vector shift by vector
       VSHLs,        // ...left/right by signed
       VSHLu,        // ...left/right by unsigned
 

llvm/lib/Target/ARM/ARMISelLowering.cpp

const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::WIN__DBZCHK:   return "ARMISD::WIN__DBZCHK";
 
   case ARMISD::VCEQ:          return "ARMISD::VCEQ";
   case ARMISD::VCEQZ:         return "ARMISD::VCEQZ";
   case ARMISD::VCNE:          return "ARMISD::VCNE";
   case ARMISD::VCNEZ:         return "ARMISD::VCNEZ";
   case ARMISD::VCGE:          return "ARMISD::VCGE";
   case ARMISD::VCGEZ:         return "ARMISD::VCGEZ";
+  case ARMISD::VCLE:          return "ARMISD::VCLE";
   case ARMISD::VCLEZ:         return "ARMISD::VCLEZ";
   case ARMISD::VCGEU:         return "ARMISD::VCGEU";
   case ARMISD::VCGT:          return "ARMISD::VCGT";
   case ARMISD::VCGTZ:         return "ARMISD::VCGTZ";
+  case ARMISD::VCLT:          return "ARMISD::VCLT";
   case ARMISD::VCLTZ:         return "ARMISD::VCLTZ";
   case ARMISD::VCGTU:         return "ARMISD::VCGTU";
   case ARMISD::VTST:          return "ARMISD::VTST";
 
   case ARMISD::VSHLs:         return "ARMISD::VSHLs";
   case ARMISD::VSHLu:         return "ARMISD::VSHLu";
   case ARMISD::VSHLIMM:       return "ARMISD::VSHLIMM";
   case ARMISD::VSHRsIMM:      return "ARMISD::VSHRsIMM";
if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
     // Return value from the original node to inform the combiner than N is
     // now dead.
     return SDValue(N, 0);
   }
 
   return SDValue();
 }
 
+static SDValue PerformORCombine_i1(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI,
+                                   const ARMSubtarget *Subtarget) {
+  // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
+  // together with predicates
+  struct Codes {
+    unsigned Opcode;
+    unsigned Opposite;
+  } InvertCodes[] = {
+      {ARMISD::VCEQ, ARMISD::VCNE},
+      {ARMISD::VCEQZ, ARMISD::VCNEZ},
+      {ARMISD::VCGE, ARMISD::VCLT},
+      {ARMISD::VCGEZ, ARMISD::VCLTZ},
+      {ARMISD::VCGT, ARMISD::VCLE},
+      {ARMISD::VCGTZ, ARMISD::VCLEZ},
+  };
+
+  EVT VT = N->getValueType(0);
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  unsigned Opposite0 = 0;
+  unsigned Opposite1 = 0;
+  for (auto Code : InvertCodes) {
+    if (N0->getOpcode() == Code.Opcode)
+      Opposite0 = Code.Opposite;
+    if (N0->getOpcode() == Code.Opposite)
+      Opposite0 = Code.Opcode;
+    if (N1->getOpcode() == Code.Opcode)
+      Opposite1 = Code.Opposite;
+    if (N1->getOpcode() == Code.Opposite)
+      Opposite1 = Code.Opcode;
+  }
+
+  if (!Opposite0 || !Opposite1)
+    return SDValue();
+
+  SmallVector<SDValue, 4> Ops0;
+  for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i)
+    Ops0.push_back(N0->getOperand(i));
+  SmallVector<SDValue, 4> Ops1;
+  for (unsigned i = 0, e = N1->getNumOperands(); i != e; ++i)
+    Ops1.push_back(N1->getOperand(i));
+
+  SDValue NewN0 = DCI.DAG.getNode(Opposite0, SDLoc(N0), VT, Ops0);
+  SDValue NewN1 = DCI.DAG.getNode(Opposite1, SDLoc(N1), VT, Ops1);
+  SDValue And = DCI.DAG.getNode(ISD::AND, SDLoc(N), VT, NewN0, NewN1);
+  return DCI.DAG.getNode(ISD::XOR, SDLoc(N), VT, And,
+                         DCI.DAG.getAllOnesConstant(SDLoc(N), VT));
+}
+
 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR
 static SDValue PerformORCombine(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const ARMSubtarget *Subtarget) {
   // Attempt to use immediate-form VORR
   BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
                                              N0->getOperand(0),
                                              N1->getOperand(0));
                 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
             }
         }
     }
   }
 
+  if (Subtarget->hasMVEIntegerOps() &&
+      (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1))
+    return PerformORCombine_i1(N, DCI, Subtarget);
+
   // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
   // reasonable.
   if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
     if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
       return Res;
   }
 
   if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))

llvm/lib/Target/ARM/ARMInstrInfo.td

 def SDTARMVCMPZ   : SDTypeProfile<1, 1, []>;
 
 def ARMvceq      : SDNode<"ARMISD::VCEQ", SDTARMVCMP>;
 def ARMvceqz     : SDNode<"ARMISD::VCEQZ", SDTARMVCMPZ>;
 def ARMvcne      : SDNode<"ARMISD::VCNE", SDTARMVCMP>;
 def ARMvcnez     : SDNode<"ARMISD::VCNEZ", SDTARMVCMPZ>;
 def ARMvcge      : SDNode<"ARMISD::VCGE", SDTARMVCMP>;
 def ARMvcgez     : SDNode<"ARMISD::VCGEZ", SDTARMVCMPZ>;
+def ARMvcle      : SDNode<"ARMISD::VCLE", SDTARMVCMP>;
 def ARMvclez     : SDNode<"ARMISD::VCLEZ", SDTARMVCMPZ>;
 def ARMvcgeu     : SDNode<"ARMISD::VCGEU", SDTARMVCMP>;
 def ARMvcgt      : SDNode<"ARMISD::VCGT", SDTARMVCMP>;
 def ARMvcgtz     : SDNode<"ARMISD::VCGTZ", SDTARMVCMPZ>;
+def ARMvclt      : SDNode<"ARMISD::VCLT", SDTARMVCMP>;
 def ARMvcltz     : SDNode<"ARMISD::VCLTZ", SDTARMVCMPZ>;
 def ARMvcgtu     : SDNode<"ARMISD::VCGTU", SDTARMVCMP>;
 
 //===----------------------------------------------------------------------===//
 // ARM Flag Definitions.
 
 class RegConstraint<string C> {
   string Constraints = C;

llvm/lib/Target/ARM/ARMInstrMVE.td

def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (opnode (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))))),
             (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc, 1, VCCR:$p1))>;
   def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (opnode (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))))),
             (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc, 1, VCCR:$p1))>;
 }
 
 let Predicates = [HasMVEInt] in {
   defm MVE_VCEQZ  : unpred_vcmp_z<ARMvceqz, "i", 0>;
   defm MVE_VCNEZ  : unpred_vcmp_z<ARMvcnez, "i", 1>;
-  defm MVE_VCLEZ  : unpred_vcmp_z<ARMvclez, "s", 13>;
-  defm MVE_VCGTZ  : unpred_vcmp_z<ARMvcgtz, "s", 12>;
-  defm MVE_VCLTZ  : unpred_vcmp_z<ARMvcltz, "s", 11>;
   defm MVE_VCGEZ  : unpred_vcmp_z<ARMvcgez, "s", 10>;
+  defm MVE_VCLTZ  : unpred_vcmp_z<ARMvcltz, "s", 11>;
+  defm MVE_VCGTZ  : unpred_vcmp_z<ARMvcgtz, "s", 12>;
+  defm MVE_VCLEZ  : unpred_vcmp_z<ARMvclez, "s", 13>;
 
   defm MVE_VCEQ   : unpred_vcmp_r<ARMvceq, "i", 0>;
   defm MVE_VCNE   : unpred_vcmp_r<ARMvcne, "i", 1>;
-  defm MVE_VCGT   : unpred_vcmp_r<ARMvcgt, "s", 12>;
   defm MVE_VCGE   : unpred_vcmp_r<ARMvcge, "s", 10>;
+  defm MVE_VCLT   : unpred_vcmp_r<ARMvclt, "s", 11>;
+  defm MVE_VCGT   : unpred_vcmp_r<ARMvcgt, "s", 12>;
+  defm MVE_VCLE   : unpred_vcmp_r<ARMvcle, "s", 13>;
   defm MVE_VCGTU  : unpred_vcmp_r<ARMvcgtu, "u", 8>;
   defm MVE_VCGEU  : unpred_vcmp_r<ARMvcgeu, "u", 2>;
 }
 
 let Predicates = [HasMVEFloat] in {
   defm MVE_VFCEQZ  : unpred_vcmpf_z<ARMvceqz, 0>;
   defm MVE_VFCNEZ  : unpred_vcmpf_z<ARMvcnez, 1>;
-  defm MVE_VFCLEZ  : unpred_vcmpf_z<ARMvclez, 13>;
-  defm MVE_VFCGTZ  : unpred_vcmpf_z<ARMvcgtz, 12>;
-  defm MVE_VFCLTZ  : unpred_vcmpf_z<ARMvcltz, 11>;
   defm MVE_VFCGEZ  : unpred_vcmpf_z<ARMvcgez, 10>;
+  defm MVE_VFCLTZ  : unpred_vcmpf_z<ARMvcltz, 11>;
+  defm MVE_VFCGTZ  : unpred_vcmpf_z<ARMvcgtz, 12>;
+  defm MVE_VFCLEZ  : unpred_vcmpf_z<ARMvclez, 13>;
 
-  defm MVE_VFCGT   : unpred_vcmpf_r<ARMvcgt, 12>;
   defm MVE_VFCGE   : unpred_vcmpf_r<ARMvcge, 10>;
+  defm MVE_VFCLT   : unpred_vcmpf_r<ARMvclt, 11>;
+  defm MVE_VFCGT   : unpred_vcmpf_r<ARMvcgt, 12>;
+  defm MVE_VFCLE   : unpred_vcmpf_r<ARMvcle, 13>;
   defm MVE_VFCEQ   : unpred_vcmpf_r<ARMvceq, 0>;
   defm MVE_VFCNE   : unpred_vcmpf_r<ARMvcne, 1>;
 }
 
 
 // Extra "worst case" and/or/xor partterns, going into and out of GRP
 multiclass two_predops<SDPatternOperator opnode, Instruction insn> {
   def v16i1 : Pat<(v16i1 (opnode (v16i1 VCCR:$p1), (v16i1 VCCR:$p2))),

llvm/test/CodeGen/Thumb2/mve-pred-or.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpeqz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp eq <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpnez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsltz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 lt, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 le, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpslez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 le, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
entry:
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpugtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, zr
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ugt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
entry:
   ret <4 x i32> %s
 }
 
 
 
 define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpeq_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q1, q2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp eq <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpne_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q1, q2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpslt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q2, q1
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 le, q2, q1
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsgt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 gt, q1, q2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 le, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsle_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q2, q1
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 lt, q2, q1
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 ge, q1, q2
+; CHECK-NEXT:    vcmp.i32 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.s32 lt, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b

 }
 
 
 
 
 define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: cmpeqz_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 eq, q1, zr
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i16 ne, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <8 x i16> %a, zeroinitializer
   %c2 = icmp eq <8 x i16> %b, zeroinitializer
   %o = or <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
 }
 
 define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
 ; CHECK-LABEL: cmpeq_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 eq, q1, q2
+; CHECK-NEXT:    vcmp.i16 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i16 ne, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <8 x i16> %a, zeroinitializer
   %c2 = icmp eq <8 x i16> %b, %c
   %o = or <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
 }
 
 
 define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: cmpeqz_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 eq, q1, zr
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i8 ne, q1, zr
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i8 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <16 x i8> %a, zeroinitializer
   %c2 = icmp eq <16 x i8> %b, zeroinitializer
   %o = or <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %s
 }
 
 define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
 ; CHECK-LABEL: cmpeq_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 eq, q1, q2
+; CHECK-NEXT:    vcmp.i8 ne, q0, zr
+; CHECK-NEXT:    movw r1, #65535
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vcmpt.i8 ne, q1, q2
 ; CHECK-NEXT:    vmrs r0, p0
-; CHECK-NEXT:    vcmp.i8 eq, q0, zr
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    eors r0, r1
 ; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <16 x i8> %a, zeroinitializer
   %c2 = icmp eq <16 x i8> %b, %c
   %o = or <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b

llvm/test/CodeGen/Thumb2/mve-vcmpf.ll

 ; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q1
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    cmp r1, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q1
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q1
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q1
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q1
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q1
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q3
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q3
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q3
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    cmp r2, #0
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q3
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q3
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q3
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q3
+; CHECK-MVEFP-NEXT:    movw r1, #65535
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
+; CHECK-MVEFP-NEXT:    eors r0, r1
 ; CHECK-MVEFP-NEXT:    vmsr p0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q3
-; CHECK-MVEFP-NEXT:    vmrs r0, p0
-; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q3, q0
-; CHECK-MVEFP-NEXT:    vmrs r1, p0
-; CHECK-MVEFP-NEXT:    orrs r0, r1
-; CHECK-MVEFP-NEXT:    movw r1, #65535
-; CHECK-MVEFP-NEXT:    eors r0, r1
-; CHECK-MVEFP-NEXT:    vmsr p0, r0
+; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }