[Target][ARM] Add PerformVSELECTCombine for MVE Integer Ops

[Pierre-vh]

Summary

This patch adds an implementation of PerformVSELECTCombine in the ARM DAG Combiner that transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).

• Isn't this supposed to be done in the (Target-Independent) DAGCombiner instead?
  • Yes, and I tried to do it with D77201, which caused issues with the X86 (AVX512) target (= it broke unit tests, which I unfortunately only noticed when it was upstream and I had to revert the change). The DAGCombiner doesn't optimize this for our cases as it doesn't allow truncation of the operands, so we have to do it here.
• Why do it for MVE Integer Ops only?
  • It's the only target that was changed by this patch. I get the same result (tests all pass) by removing the if around the setTargetDAGCombine, so let's be cautious and just enable it for this scenario at first.
  • I believe all others cases are already handled by the DAGCombiner, it's just this case that needs special handling.

[dmgreen]

There is the opposite transform in X86DAGCombiner? Hmm, I'm not sure if that's a great reason but I guess it's fine to put this here. I imagine the X86 code might have been different if the DAGCombine change had worked the other way all along. But it is probably simpler at least to do this here.

llvm/lib/Target/ARM/ARMISelLowering.cpp

949

You can change this.

11735–11736

It's probably best to put a check for if (!Subtarget->hasMVEIntegerOps()) return SDValue(); in case anyone in the future wants to add something Neon here too.

11738

I don't think this kind of comment adds a lot. It just says what the code says.

15267

You don't need to change this.

[Pierre-vh]

• Removed useless comment.
• Now calls setTargetDAGCombine(ISD::VSELECT) everytime, but the PerformVSELECTCombine function will return early if MVE Integer Ops are not enabled.

[dmgreen]

llvm/lib/Target/ARM/ARMISelLowering.cpp

1462

This can still be in a hasMVE block. I was more talking about _if_ in the future someone added Neon code to it. It doesn't need it yet.

[Pierre-vh]

• Adding if (Subtarget->hasMVEIntegerOps()) before setTargetDAGCombine(ISD::VSELECT)

[dmgreen]

LGTM

llvm/lib/Target/ARM/ARMISelLowering.cpp

if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
     setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
     setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
     setTargetDAGCombine(ISD::INTRINSIC_VOID);
     setTargetDAGCombine(ISD::VECREDUCE_ADD);
     setTargetDAGCombine(ISD::ADD);
     setTargetDAGCombine(ISD::BITCAST);
   }
 
   if (!Subtarget->hasFP64()) {
     // When targeting a floating-point unit with only single-precision
     // operations, f64 is legal for the few double-precision instructions which
     // are present However, no double-precision operations other than moves,
     // loads and stores are provided by the hardware.
     setOperationAction(ISD::FADD,       MVT::f64, Expand);
     setOperationAction(ISD::FSUB,       MVT::f64, Expand);
     setOperationAction(ISD::FMUL,       MVT::f64, Expand);
     setOperationAction(ISD::FMA,        MVT::f64, Expand);

   // We have target-specific dag combine patterns for the following nodes:
   // ARMISD::VMOVRRD  - No need to call setTargetDAGCombine
   setTargetDAGCombine(ISD::ADD);
   setTargetDAGCombine(ISD::SUB);
   setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::AND);
   setTargetDAGCombine(ISD::OR);
   setTargetDAGCombine(ISD::XOR);
 
+  if (Subtarget->hasMVEIntegerOps())
+    setTargetDAGCombine(ISD::VSELECT);
+
   if (Subtarget->hasV6Ops())
     setTargetDAGCombine(ISD::SRL);
   if (Subtarget->isThumb1Only())
     setTargetDAGCombine(ISD::SHL);
 
   setStackPointerRegisterToSaveRestore(ARM::SP);
 
   if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
       }
     }
   } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) {
     return AddCombineTo64bitMLAL(N, DCI, Subtarget);
   }
   return SDValue();
 }
 
+static SDValue PerformVSELECTCombine(SDNode *N,
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const ARMSubtarget *Subtarget) {
+  // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
+  //
+  // We need to re-implement this optimization here as the implementation in the
+  // Target-Independent DAGCombiner does not handle the kind of constant we make
+  // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
+  // good reason, allowing truncation there would break other targets).
+  //
+  // Currently, this is only done for MVE, as it's the only target that benefits
+  // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
+  if (!Subtarget->hasMVEIntegerOps())
+    return SDValue();
+
+  if (N->getOperand(0).getOpcode() != ISD::XOR)
+    return SDValue();
+  SDValue XOR = N->getOperand(0);
+
+  // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
+  // It is important to check with truncation allowed as the BUILD_VECTORs we
+  // generate in those situations will truncate their operands.
+  ConstantSDNode *Const =
+      isConstOrConstSplat(XOR->getOperand(1), /*AllowUndefs*/ false,
+                          /*AllowTruncation*/ true);
+  if (!Const || !Const->isOne())
+    return SDValue();
+
+  // Rewrite into vselect(cond, rhs, lhs).
+  SDValue Cond = XOR->getOperand(0);
+  SDValue LHS = N->getOperand(1);
+  SDValue RHS = N->getOperand(2);
+  EVT Type = N->getValueType(0);
+  return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS);
+}
+
 static SDValue PerformABSCombine(SDNode *N,
                                   TargetLowering::DAGCombinerInfo &DCI,
                                   const ARMSubtarget *Subtarget) {
   SDValue res;
   SelectionDAG &DAG = DCI.DAG;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   if (TLI.isOperationLegal(N->getOpcode(), N->getValueType(0)))
static SDValue PerformBITCASTCombine(SDNode *N, SelectionDAG &DAG) {
 
   return SDValue();
 }
 
 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   switch (N->getOpcode()) {
   default: break;
+  case ISD::VSELECT:    return PerformVSELECTCombine(N, DCI, Subtarget);
   case ISD::ABS:        return PerformABSCombine(N, DCI, Subtarget);
   case ARMISD::ADDE:    return PerformADDECombine(N, DCI, Subtarget);
   case ARMISD::UMLAL:   return PerformUMLALCombine(N, DCI.DAG, Subtarget);
   case ISD::ADD:        return PerformADDCombine(N, DCI, Subtarget);
   case ISD::SUB:        return PerformSUBCombine(N, DCI, Subtarget);
   case ISD::MUL:        return PerformMULCombine(N, DCI, Subtarget);
   case ISD::OR:         return PerformORCombine(N, DCI, Subtarget);
   case ISD::XOR:        return PerformXORCombine(N, DCI, Subtarget);

llvm/test/CodeGen/Thumb2/mve-pred-or.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpeqz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp eq <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpnez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsltz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpslez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
entry:
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpugtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ugt <4 x i32> %b, zeroinitializer
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }

 
 
 
 define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpeq_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp eq <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpne_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp ne <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpslt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q2, q1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp slt <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsgt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sgt <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsle_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q2, q1
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sle <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }
 
 define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <4 x i32> %a, zeroinitializer
   %c2 = icmp sge <4 x i32> %b, %c
   %o = or <4 x i1> %c1, %c2
   %s = select <4 x i1> %o, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %s
 }

 
 
 
 define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: cmpeqz_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i16 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 ne, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <8 x i16> %a, zeroinitializer
   %c2 = icmp eq <8 x i16> %b, zeroinitializer
   %o = or <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
 }
 
 define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
 ; CHECK-LABEL: cmpeq_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i16 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 ne, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <8 x i16> %a, zeroinitializer
   %c2 = icmp eq <8 x i16> %b, %c
   %o = or <8 x i1> %c1, %c2
   %s = select <8 x i1> %o, <8 x i16> %a, <8 x i16> %b
   ret <8 x i16> %s
 }
 
 
 define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: cmpeqz_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i8 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 ne, q1, zr
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <16 x i8> %a, zeroinitializer
   %c2 = icmp eq <16 x i8> %b, zeroinitializer
   %o = or <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %s
 }
 
 define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
 ; CHECK-LABEL: cmpeq_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vpt.i8 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 ne, q1, q2
-; CHECK-NEXT:    vpnot
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    bx lr
 entry:
   %c1 = icmp eq <16 x i8> %a, zeroinitializer
   %c2 = icmp eq <16 x i8> %b, %c
   %o = or <16 x i1> %c1, %c2
   %s = select <16 x i1> %o, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %s
 }

llvm/test/CodeGen/Thumb2/mve-vcmpf.ll

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q1, q0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s13, s9
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> %src, %src2
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %src2, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q1, q0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q1, q0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q4[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q4
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> %src, %src2
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %src2, <8 x half> %a, <8 x half> %b) {

llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll

 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp one <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ugt <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp uge <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ult <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ule <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ord <4 x float> %src, %sp
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp one <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ugt <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp uge <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ult <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ule <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ord <8 x half> %src, %sp
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp one <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ugt_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ugt <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uge_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp uge <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ult_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ult <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ule_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ule <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s12, s8
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
+; CHECK-MVEFP-NEXT:    vpsel q0, q3, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %i = insertelement <4 x float> undef, float %src2, i32 0
   %sp = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
   %c = fcmp ord <4 x float> %sp, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp one <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ugt_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ugt <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uge_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp uge <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ult_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ult <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ule_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ule <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

 ; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    ldrh r0, [r0]
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, r0
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %src2 = load half, half* %src2p
   %i = insertelement <8 x half> undef, half %src2, i32 0
   %sp = shufflevector <8 x half> %i, <8 x half> undef, <8 x i32> zeroinitializer
   %c = fcmp ord <8 x half> %sp, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s

llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ogt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ugt_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_uge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uge_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ult_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ult_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ule_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ule_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ogt_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ugt_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uge_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uge_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ult_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ult_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ule_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ule_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f32 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_ord_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 
 define arm_aapcs_vfpcc <4 x float> @vcmp_r_uno_v4f32(<4 x float> %src, <4 x float> %a, <4 x float> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_ogt_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ugt_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 ge, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ugt <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_uge_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uge_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uge <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_ult_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ult_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 le, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ult <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_ule_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ule_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vcmp.f16 lt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ule <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_ord_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {

 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 gt, q0, zr
-; CHECK-MVEFP-NEXT:    vpnot
-; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
+; CHECK-MVEFP-NEXT:    vpsel q0, q2, q1
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 
 define arm_aapcs_vfpcc <8 x half> @vcmp_r_uno_v8f16(<8 x half> %src, <8 x half> %a, <8 x half> %b) {