[ARM] Add some VCMP folding and canonicalisation

[dmgreen]

Summary

The VCMP instructions in MVE can accept a register or ZR, but only as the right hand operator. Most of the time this will already be correct because the icmp will have been canonicalised that way already. There are some cases in the lowering of float conditions that this will not apply to though. This code should fix up those cases.

[SjoerdMeijer]

Yep, nice combine

llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp

case ARM::t2ANDri:
       if (SrcReg == MI->getOperand(CommonUse ? 1 : 0).getReg())
         return true;
       break;
   }
 
   return false;
 }
 
-/// getSwappedCondition - assume the flags are set by MI(a,b), return
-/// the condition code if we modify the instructions such that flags are
-/// set by MI(b,a).
-inline static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC) {
-  switch (CC) {
-  default: return ARMCC::AL;
-  case ARMCC::EQ: return ARMCC::EQ;
-  case ARMCC::NE: return ARMCC::NE;
-  case ARMCC::HS: return ARMCC::LS;
-  case ARMCC::LO: return ARMCC::HI;
-  case ARMCC::HI: return ARMCC::LO;
-  case ARMCC::LS: return ARMCC::HS;
-  case ARMCC::GE: return ARMCC::LE;
-  case ARMCC::LT: return ARMCC::GT;
-  case ARMCC::GT: return ARMCC::LT;
-  case ARMCC::LE: return ARMCC::GE;
-  }
-}
-
 /// getCmpToAddCondition - assume the flags are set by CMP(a,b), return
 /// the condition code if we modify the instructions such that flags are
 /// set by ADD(a,b,X).
 inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
   switch (CC) {
   default: return ARMCC::AL;
   case ARMCC::HS: return ARMCC::LO;
   case ARMCC::LO: return ARMCC::HS;

llvm/lib/Target/ARM/ARMISelLowering.cpp

static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
   }
   SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
   return DAG.getTruncStore(
       ST->getChain(), dl, GRP, ST->getBasePtr(),
       EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
       ST->getMemOperand());
 }
 
+static bool isZeroVector(SDValue N) {
+  return (ISD::isBuildVectorAllZeros(N.getNode()) ||
+          (N->getOpcode() == ARMISD::VMOVIMM &&
+           isNullConstant(N->getOperand(0))));
+}
+
 static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
   MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
   MVT VT = Op.getSimpleValueType();
   SDValue Mask = N->getMask();
   SDValue PassThru = N->getPassThru();
   SDLoc dl(Op);
 
-  auto IsZero = [](SDValue PassThru) {
-    return (ISD::isBuildVectorAllZeros(PassThru.getNode()) ||
-      (PassThru->getOpcode() == ARMISD::VMOVIMM &&
-       isNullConstant(PassThru->getOperand(0))));
-  };
-
-  if (IsZero(PassThru))
+  if (isZeroVector(PassThru))
     return Op;
 
   // MVE Masked loads use zero as the passthru value. Here we convert undef to
   // zero too, and other values are lowered to a select.
   SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
                                 DAG.getTargetConstant(0, dl, MVT::i32));
   SDValue NewLoad = DAG.getMaskedLoad(
       VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec,
       N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
       N->getExtensionType(), N->isExpandingLoad());
   SDValue Combo = NewLoad;
   if (!PassThru.isUndef() &&
       (PassThru.getOpcode() != ISD::BITCAST ||
-       !IsZero(PassThru->getOperand(0))))
+       !isZeroVector(PassThru->getOperand(0))))
     Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
   return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
 }
 
 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
   if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering()))
     // Acquire/Release load/store is not legal for targets without a dmb or
     // equivalent available.
if (Op->getOperand(0).getValueType() == VT)
       return Op->getOperand(0);
     return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl,
                            Op->getOperand(0).getValueType(), Op->getOperand(0));
   }
 
   return SDValue();
 }
 
+static SDValue PerformVCMPCombine(SDNode *N,
+                                  TargetLowering::DAGCombinerInfo &DCI,
+                                  const ARMSubtarget *Subtarget) {
+  if (!Subtarget->hasMVEIntegerOps())
+    return SDValue();
+
+  EVT VT = N->getValueType(0);
+  SDValue Op0 = N->getOperand(0);
+  SDValue Op1 = N->getOperand(1);
+  ARMCC::CondCodes Cond =
+      (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+  SDLoc dl(N);
+
+  // vcmp X, 0, cc -> vcmpz X, cc
+  if (isZeroVector(Op1))
+    return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0,
+                           N->getOperand(2));
+
+  unsigned SwappedCond = getSwappedCondition(Cond);
+  if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) {
+    // vcmp 0, X, cc -> vcmpz X, reversed(cc)
+    if (isZeroVector(Op0))
+      return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1,
+                             DCI.DAG.getConstant(SwappedCond, dl, MVT::i32));
+    // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
+    if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
+      return DCI.DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
+                             DCI.DAG.getConstant(SwappedCond, dl, MVT::i32));
+  }
+
+  return SDValue();
+}
+
 /// PerformInsertEltCombine - Target-specific dag combine xforms for
 /// ISD::INSERT_VECTOR_ELT.
 static SDValue PerformInsertEltCombine(SDNode *N,
                                        TargetLowering::DAGCombinerInfo &DCI) {
   // Bitcast an i64 load inserted into a vector to f64.
   // Otherwise, the i64 value will be legalized to a pair of i32 values.
   EVT VT = N->getValueType(0);
   SDNode *Elt = N->getOperand(1).getNode();
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ARMISD::VLD2DUP:
   case ARMISD::VLD3DUP:
   case ARMISD::VLD4DUP:
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
   case ARMISD::PREDICATE_CAST:
     return PerformPREDICATE_CASTCombine(N, DCI);
+  case ARMISD::VCMP:
+    return PerformVCMPCombine(N, DCI, Subtarget);
   case ARMISD::SMULWB: {
     unsigned BitWidth = N->getValueType(0).getSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
     if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
       return SDValue();
     break;
   }
   case ARMISD::SMULWT: {

llvm/lib/Target/ARM/Utils/ARMBaseInfo.h

inline static CondCodes getOppositeCondition(CondCodes CC) {
   case HI: return LS;
   case LS: return HI;
   case GE: return LT;
   case LT: return GE;
   case GT: return LE;
   case LE: return GT;
   }
 }
+
+/// getSwappedCondition - assume the flags are set by MI(a,b), return
+/// the condition code if we modify the instructions such that flags are
+/// set by MI(b,a).
+inline static ARMCC::CondCodes getSwappedCondition(ARMCC::CondCodes CC) {
+  switch (CC) {
+  default: return ARMCC::AL;
+  case ARMCC::EQ: return ARMCC::EQ;
+  case ARMCC::NE: return ARMCC::NE;
+  case ARMCC::HS: return ARMCC::LS;
+  case ARMCC::LO: return ARMCC::HI;
+  case ARMCC::HI: return ARMCC::LO;
+  case ARMCC::LS: return ARMCC::HS;
+  case ARMCC::GE: return ARMCC::LE;
+  case ARMCC::LT: return ARMCC::GT;
+  case ARMCC::GT: return ARMCC::LT;
+  case ARMCC::LE: return ARMCC::GE;
+  }
+}
 } // end namespace ARMCC
 
 namespace ARMVCC {
   enum VPTCodes {
     None = 0,
     Then,
     Else
   };

llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f32 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <4 x float> %src, zeroinitializer
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
+; CHECK-MVEFP-NEXT:    vpt.f16 ge, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <8 x half> %src, zeroinitializer
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }

 ; CHECK-MVE-NEXT:    lsls r0, r2, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s1, s9, s5
 ; CHECK-MVE-NEXT:    lsls r0, r1, #31
 ; CHECK-MVE-NEXT:    vseleq.f32 s0, s8, s4
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <4 x float> zeroinitializer, %src
   %s = select <4 x i1> %c, <4 x float> %a, <4 x float> %b
   ret <4 x float> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp one <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 ge, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ueq <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }
 

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp ord <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }

 ; CHECK-MVE-NEXT:    vmov r0, s0
 ; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
 ; CHECK-MVE-NEXT:    vmov q0, q3
 ; CHECK-MVE-NEXT:    vpop {d8, d9}
 ; CHECK-MVE-NEXT:    bx lr
 ;
 ; CHECK-MVEFP-LABEL: vcmp_r_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, q3
-; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q3, q0
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q0, zr
+; CHECK-MVEFP-NEXT:    vcmpt.f32 gt, q0, zr
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
 entry:
   %c = fcmp uno <8 x half> zeroinitializer, %src
   %s = select <8 x i1> %c, <8 x half> %a, <8 x half> %b
   ret <8 x half> %s
 }