diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -176,13 +176,20 @@
 
 def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg]>;
 
+def lower_vector_fcmp : GICombineRule<
+  (defs root:$root),
+  (match (wip_match_opcode G_FCMP):$root,
+    [{ return lowerVectorFCMP(*${root}, MRI, B); }]),
+  (apply [{}])>;
+
 // Post-legalization combines which should happen at all optimization levels.
 // (E.g. ones that facilitate matching for the selector) For example, matching
 // pseudos.
 def AArch64PostLegalizerLoweringHelper
     : GICombinerHelper<"AArch64GenPostLegalizerLoweringHelper",
                        [shuffle_vector_lowering, vashr_vlshr_imm,
-                        icmp_lowering, build_vector_lowering]> {
+                        icmp_lowering, build_vector_lowering,
+                        lower_vector_fcmp]> {
   let DisableRuleOption = "aarch64postlegalizerlowering-disable-rule";
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
--- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td
@@ -156,6 +156,54 @@
   let InOperandList = (ins type0:$src);
 }
 
+def G_FCMEQ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGE : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGT : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMEQZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGEZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGTZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMLEZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMLTZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
 def : GINodeEquiv<G_REV16, AArch64rev16>;
 def : GINodeEquiv<G_REV32, AArch64rev32>;
 def : GINodeEquiv<G_REV64, AArch64rev64>;
@@ -176,6 +224,16 @@
 def : GINodeEquiv<G_SITOF, AArch64sitof>;
 def : GINodeEquiv<G_UITOF, AArch64uitof>;
 
+def : GINodeEquiv<G_FCMEQ, AArch64fcmeq>;
+def : GINodeEquiv<G_FCMGE, AArch64fcmge>;
+def : GINodeEquiv<G_FCMGT, AArch64fcmgt>;
+
+def : GINodeEquiv<G_FCMEQZ, AArch64fcmeqz>;
+def : GINodeEquiv<G_FCMGEZ, AArch64fcmgez>;
+def : GINodeEquiv<G_FCMGTZ, AArch64fcmgtz>;
+def : GINodeEquiv<G_FCMLEZ, AArch64fcmlez>;
+def : GINodeEquiv<G_FCMLTZ, AArch64fcmltz>;
+
 def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>;
 
 // These are patterns that we only use for GlobalISel via the importer.
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h
@@ -11,12 +11,12 @@
 
 #ifndef LLVM_LIB_TARGET_AARCH64_GISEL_AARCH64GLOBALISELUTILS_H
 #define LLVM_LIB_TARGET_AARCH64_GISEL_AARCH64GLOBALISELUTILS_H
-
+#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/Register.h"
-#include "MCTargetDesc/AArch64AddressingModes.h"
 #include "llvm/IR/InstrTypes.h"
 #include <cstdint>
 
@@ -53,6 +53,28 @@
 /// \returns true if \p MI was replaced with a G_BZERO.
 bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder, bool MinSize);
 
+/// Find the AArch64 condition codes necessary to represent \p P for a scalar
+/// floating point comparison.
+///
+/// \param [out] CondCode is the first condition code.
+/// \param [out] CondCode2 is the second condition code if necessary.
+/// AArch64CC::AL otherwise.
+void changeFCMPPredToAArch64CC(const CmpInst::Predicate P,
+                               AArch64CC::CondCode &CondCode,
+                               AArch64CC::CondCode &CondCode2);
+
+/// Find the AArch64 condition codes necessary to represent \p P for a vector
+/// floating point comparison.
+///
+/// \param [out] CondCode - The first condition code.
+/// \param [out] CondCode2 - The second condition code if necessary.
+/// AArch64CC::AL otherwise.
+/// \param [out] Invert - True if the comparison must be inverted with a NOT.
+void changeVectorFCMPPredToAArch64CC(const CmpInst::Predicate P,
+                                     AArch64CC::CondCode &CondCode,
+                                     AArch64CC::CondCode &CondCode2,
+                                     bool &Invert);
+
 } // namespace AArch64GISelUtils
 } // namespace llvm
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
--- a/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp
@@ -12,6 +12,7 @@
 #include "AArch64InstrInfo.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -93,3 +94,87 @@
   MI.eraseFromParent();
   return true;
 }
+
+void AArch64GISelUtils::changeFCMPPredToAArch64CC(
+    const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
+    AArch64CC::CondCode &CondCode2) {
+  CondCode2 = AArch64CC::AL;
+  switch (P) {
+  default:
+    llvm_unreachable("Unknown FP condition!");
+  case CmpInst::FCMP_OEQ:
+    CondCode = AArch64CC::EQ;
+    break;
+  case CmpInst::FCMP_OGT:
+    CondCode = AArch64CC::GT;
+    break;
+  case CmpInst::FCMP_OGE:
+    CondCode = AArch64CC::GE;
+    break;
+  case CmpInst::FCMP_OLT:
+    CondCode = AArch64CC::MI;
+    break;
+  case CmpInst::FCMP_OLE:
+    CondCode = AArch64CC::LS;
+    break;
+  case CmpInst::FCMP_ONE:
+    CondCode = AArch64CC::MI;
+    CondCode2 = AArch64CC::GT;
+    break;
+  case CmpInst::FCMP_ORD:
+    CondCode = AArch64CC::VC;
+    break;
+  case CmpInst::FCMP_UNO:
+    CondCode = AArch64CC::VS;
+    break;
+  case CmpInst::FCMP_UEQ:
+    CondCode = AArch64CC::EQ;
+    CondCode2 = AArch64CC::VS;
+    break;
+  case CmpInst::FCMP_UGT:
+    CondCode = AArch64CC::HI;
+    break;
+  case CmpInst::FCMP_UGE:
+    CondCode = AArch64CC::PL;
+    break;
+  case CmpInst::FCMP_ULT:
+    CondCode = AArch64CC::LT;
+    break;
+  case CmpInst::FCMP_ULE:
+    CondCode = AArch64CC::LE;
+    break;
+  case CmpInst::FCMP_UNE:
+    CondCode = AArch64CC::NE;
+    break;
+  }
+}
+
+void AArch64GISelUtils::changeVectorFCMPPredToAArch64CC(
+    const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
+    AArch64CC::CondCode &CondCode2, bool &Invert) {
+  Invert = false;
+  switch (P) {
+  default:
+    // Mostly the scalar mappings work fine.
+    changeFCMPPredToAArch64CC(P, CondCode, CondCode2);
+    break;
+  case CmpInst::FCMP_UNO:
+    Invert = true;
+    LLVM_FALLTHROUGH;
+  case CmpInst::FCMP_ORD:
+    CondCode = AArch64CC::MI;
+    CondCode2 = AArch64CC::GE;
+    break;
+  case CmpInst::FCMP_UEQ:
+  case CmpInst::FCMP_ULT:
+  case CmpInst::FCMP_ULE:
+  case CmpInst::FCMP_UGT:
+  case CmpInst::FCMP_UGE:
+    // All of the compare-mask comparisons are ordered, but we can switch
+    // between the two by a double inversion. E.g. ULE == !OGT.
+    Invert = true;
+    changeFCMPPredToAArch64CC(CmpInst::getInversePredicate(P), CondCode,
+                              CondCode2);
+    break;
+  }
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1231,60 +1231,6 @@
   }
 }
 
-static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
-                                      AArch64CC::CondCode &CondCode,
-                                      AArch64CC::CondCode &CondCode2) {
-  CondCode2 = AArch64CC::AL;
-  switch (P) {
-  default:
-    llvm_unreachable("Unknown FP condition!");
-  case CmpInst::FCMP_OEQ:
-    CondCode = AArch64CC::EQ;
-    break;
-  case CmpInst::FCMP_OGT:
-    CondCode = AArch64CC::GT;
-    break;
-  case CmpInst::FCMP_OGE:
-    CondCode = AArch64CC::GE;
-    break;
-  case CmpInst::FCMP_OLT:
-    CondCode = AArch64CC::MI;
-    break;
-  case CmpInst::FCMP_OLE:
-    CondCode = AArch64CC::LS;
-    break;
-  case CmpInst::FCMP_ONE:
-    CondCode = AArch64CC::MI;
-    CondCode2 = AArch64CC::GT;
-    break;
-  case CmpInst::FCMP_ORD:
-    CondCode = AArch64CC::VC;
-    break;
-  case CmpInst::FCMP_UNO:
-    CondCode = AArch64CC::VS;
-    break;
-  case CmpInst::FCMP_UEQ:
-    CondCode = AArch64CC::EQ;
-    CondCode2 = AArch64CC::VS;
-    break;
-  case CmpInst::FCMP_UGT:
-    CondCode = AArch64CC::HI;
-    break;
-  case CmpInst::FCMP_UGE:
-    CondCode = AArch64CC::PL;
-    break;
-  case CmpInst::FCMP_ULT:
-    CondCode = AArch64CC::LT;
-    break;
-  case CmpInst::FCMP_ULE:
-    CondCode = AArch64CC::LE;
-    break;
-  case CmpInst::FCMP_UNE:
-    CondCode = AArch64CC::NE;
-    break;
-  }
-}
-
 /// Return a register which can be used as a bit to test in a TB(N)Z.
 static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
                               MachineRegisterInfo &MRI) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -19,9 +19,13 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "AArch64TargetMachine.h"
 #include "AArch64GlobalISelUtils.h"
+#include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
+#include "GISel/AArch64LegalizerInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
+#include "TargetInfo/AArch64TargetInfo.h"
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
 #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
@@ -33,8 +37,10 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 
 #define DEBUG_TYPE "aarch64-postlegalizer-lowering"
 
@@ -842,6 +848,109 @@
   return true;
 }
 
+/// \returns a function which builds a vector floating point compare instruction
+/// for a condition code \p CC.
+/// \param [in] IsZero - True if the comparison is against 0.
+/// \param [in] NoNans - True if the target has NoNansFPMath.
+static std::function<Register(MachineIRBuilder &)>
+getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool IsZero,
+              bool NoNans, MachineRegisterInfo &MRI) {
+  LLT DstTy = MRI.getType(LHS);
+  assert(DstTy.isVector() && "Expected vector types only?");
+  assert(DstTy == MRI.getType(RHS) && "Src and Dst types must match!");
+  switch (CC) {
+  default:
+    llvm_unreachable("Unexpected condition code!");
+  case AArch64CC::NE:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      auto FCmp = IsZero
+                      ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS})
+                      : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS});
+      return MIB.buildNot(DstTy, FCmp).getReg(0);
+    };
+  case AArch64CC::EQ:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::GE:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMGEZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::GT:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMGTZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::LS:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMLEZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {RHS, LHS})
+                       .getReg(0);
+    };
+  case AArch64CC::MI:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMLTZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {RHS, LHS})
+                       .getReg(0);
+    };
+  }
+}
+
+/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
+static bool lowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
+                            MachineIRBuilder &MIB) {
+  assert(MI.getOpcode() == TargetOpcode::G_FCMP);
+  const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
+  Register Dst = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(Dst);
+  if (!DstTy.isVector() || !ST.hasNEON())
+    return false;
+  const auto Pred =
+      static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
+  Register LHS = MI.getOperand(2).getReg();
+  // TODO: Handle v4s16 case.
+  unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
+  if (EltSize != 32 && EltSize != 64)
+    return false;
+  Register RHS = MI.getOperand(3).getReg();
+  auto Splat = getAArch64VectorSplat(*MRI.getVRegDef(RHS), MRI);
+
+  // Compares against 0 have special target-specific pseudos.
+  bool IsZero = Splat && Splat->isCst() && Splat->getCst() == 0;
+  bool Invert;
+  AArch64CC::CondCode CC, CC2;
+  changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);
+  bool NoNans = ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
+
+  // Instead of having an apply function, just build here to simplify things.
+  MIB.setInstrAndDebugLoc(MI);
+  auto Cmp = getVectorFCMP(CC, LHS, RHS, IsZero, NoNans, MRI);
+  Register CmpRes;
+  if (CC2 == AArch64CC::AL)
+    CmpRes = Cmp(MIB);
+  else {
+    auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans, MRI);
+    auto Cmp2Dst = Cmp2(MIB);
+    auto Cmp1Dst = Cmp(MIB);
+    CmpRes = MIB.buildOr(DstTy, Cmp1Dst, Cmp2Dst).getReg(0);
+  }
+  if (Invert)
+    CmpRes = MIB.buildNot(DstTy, CmpRes).getReg(0);
+  MRI.replaceRegWith(Dst, CmpRes);
+  MI.eraseFromParent();
+  return false;
+}
+
 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
 #include "AArch64GenPostLegalizeGILowering.inc"
 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/lower-neon-vector-fcmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/lower-neon-vector-fcmp.mir
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/lower-neon-vector-fcmp.mir
@@ -0,0 +1,672 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            oeq
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: oeq
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMEQ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oeq_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: oeq_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK: $q0 = COPY [[FCMEQZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+
+...
+---
+name:            ogt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ogt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ogt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ogt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: $q0 = COPY [[FCMGTZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oge
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: oge
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oge_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: oge_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: $q0 = COPY [[FCMGEZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+
+...
+---
+name:            olt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: olt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            olt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: olt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: $q0 = COPY [[FCMLTZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ole
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ole
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ole_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ole_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK: $q0 = COPY [[FCMLEZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            one
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Two compares.
+
+    ; CHECK-LABEL: name: one
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT1:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT1]], [[FCMGT]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            one_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Two compares.
+
+    ; CHECK-LABEL: name: one_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGTZ]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uno
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uno
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uno_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uno_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ord
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Needs two compares. No invert.
+
+    ; CHECK-LABEL: name: ord
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ord_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Needs two compares. No invert.
+
+    ; CHECK-LABEL: name: ord_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ult
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ult
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ueq_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ueq_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGEZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ule
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ule
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ule_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ule_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGTZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ugt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ugt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ugt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ugt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLEZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uge
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uge
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uge_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uge_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLTZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            une
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Negated EQ.
+
+    ; CHECK-LABEL: name: une
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            une_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Negated EQ.
+
+    ; CHECK-LABEL: name: une_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            dont_lower_s16
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: dont_lower_s16
+    ; CHECK: %lhs:_(<8 x s16>) = COPY $q0
+    ; CHECK: %rhs:_(<8 x s16>) = COPY $q1
+    ; CHECK: %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
+    ; CHECK: $q0 = COPY %fcmp(<8 x s16>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<8 x s16>) = COPY $q0
+    %rhs:_(<8 x s16>) = COPY $q1
+    %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
+    $q0 = COPY %fcmp(<8 x s16>)
+    RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-neon-vector-fcmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-neon-vector-fcmp.mir
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-neon-vector-fcmp.mir
@@ -0,0 +1,162 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name:            fcmeq
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmeq
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMEQv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmge
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmge
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMGEv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgt
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgt
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMGTv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmeqz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmeqz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMEQv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMEQZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgez
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgez
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMGEv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMGEZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgtz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgtz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMGTv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMGTZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmlez
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmlez
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMLEv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMLEZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmltz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmltz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMLTv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMLTZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
--- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -1,11 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon -global-isel -global-isel-abort=2 %s -o - | FileCheck %s --check-prefix=GISEL
 
 define <8 x i8> @cmeq8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmeq8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -16,6 +24,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -26,6 +41,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -36,6 +58,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -46,6 +75,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -56,6 +92,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -66,6 +109,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -77,6 +127,14 @@
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -88,6 +146,14 @@
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -99,6 +165,14 @@
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -110,6 +184,14 @@
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -121,6 +203,14 @@
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -132,6 +222,14 @@
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -143,6 +241,14 @@
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -153,6 +259,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -163,6 +276,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -173,6 +293,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -183,6 +310,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -193,6 +327,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -203,6 +344,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -213,6 +361,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -224,6 +379,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -235,6 +397,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -246,6 +415,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -257,6 +433,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -268,6 +451,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -279,6 +469,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -290,6 +487,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -300,6 +504,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -310,6 +521,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -320,6 +538,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -330,6 +555,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -340,6 +572,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -350,6 +589,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -360,6 +606,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -371,6 +624,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -382,6 +642,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -393,6 +660,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -404,6 +678,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -415,6 +696,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -426,6 +714,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -437,6 +732,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -447,6 +749,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -457,6 +766,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -467,6 +783,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -477,6 +800,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -487,6 +817,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -497,6 +834,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -507,6 +851,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -518,6 +869,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -529,6 +887,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -540,6 +905,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -551,6 +923,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -562,6 +941,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -573,6 +959,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -584,6 +977,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -594,6 +994,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -604,6 +1011,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -614,6 +1028,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -624,6 +1045,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -634,6 +1062,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -644,6 +1079,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -654,6 +1096,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -665,6 +1114,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -676,6 +1132,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -687,6 +1150,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -698,6 +1168,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -709,6 +1186,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -720,6 +1204,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -731,6 +1222,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -741,6 +1239,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = and <8 x i8> %A, %B
   %tmp4 = icmp ne <8 x i8> %tmp3, zeroinitializer
   %tmp5 = sext <8 x i1> %tmp4 to <8 x i8>
@@ -752,6 +1260,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = and <16 x i8> %A, %B
   %tmp4 = icmp ne <16 x i8> %tmp3, zeroinitializer
   %tmp5 = sext <16 x i1> %tmp4 to <16 x i8>
@@ -763,6 +1281,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v2.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = and <4 x i16> %A, %B
   %tmp4 = icmp ne <4 x i16> %tmp3, zeroinitializer
   %tmp5 = sext <4 x i1> %tmp4 to <4 x i16>
@@ -774,6 +1302,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v2.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = and <8 x i16> %A, %B
   %tmp4 = icmp ne <8 x i16> %tmp3, zeroinitializer
   %tmp5 = sext <8 x i1> %tmp4 to <8 x i16>
@@ -785,6 +1323,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v2.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = and <2 x i32> %A, %B
   %tmp4 = icmp ne <2 x i32> %tmp3, zeroinitializer
   %tmp5 = sext <2 x i1> %tmp4 to <2 x i32>
@@ -796,6 +1344,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = and <4 x i32> %A, %B
   %tmp4 = icmp ne <4 x i32> %tmp3, zeroinitializer
   %tmp5 = sext <4 x i1> %tmp4 to <4 x i32>
@@ -807,6 +1365,16 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = and <2 x i64> %A, %B
   %tmp4 = icmp ne <2 x i64> %tmp3, zeroinitializer
   %tmp5 = sext <2 x i1> %tmp4 to <2 x i64>
@@ -820,6 +1388,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -830,6 +1406,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -840,6 +1424,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -850,6 +1442,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -860,6 +1460,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -870,6 +1478,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -880,6 +1496,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -891,6 +1515,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -901,6 +1533,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -911,6 +1551,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -921,6 +1569,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -931,6 +1587,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -941,6 +1605,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -951,6 +1623,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -962,6 +1642,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi8_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <8 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   %not = xor <8 x i8> %sign, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   ret <8 x i8> %not
@@ -972,6 +1658,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez16xi8_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <16 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   %not = xor <16 x i8> %sign, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   ret <16 x i8> %not
@@ -982,6 +1674,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi16_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <4 x i16> %A, <i16 15, i16 15, i16 15, i16 15>
   %not = xor <4 x i16> %sign, <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %not
@@ -992,6 +1690,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi16_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <8 x i16> %A, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
   %not = xor <8 x i16> %sign, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   ret <8 x i16> %not
@@ -1002,6 +1706,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi32_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <2 x i32> %A, <i32 31, i32 31>
   %not = xor <2 x i32> %sign, <i32 -1, i32 -1>
   ret <2 x i32> %not
@@ -1012,6 +1722,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi32_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <4 x i32> %A, <i32 31, i32 31, i32 31, i32 31>
   %not = xor <4 x i32> %sign, <i32 -1, i32 -1, i32 -1, i32 -1>
   ret <4 x i32> %not
@@ -1022,6 +1738,12 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi64_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <2 x i64> %A, <i64 63, i64 63>
   %not = xor <2 x i64> %sign, <i64 -1, i64 -1>
   ret <2 x i64> %not
@@ -1033,6 +1755,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1043,6 +1773,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1053,6 +1791,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1063,6 +1809,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1073,6 +1827,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1083,6 +1845,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1093,6 +1863,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1103,6 +1881,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1113,6 +1899,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1123,6 +1917,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1133,6 +1935,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1143,6 +1953,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1153,6 +1971,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1163,6 +1989,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1173,6 +2007,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1183,6 +2025,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1193,6 +2043,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1203,6 +2061,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1213,6 +2079,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1223,6 +2097,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1233,6 +2115,14 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1243,6 +2133,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1253,6 +2152,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1263,6 +2171,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4h, v0.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1273,6 +2190,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8h, v0.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1283,6 +2209,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2s, v0.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1293,6 +2228,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4s, v0.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1303,6 +2247,15 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2d, v0.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1314,6 +2267,15 @@
 ; CHECK-NEXT:    movi v1.8b, #2
 ; CHECK-NEXT:    cmhs v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI126_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI126_0]
+; GISEL-NEXT:    cmhs v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1325,6 +2287,15 @@
 ; CHECK-NEXT:    movi v1.16b, #2
 ; CHECK-NEXT:    cmhs v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI127_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI127_0]
+; GISEL-NEXT:    cmhs v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <16 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1336,6 +2307,15 @@
 ; CHECK-NEXT:    movi v1.4h, #2
 ; CHECK-NEXT:    cmhs v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI128_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI128_0]
+; GISEL-NEXT:    cmhs v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i16> %A, <i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1347,6 +2327,15 @@
 ; CHECK-NEXT:    movi v1.8h, #2
 ; CHECK-NEXT:    cmhs v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI129_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI129_0]
+; GISEL-NEXT:    cmhs v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i16> %A, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1358,6 +2347,15 @@
 ; CHECK-NEXT:    movi v1.2s, #2
 ; CHECK-NEXT:    cmhs v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI130_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI130_0]
+; GISEL-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i32> %A, <i32 2, i32 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1369,6 +2367,15 @@
 ; CHECK-NEXT:    movi v1.4s, #2
 ; CHECK-NEXT:    cmhs v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI131_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI131_0]
+; GISEL-NEXT:    cmhs v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i32> %A, <i32 2, i32 2, i32 2, i32 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1381,6 +2388,15 @@
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI132_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI132_0]
+; GISEL-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i64> %A, <i64 2, i64 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1393,6 +2409,15 @@
 ; CHECK-NEXT:    movi v1.8b, #1
 ; CHECK-NEXT:    cmhi v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI133_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI133_0]
+; GISEL-NEXT:    cmhi v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i8> %A, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1404,6 +2429,15 @@
 ; CHECK-NEXT:    movi v1.16b, #1
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI134_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI134_0]
+; GISEL-NEXT:    cmhi v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <16 x i8> %A, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1415,6 +2449,15 @@
 ; CHECK-NEXT:    movi v1.4h, #1
 ; CHECK-NEXT:    cmhi v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI135_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI135_0]
+; GISEL-NEXT:    cmhi v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i16> %A, <i16 1, i16 1, i16 1, i16 1>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1426,6 +2469,15 @@
 ; CHECK-NEXT:    movi v1.8h, #1
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI136_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI136_0]
+; GISEL-NEXT:    cmhi v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i16> %A, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1437,6 +2489,15 @@
 ; CHECK-NEXT:    movi v1.2s, #1
 ; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI137_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI137_0]
+; GISEL-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i32> %A, <i32 1, i32 1>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1448,6 +2509,15 @@
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI138_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI138_0]
+; GISEL-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i32> %A, <i32 1, i32 1, i32 1, i32 1>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1460,6 +2530,15 @@
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI139_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI139_0]
+; GISEL-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i64> %A, <i64 1, i64 1>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1472,6 +2551,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1484,6 +2571,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1496,6 +2591,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1508,6 +2611,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1520,6 +2631,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1532,6 +2651,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1544,6 +2671,14 @@
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1556,6 +2691,15 @@
 ; CHECK-NEXT:    movi v1.8b, #2
 ; CHECK-NEXT:    cmhi v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI147_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI147_0]
+; GISEL-NEXT:    cmhi v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
@@ -1568,6 +2712,15 @@
 ; CHECK-NEXT:    movi v1.16b, #2
 ; CHECK-NEXT:    cmhi v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI148_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI148_0]
+; GISEL-NEXT:    cmhi v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <16 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
@@ -1580,6 +2733,15 @@
 ; CHECK-NEXT:    movi v1.4h, #2
 ; CHECK-NEXT:    cmhi v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI149_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI149_0]
+; GISEL-NEXT:    cmhi v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i16> %A, <i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
@@ -1592,6 +2754,15 @@
 ; CHECK-NEXT:    movi v1.8h, #2
 ; CHECK-NEXT:    cmhi v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI150_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI150_0]
+; GISEL-NEXT:    cmhi v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i16> %A, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
@@ -1604,6 +2775,15 @@
 ; CHECK-NEXT:    movi v1.2s, #2
 ; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI151_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI151_0]
+; GISEL-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i32> %A, <i32 2, i32 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1616,6 +2796,15 @@
 ; CHECK-NEXT:    movi v1.4s, #2
 ; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI152_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI152_0]
+; GISEL-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i32> %A, <i32 2, i32 2, i32 2, i32 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1629,6 +2818,15 @@
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI153_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI153_0]
+; GISEL-NEXT:    cmhi v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i64> %A, <i64 2, i64 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1639,6 +2837,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1649,6 +2854,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1658,6 +2870,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1668,6 +2887,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1678,6 +2904,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1687,6 +2920,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1697,6 +2937,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1707,6 +2954,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1716,6 +2970,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1727,6 +2988,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1738,6 +3006,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1749,6 +3024,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1760,6 +3042,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1771,6 +3060,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1782,6 +3078,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1795,6 +3098,15 @@
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1808,6 +3120,15 @@
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1822,6 +3143,15 @@
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1835,6 +3165,15 @@
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1848,6 +3187,15 @@
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1861,6 +3209,15 @@
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1876,6 +3233,16 @@
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1890,6 +3257,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1904,6 +3281,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1918,6 +3305,16 @@
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1932,6 +3329,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1946,6 +3353,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1958,6 +3375,14 @@
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -1970,6 +3395,14 @@
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -1982,6 +3415,14 @@
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -1994,6 +3435,14 @@
 ; CHECK-NEXT:    fcmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2006,6 +3455,14 @@
 ; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2017,6 +3474,14 @@
 ; CHECK-NEXT:    fcmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2029,6 +3494,14 @@
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2041,6 +3514,14 @@
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2053,6 +3534,14 @@
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2065,6 +3554,14 @@
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2077,6 +3574,14 @@
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2089,6 +3594,14 @@
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2101,6 +3614,14 @@
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2113,6 +3634,14 @@
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2125,6 +3654,14 @@
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2135,6 +3672,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2145,6 +3689,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2154,6 +3705,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2165,6 +3723,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2175,6 +3740,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2184,6 +3756,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2194,6 +3773,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2204,6 +3790,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2213,6 +3806,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2223,6 +3823,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2233,6 +3840,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2243,6 +3857,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2253,6 +3874,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2263,6 +3891,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2273,6 +3908,13 @@
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2286,6 +3928,15 @@
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2299,6 +3950,15 @@
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2312,6 +3972,15 @@
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2325,6 +3994,15 @@
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2338,6 +4016,15 @@
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2351,6 +4038,15 @@
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2365,6 +4061,16 @@
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2379,6 +4085,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2393,6 +4109,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2405,6 +4131,14 @@
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2417,6 +4151,14 @@
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2429,6 +4171,14 @@
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2441,6 +4191,14 @@
 ; CHECK-NEXT:    fcmle v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2453,6 +4211,14 @@
 ; CHECK-NEXT:    fcmle v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2465,6 +4231,14 @@
 ; CHECK-NEXT:    fcmle v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2477,6 +4251,14 @@
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2488,6 +4270,14 @@
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2499,6 +4289,14 @@
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2511,6 +4309,14 @@
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2523,6 +4329,14 @@
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2535,6 +4349,14 @@
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2547,6 +4369,14 @@
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2559,6 +4389,14 @@
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2571,6 +4409,14 @@
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
@@ -2585,6 +4431,16 @@
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
@@ -2599,6 +4455,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
@@ -2613,6 +4479,16 @@
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4