[AArch64][GlobalISel] Add post-legalizer lowering for NEON vector fcmps

[paquette]

Summary

This is roughly equivalent to the floating point portion of AArch64TargetLowering::LowerVSETCC. Main part that's missing is the v4s16 bit.

This also adds helpers equivalent to EmitVectorComparison, and changeVectorFPCCToAArch64CC. This moves changeFCMPPredToAArch64CC out of the selector into AArch64GlobalISelUtils for the sake of code reuse.

This is done in post-legalizer lowering with pseudos to simplify selection. The imported patterns end up handling selection for us this way.

[aemerson]

llvm/lib/Target/AArch64/AArch64Combine.td

183

apply functions don't need to return a value.

llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

859

Should assert that the type is a vector.

916

Why not just return the lambda directly instead of via Func?

948

When can this be None? Is there missing support that needs to be added later, or should this be more of an assert?

[paquette]

llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

948

Yeah, actually, I couldn't find any place where this actually makes sense in SDAG after a second look. This can probably all be simpler.

[paquette]

Address comments

[aemerson]

LGTM.

llvm/lib/Target/AArch64/AArch64Combine.td

 def bitfield_extract_from_sext_inreg : GICombineRule<
   (defs root:$root, build_fn_matchinfo:$info),
   (match (wip_match_opcode G_SEXT_INREG):$root,
     [{ return matchBitfieldExtractFromSExtInReg(*${root}, MRI, ${info}); }]),
   (apply [{ return Helper.applyBuildFn(*${root}, ${info}); }])>;
 
 def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg]>;
 
+def lower_vector_fcmp : GICombineRule<
+  (defs root:$root),
+  (match (wip_match_opcode G_FCMP):$root,
+    [{ return lowerVectorFCMP(*${root}, MRI, B); }]),
+  (apply [{}])>;
+
 // Post-legalization combines which should happen at all optimization levels.
 // (E.g. ones that facilitate matching for the selector) For example, matching
 // pseudos.
 def AArch64PostLegalizerLoweringHelper
     : GICombinerHelper<"AArch64GenPostLegalizerLoweringHelper",
                        [shuffle_vector_lowering, vashr_vlshr_imm,
-                        icmp_lowering, build_vector_lowering]> {
+                        icmp_lowering, build_vector_lowering,
+                        lower_vector_fcmp]> {
   let DisableRuleOption = "aarch64postlegalizerlowering-disable-rule";
 }
 
 // Post-legalization combines which are primarily optimizations.
 def AArch64PostLegalizerCombinerHelper
     : GICombinerHelper<"AArch64GenPostLegalizerCombinerHelper",
                        [copy_prop, erase_undef_store, combines_for_extload,
                         sext_trunc_sextload,
                         hoist_logic_op_with_same_opcode_hands,
                         redundant_and, xor_of_and_with_same_reg,
                         extractvecelt_pairwise_add, redundant_or,
                         mul_const, redundant_sext_inreg,
                         form_bitfield_extract, rotate_out_of_range]> {
   let DisableRuleOption = "aarch64postlegalizercombiner-disable-rule";
 }

llvm/lib/Target/AArch64/AArch64InstrGISel.td

def G_SITOF : AArch64GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$src);
 }
 def G_UITOF : AArch64GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type0:$src);
 }
 
+def G_FCMEQ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGE : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGT : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1, type1:$src2);
+  let hasSideEffects = 0;
+}
+
+def G_FCMEQZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGEZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMGTZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMLEZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
+def G_FCMLTZ : AArch64GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
 def : GINodeEquiv<G_REV16, AArch64rev16>;
 def : GINodeEquiv<G_REV32, AArch64rev32>;
 def : GINodeEquiv<G_REV64, AArch64rev64>;
 def : GINodeEquiv<G_UZP1, AArch64uzp1>;
 def : GINodeEquiv<G_UZP2, AArch64uzp2>;
 def : GINodeEquiv<G_ZIP1, AArch64zip1>;
 def : GINodeEquiv<G_ZIP2, AArch64zip2>;
 def : GINodeEquiv<G_DUP, AArch64dup>;
 def : GINodeEquiv<G_DUPLANE8, AArch64duplane8>;
 def : GINodeEquiv<G_DUPLANE16, AArch64duplane16>;
 def : GINodeEquiv<G_DUPLANE32, AArch64duplane32>;
 def : GINodeEquiv<G_DUPLANE64, AArch64duplane64>;
 def : GINodeEquiv<G_TRN1, AArch64trn1>;
 def : GINodeEquiv<G_TRN2, AArch64trn2>;
 def : GINodeEquiv<G_EXT, AArch64ext>;
 def : GINodeEquiv<G_VASHR, AArch64vashr>;
 def : GINodeEquiv<G_VLSHR, AArch64vlshr>;
 def : GINodeEquiv<G_SITOF, AArch64sitof>;
 def : GINodeEquiv<G_UITOF, AArch64uitof>;
 
+def : GINodeEquiv<G_FCMEQ, AArch64fcmeq>;
+def : GINodeEquiv<G_FCMGE, AArch64fcmge>;
+def : GINodeEquiv<G_FCMGT, AArch64fcmgt>;
+
+def : GINodeEquiv<G_FCMEQZ, AArch64fcmeqz>;
+def : GINodeEquiv<G_FCMGEZ, AArch64fcmgez>;
+def : GINodeEquiv<G_FCMGTZ, AArch64fcmgtz>;
+def : GINodeEquiv<G_FCMLEZ, AArch64fcmlez>;
+def : GINodeEquiv<G_FCMLTZ, AArch64fcmltz>;
+
 def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>;
 
 // These are patterns that we only use for GlobalISel via the importer.
 def : Pat<(f32 (fadd (vector_extract (v2f32 FPR64:$Rn), (i64 0)),
                      (vector_extract (v2f32 FPR64:$Rn), (i64 1)))),
            (f32 (FADDPv2i32p (v2f32 FPR64:$Rn)))>;
 
 let Predicates = [HasNEON] in {

llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.h

 //===- AArch64GlobalISelUtils.h ----------------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file APIs for AArch64-specific helper functions used in the GlobalISel
 /// pipeline.
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AARCH64_GISEL_AARCH64GLOBALISELUTILS_H
 #define LLVM_LIB_TARGET_AARCH64_GISEL_AARCH64GLOBALISELUTILS_H
-
+#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/Register.h"
-#include "MCTargetDesc/AArch64AddressingModes.h"
 #include "llvm/IR/InstrTypes.h"
 #include <cstdint>
 
 namespace llvm {
 
 namespace AArch64GISelUtils {
 
 /// \returns true if \p C is a legal immediate operand for an arithmetic

 /// Replace a G_MEMSET with a value of 0 with a G_BZERO instruction if it is
 /// supported and beneficial to do so.
 ///
 /// \note This only applies on Darwin.
 ///
 /// \returns true if \p MI was replaced with a G_BZERO.
 bool tryEmitBZero(MachineInstr &MI, MachineIRBuilder &MIRBuilder, bool MinSize);
 
+/// Find the AArch64 condition codes necessary to represent \p P for a scalar
+/// floating point comparison.
+///
+/// \param [out] CondCode is the first condition code.
+/// \param [out] CondCode2 is the second condition code if necessary.
+/// AArch64CC::AL otherwise.
+void changeFCMPPredToAArch64CC(const CmpInst::Predicate P,
+                               AArch64CC::CondCode &CondCode,
+                               AArch64CC::CondCode &CondCode2);
+
+/// Find the AArch64 condition codes necessary to represent \p P for a vector
+/// floating point comparison.
+///
+/// \param [out] CondCode - The first condition code.
+/// \param [out] CondCode2 - The second condition code if necessary.
+/// AArch64CC::AL otherwise.
+/// \param [out] Invert - True if the comparison must be inverted with a NOT.
+void changeVectorFCMPPredToAArch64CC(const CmpInst::Predicate P,
+                                     AArch64CC::CondCode &CondCode,
+                                     AArch64CC::CondCode &CondCode2,
+                                     bool &Invert);
+
 } // namespace AArch64GISelUtils
 } // namespace llvm
 
 #endif

llvm/lib/Target/AArch64/GISel/AArch64GlobalISelUtils.cpp

 //===- AArch64GlobalISelUtils.cpp --------------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file Implementations of AArch64-specific helper functions used in the
 /// GlobalISel pipeline.
 //===----------------------------------------------------------------------===//
 #include "AArch64GlobalISelUtils.h"
 #include "AArch64InstrInfo.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
 Optional<RegOrConstant>
 AArch64GISelUtils::getAArch64VectorSplat(const MachineInstr &MI,
                                          const MachineRegisterInfo &MRI) {
   if (auto Splat = getVectorSplat(MI, MRI))
bool AArch64GISelUtils::tryEmitBZero(MachineInstr &MI,
   MIRBuilder
       .buildInstr(TargetOpcode::G_BZERO, {},
                   {MI.getOperand(0), MI.getOperand(2)})
       .addImm(MI.getOperand(3).getImm())
       .addMemOperand(*MI.memoperands_begin());
   MI.eraseFromParent();
   return true;
 }
+
+void AArch64GISelUtils::changeFCMPPredToAArch64CC(
+    const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
+    AArch64CC::CondCode &CondCode2) {
+  CondCode2 = AArch64CC::AL;
+  switch (P) {
+  default:
+    llvm_unreachable("Unknown FP condition!");
+  case CmpInst::FCMP_OEQ:
+    CondCode = AArch64CC::EQ;
+    break;
+  case CmpInst::FCMP_OGT:
+    CondCode = AArch64CC::GT;
+    break;
+  case CmpInst::FCMP_OGE:
+    CondCode = AArch64CC::GE;
+    break;
+  case CmpInst::FCMP_OLT:
+    CondCode = AArch64CC::MI;
+    break;
+  case CmpInst::FCMP_OLE:
+    CondCode = AArch64CC::LS;
+    break;
+  case CmpInst::FCMP_ONE:
+    CondCode = AArch64CC::MI;
+    CondCode2 = AArch64CC::GT;
+    break;
+  case CmpInst::FCMP_ORD:
+    CondCode = AArch64CC::VC;
+    break;
+  case CmpInst::FCMP_UNO:
+    CondCode = AArch64CC::VS;
+    break;
+  case CmpInst::FCMP_UEQ:
+    CondCode = AArch64CC::EQ;
+    CondCode2 = AArch64CC::VS;
+    break;
+  case CmpInst::FCMP_UGT:
+    CondCode = AArch64CC::HI;
+    break;
+  case CmpInst::FCMP_UGE:
+    CondCode = AArch64CC::PL;
+    break;
+  case CmpInst::FCMP_ULT:
+    CondCode = AArch64CC::LT;
+    break;
+  case CmpInst::FCMP_ULE:
+    CondCode = AArch64CC::LE;
+    break;
+  case CmpInst::FCMP_UNE:
+    CondCode = AArch64CC::NE;
+    break;
+  }
+}
+
+void AArch64GISelUtils::changeVectorFCMPPredToAArch64CC(
+    const CmpInst::Predicate P, AArch64CC::CondCode &CondCode,
+    AArch64CC::CondCode &CondCode2, bool &Invert) {
+  Invert = false;
+  switch (P) {
+  default:
+    // Mostly the scalar mappings work fine.
+    changeFCMPPredToAArch64CC(P, CondCode, CondCode2);
+    break;
+  case CmpInst::FCMP_UNO:
+    Invert = true;
+    LLVM_FALLTHROUGH;
+  case CmpInst::FCMP_ORD:
+    CondCode = AArch64CC::MI;
+    CondCode2 = AArch64CC::GE;
+    break;
+  case CmpInst::FCMP_UEQ:
+  case CmpInst::FCMP_ULT:
+  case CmpInst::FCMP_ULE:
+  case CmpInst::FCMP_UGT:
+  case CmpInst::FCMP_UGE:
+    // All of the compare-mask comparisons are ordered, but we can switch
+    // between the two by a double inversion. E.g. ULE == !OGT.
+    Invert = true;
+    changeFCMPPredToAArch64CC(CmpInst::getInversePredicate(P), CondCode,
+                              CondCode2);
+    break;
+  }
+}

llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp

case CmpInst::ICMP_UGE:
     return AArch64CC::HS;
   case CmpInst::ICMP_ULT:
     return AArch64CC::LO;
   case CmpInst::ICMP_ULE:
     return AArch64CC::LS;
   }
 }
 
-static void changeFCMPPredToAArch64CC(CmpInst::Predicate P,
-                                      AArch64CC::CondCode &CondCode,
-                                      AArch64CC::CondCode &CondCode2) {
-  CondCode2 = AArch64CC::AL;
-  switch (P) {
-  default:
-    llvm_unreachable("Unknown FP condition!");
-  case CmpInst::FCMP_OEQ:
-    CondCode = AArch64CC::EQ;
-    break;
-  case CmpInst::FCMP_OGT:
-    CondCode = AArch64CC::GT;
-    break;
-  case CmpInst::FCMP_OGE:
-    CondCode = AArch64CC::GE;
-    break;
-  case CmpInst::FCMP_OLT:
-    CondCode = AArch64CC::MI;
-    break;
-  case CmpInst::FCMP_OLE:
-    CondCode = AArch64CC::LS;
-    break;
-  case CmpInst::FCMP_ONE:
-    CondCode = AArch64CC::MI;
-    CondCode2 = AArch64CC::GT;
-    break;
-  case CmpInst::FCMP_ORD:
-    CondCode = AArch64CC::VC;
-    break;
-  case CmpInst::FCMP_UNO:
-    CondCode = AArch64CC::VS;
-    break;
-  case CmpInst::FCMP_UEQ:
-    CondCode = AArch64CC::EQ;
-    CondCode2 = AArch64CC::VS;
-    break;
-  case CmpInst::FCMP_UGT:
-    CondCode = AArch64CC::HI;
-    break;
-  case CmpInst::FCMP_UGE:
-    CondCode = AArch64CC::PL;
-    break;
-  case CmpInst::FCMP_ULT:
-    CondCode = AArch64CC::LT;
-    break;
-  case CmpInst::FCMP_ULE:
-    CondCode = AArch64CC::LE;
-    break;
-  case CmpInst::FCMP_UNE:
-    CondCode = AArch64CC::NE;
-    break;
-  }
-}
-
 /// Return a register which can be used as a bit to test in a TB(N)Z.
 static Register getTestBitReg(Register Reg, uint64_t &Bit, bool &Invert,
                               MachineRegisterInfo &MRI) {
   assert(Reg.isValid() && "Expected valid register!");
   while (MachineInstr *MI = getDefIgnoringCopies(Reg, MRI)) {
     unsigned Opc = MI->getOpcode();
 
     if (!MI->getOperand(0).isReg() ||

llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

 /// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
 /// a G_ZIP, G_UZP, etc.
 ///
 /// General optimization combines should be handled by either the
 /// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
 ///
 //===----------------------------------------------------------------------===//
 
-#include "AArch64TargetMachine.h"
 #include "AArch64GlobalISelUtils.h"
+#include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
+#include "GISel/AArch64LegalizerInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
+#include "TargetInfo/AArch64TargetInfo.h"
+#include "Utils/AArch64BaseInfo.h"
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
 #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 
 #define DEBUG_TYPE "aarch64-postlegalizer-lowering"
 
 using namespace llvm;
 using namespace MIPatternMatch;
 using namespace AArch64GISelUtils;
 
 /// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
static bool applySwapICmpOperands(MachineInstr &MI,
   Observer.changedInstr(MI);
   MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));
   MI.getOperand(2).setReg(RHS);
   MI.getOperand(3).setReg(LHS);
   Observer.changedInstr(MI);
   return true;
 }
 
+/// \returns a function which builds a vector floating point compare instruction
+/// for a condition code \p CC.
+/// \param [in] IsZero - True if the comparison is against 0.
+/// \param [in] NoNans - True if the target has NoNansFPMath.
+static std::function<Register(MachineIRBuilder &)>
+getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool IsZero,
+              bool NoNans, MachineRegisterInfo &MRI) {
+  LLT DstTy = MRI.getType(LHS);
+  assert(DstTy.isVector() && "Expected vector types only?");
+  assert(DstTy == MRI.getType(RHS) && "Src and Dst types must match!");
+  switch (CC) {
+  default:
+    llvm_unreachable("Unexpected condition code!");
+  case AArch64CC::NE:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      auto FCmp = IsZero
+                      ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS})
+                      : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS});
+      return MIB.buildNot(DstTy, FCmp).getReg(0);
+    };
+  case AArch64CC::EQ:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMEQZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMEQ, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::GE:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMGEZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::GT:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMGTZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {LHS, RHS})
+                       .getReg(0);
+    };
+  case AArch64CC::LS:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMLEZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGE, {DstTy}, {RHS, LHS})
+                       .getReg(0);
+    };
+  case AArch64CC::MI:
+    return [LHS, RHS, IsZero, DstTy](MachineIRBuilder &MIB) {
+      return IsZero
+                 ? MIB.buildInstr(AArch64::G_FCMLTZ, {DstTy}, {LHS}).getReg(0)
+                 : MIB.buildInstr(AArch64::G_FCMGT, {DstTy}, {RHS, LHS})
+                       .getReg(0);
+    };
+  }
+}
+
+/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
+static bool lowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
+                            MachineIRBuilder &MIB) {
+  assert(MI.getOpcode() == TargetOpcode::G_FCMP);
+  const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
+  Register Dst = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(Dst);
+  if (!DstTy.isVector() || !ST.hasNEON())
+    return false;
+  const auto Pred =
+      static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
+  Register LHS = MI.getOperand(2).getReg();
+  // TODO: Handle v4s16 case.
+  unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
+  if (EltSize != 32 && EltSize != 64)
+    return false;
+  Register RHS = MI.getOperand(3).getReg();
+  auto Splat = getAArch64VectorSplat(*MRI.getVRegDef(RHS), MRI);
+
+  // Compares against 0 have special target-specific pseudos.
+  bool IsZero = Splat && Splat->isCst() && Splat->getCst() == 0;
+  bool Invert;
+  AArch64CC::CondCode CC, CC2;
+  changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);
+  bool NoNans = ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
+
+  // Instead of having an apply function, just build here to simplify things.
+  MIB.setInstrAndDebugLoc(MI);
+  auto Cmp = getVectorFCMP(CC, LHS, RHS, IsZero, NoNans, MRI);
+  Register CmpRes;
+  if (CC2 == AArch64CC::AL)
+    CmpRes = Cmp(MIB);
+  else {
+    auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans, MRI);
+    auto Cmp2Dst = Cmp2(MIB);
+    auto Cmp1Dst = Cmp(MIB);
+    CmpRes = MIB.buildOr(DstTy, Cmp1Dst, Cmp2Dst).getReg(0);
+  }
+  if (Invert)
+    CmpRes = MIB.buildNot(DstTy, CmpRes).getReg(0);
+  MRI.replaceRegWith(Dst, CmpRes);
+  MI.eraseFromParent();
+  return false;
+}
+
 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
 #include "AArch64GenPostLegalizeGILowering.inc"
 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
 
 namespace {
 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
 #include "AArch64GenPostLegalizeGILowering.inc"
 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H

llvm/test/CodeGen/AArch64/GlobalISel/lower-neon-vector-fcmp.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            oeq
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: oeq
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMEQ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oeq_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: oeq_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK: $q0 = COPY [[FCMEQZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+
+...
+---
+name:            ogt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ogt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ogt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ogt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: $q0 = COPY [[FCMGTZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oge
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: oge
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            oge_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: oge_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: $q0 = COPY [[FCMGEZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+
+...
+---
+name:            olt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: olt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            olt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: olt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: $q0 = COPY [[FCMLTZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ole
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ole
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ole_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+    ; CHECK-LABEL: name: ole_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK: $q0 = COPY [[FCMLEZ]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            one
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Two compares.
+
+    ; CHECK-LABEL: name: one
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT1:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT1]], [[FCMGT]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            one_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Two compares.
+
+    ; CHECK-LABEL: name: one_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGTZ]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uno
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uno
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uno_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uno_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ord
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Needs two compares. No invert.
+
+    ; CHECK-LABEL: name: ord
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ord_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Needs two compares. No invert.
+
+    ; CHECK-LABEL: name: ord_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
+    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ult
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ult
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ueq_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ueq_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGEZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ule
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ule
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ule_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ule_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGTZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ugt
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ugt
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ugt_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: ugt_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLEZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uge
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uge
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            uge_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Should be inverted. Needs two compares.
+
+    ; CHECK-LABEL: name: uge_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLTZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            une
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Negated EQ.
+
+    ; CHECK-LABEL: name: une
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %rhs:_(<2 x s64>) = COPY $q1
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %rhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            une_zero
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; Negated EQ.
+
+    ; CHECK-LABEL: name: une_zero
+    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQZ]], [[BUILD_VECTOR]]
+    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<2 x s64>) = COPY $q0
+    %zero:_(s64) = G_CONSTANT i64 0
+    %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
+    %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %zero_vec
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            dont_lower_s16
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: dont_lower_s16
+    ; CHECK: %lhs:_(<8 x s16>) = COPY $q0
+    ; CHECK: %rhs:_(<8 x s16>) = COPY $q1
+    ; CHECK: %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
+    ; CHECK: $q0 = COPY %fcmp(<8 x s16>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:_(<8 x s16>) = COPY $q0
+    %rhs:_(<8 x s16>) = COPY $q1
+    %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
+    $q0 = COPY %fcmp(<8 x s16>)
+    RET_ReallyLR implicit $q0

llvm/test/CodeGen/AArch64/GlobalISel/select-neon-vector-fcmp.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+...
+---
+name:            fcmeq
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmeq
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMEQv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmge
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmge
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMGEv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgt
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgt
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %rhs:fpr128 = COPY $q1
+    ; CHECK: %fcmp:fpr128 = FCMGTv2f64 %lhs, %rhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %rhs:fpr(<2 x s64>) = COPY $q1
+    %fcmp:fpr(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmeqz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmeqz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMEQv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMEQZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgez
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgez
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMGEv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMGEZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmgtz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmgtz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMGTv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMGTZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmlez
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmlez
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMLEv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMLEZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            fcmltz
+alignment:       4
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: fcmltz
+    ; CHECK: %lhs:fpr128 = COPY $q0
+    ; CHECK: %fcmp:fpr128 = FCMLTv2i64rz %lhs
+    ; CHECK: $q0 = COPY %fcmp
+    ; CHECK: RET_ReallyLR implicit $q0
+    %lhs:fpr(<2 x s64>) = COPY $q0
+    %zero:gpr(s64) = G_CONSTANT i64 0
+    %zero_vec:fpr(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    %fcmp:fpr(<2 x s64>) = G_FCMLTZ %lhs
+    $q0 = COPY %fcmp(<2 x s64>)
+    RET_ReallyLR implicit $q0

llvm/test/CodeGen/AArch64/neon-compare-instructions.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon -global-isel -global-isel-abort=2 %s -o - | FileCheck %s --check-prefix=GISEL
 
 define <8 x i8> @cmeq8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmeq8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmeq16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmeq16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmeq4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmeq4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmeq8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmeq8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmeq2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmeq2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmeq4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmeq4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmeq2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmeq2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeq2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmne8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmne8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmne16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmne16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmne4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmne4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmne8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmne8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmne2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmne2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmne4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmne4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmne2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmne2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmne2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmgt8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmgt8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmgt16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmgt16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmgt4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmgt4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmgt8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmgt8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmgt2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmgt2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmgt4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmgt4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmgt2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmgt2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgt2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <8 x i8> @cmlt8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmlt8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <16 x i8> @cmlt16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmlt16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <4 x i16> @cmlt4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmlt4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <8 x i16> @cmlt8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmlt8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <2 x i32> @cmlt2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmlt2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <4 x i32> @cmlt4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmlt4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LT implemented as GT, so check reversed operands.
 define <2 x i64> @cmlt2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmlt2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlt2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmge8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmge8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmge16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmge16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmge4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmge4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmge8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmge8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmge2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmge2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmge4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmge4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmge2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmge2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmge2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <8 x i8> @cmle8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmle8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <16 x i8> @cmle16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmle16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <4 x i16> @cmle4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmle4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <8 x i16> @cmle8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmle8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <2 x i32> @cmle2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmle2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <4 x i32> @cmle4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmle4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LE implemented as GE, so check reversed operands.
 define <2 x i64> @cmle2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmle2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmle2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmhi8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmhi8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmhi16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmhi16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmhi4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmhi4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmhi8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmhi8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmhi2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmhi2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmhi4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmhi4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmhi2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmhi2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhi2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <8 x i8> @cmlo8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmlo8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <16 x i8> @cmlo16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmlo16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <4 x i16> @cmlo4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmlo4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <8 x i16> @cmlo8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmlo8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <2 x i32> @cmlo2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmlo2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <4 x i32> @cmlo4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmlo4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <2 x i64> @cmlo2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmlo2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlo2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhi v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmhs8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmhs8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmhs16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmhs16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmhs4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmhs4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmhs8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmhs8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmhs2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmhs2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmhs4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmhs4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmhs2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmhs2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhs2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <8 x i8> @cmls8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmls8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i8> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <16 x i8> @cmls16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmls16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <16 x i8> %A, %B;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <4 x i16> @cmls4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmls4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i16> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <8 x i16> @cmls8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmls8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i16> %A, %B;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <2 x i32> @cmls2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmls2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i32> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <4 x i32> @cmls4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmls4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i32> %A, %B;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <2 x i64> @cmls2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmls2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmhs v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmls2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    cmhs v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i64> %A, %B;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmtst8xi8(<8 x i8> %A, <8 x i8> %B) {
 ; CHECK-LABEL: cmtst8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = and <8 x i8> %A, %B
   %tmp4 = icmp ne <8 x i8> %tmp3, zeroinitializer
   %tmp5 = sext <8 x i1> %tmp4 to <8 x i8>
   ret <8 x i8> %tmp5
 }
 
 define <16 x i8> @cmtst16xi8(<16 x i8> %A, <16 x i8> %B) {
 ; CHECK-LABEL: cmtst16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = and <16 x i8> %A, %B
   %tmp4 = icmp ne <16 x i8> %tmp3, zeroinitializer
   %tmp5 = sext <16 x i1> %tmp4 to <16 x i8>
   ret <16 x i8> %tmp5
 }
 
 define <4 x i16> @cmtst4xi16(<4 x i16> %A, <4 x i16> %B) {
 ; CHECK-LABEL: cmtst4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v2.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = and <4 x i16> %A, %B
   %tmp4 = icmp ne <4 x i16> %tmp3, zeroinitializer
   %tmp5 = sext <4 x i1> %tmp4 to <4 x i16>
   ret <4 x i16> %tmp5
 }
 
 define <8 x i16> @cmtst8xi16(<8 x i16> %A, <8 x i16> %B) {
 ; CHECK-LABEL: cmtst8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v2.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = and <8 x i16> %A, %B
   %tmp4 = icmp ne <8 x i16> %tmp3, zeroinitializer
   %tmp5 = sext <8 x i1> %tmp4 to <8 x i16>
   ret <8 x i16> %tmp5
 }
 
 define <2 x i32> @cmtst2xi32(<2 x i32> %A, <2 x i32> %B) {
 ; CHECK-LABEL: cmtst2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v2.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = and <2 x i32> %A, %B
   %tmp4 = icmp ne <2 x i32> %tmp3, zeroinitializer
   %tmp5 = sext <2 x i1> %tmp4 to <2 x i32>
   ret <2 x i32> %tmp5
 }
 
 define <4 x i32> @cmtst4xi32(<4 x i32> %A, <4 x i32> %B) {
 ; CHECK-LABEL: cmtst4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v2.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = and <4 x i32> %A, %B
   %tmp4 = icmp ne <4 x i32> %tmp3, zeroinitializer
   %tmp5 = sext <4 x i1> %tmp4 to <4 x i32>
   ret <4 x i32> %tmp5
 }
 
 define <2 x i64> @cmtst2xi64(<2 x i64> %A, <2 x i64> %B) {
 ; CHECK-LABEL: cmtst2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmtst2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v2.2d, #0000000000000000
+; GISEL-NEXT:    and v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v2.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = and <2 x i64> %A, %B
   %tmp4 = icmp ne <2 x i64> %tmp3, zeroinitializer
   %tmp5 = sext <2 x i1> %tmp4 to <2 x i64>
   ret <2 x i64> %tmp5
 }
 
 
 
 define <8 x i8> @cmeqz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmeqz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmeqz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmeqz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmeqz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmeqz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmeqz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmeqz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmeqz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmeqz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmeqz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmeqz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmeqz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmeqz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmeq v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmeqz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp eq <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 
 define <8 x i8> @cmgez8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmgez8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmgez16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmgez16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmgez4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmgez4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmgez8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmgez8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmgez2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmgez2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmgez4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmgez4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmgez2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmgez2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sge <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 
 define <8 x i8> @cmgez8xi8_alt(<8 x i8> %A) {
 ; CHECK-LABEL: cmgez8xi8_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi8_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <8 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   %not = xor <8 x i8> %sign, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   ret <8 x i8> %not
 }
 
 define <16 x i8> @cmgez16xi8_alt(<16 x i8> %A) {
 ; CHECK-LABEL: cmgez16xi8_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez16xi8_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <16 x i8> %A, <i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7, i8 7>
   %not = xor <16 x i8> %sign, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
   ret <16 x i8> %not
 }
 
 define <4 x i16> @cmgez4xi16_alt(<4 x i16> %A) {
 ; CHECK-LABEL: cmgez4xi16_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi16_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <4 x i16> %A, <i16 15, i16 15, i16 15, i16 15>
   %not = xor <4 x i16> %sign, <i16 -1, i16 -1, i16 -1, i16 -1>
   ret <4 x i16> %not
 }
 
 define <8 x i16> @cmgez8xi16_alt(<8 x i16> %A) {
 ; CHECK-LABEL: cmgez8xi16_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez8xi16_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <8 x i16> %A, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
   %not = xor <8 x i16> %sign, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   ret <8 x i16> %not
 }
 
 define <2 x i32> @cmgez2xi32_alt(<2 x i32> %A) {
 ; CHECK-LABEL: cmgez2xi32_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi32_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    ret
   %sign = ashr <2 x i32> %A, <i32 31, i32 31>
   %not = xor <2 x i32> %sign, <i32 -1, i32 -1>
   ret <2 x i32> %not
 }
 
 define <4 x i32> @cmgez4xi32_alt(<4 x i32> %A) {
 ; CHECK-LABEL: cmgez4xi32_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez4xi32_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <4 x i32> %A, <i32 31, i32 31, i32 31, i32 31>
   %not = xor <4 x i32> %sign, <i32 -1, i32 -1, i32 -1, i32 -1>
   ret <4 x i32> %not
 }
 
 define <2 x i64> @cmgez2xi64_alt(<2 x i64> %A) {
 ; CHECK-LABEL: cmgez2xi64_alt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgez2xi64_alt:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    ret
   %sign = ashr <2 x i64> %A, <i64 63, i64 63>
   %not = xor <2 x i64> %sign, <i64 -1, i64 -1>
   ret <2 x i64> %not
 }
 
 
 define <8 x i8> @cmgtz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmgtz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmgtz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmgtz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmgtz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmgtz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmgtz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmgtz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmgtz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmgtz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmgtz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmgtz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmgtz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmgtz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmgt v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmgtz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmlez8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmlez8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmlez16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmlez16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmlez4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmlez4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmlez8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmlez8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmlez2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmlez2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmlez4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmlez4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmlez2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmlez2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmle v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlez2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp sle <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmltz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmltz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.8b, v0.8b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmltz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmltz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.16b, v0.16b, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmltz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmltz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.4h, v0.4h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmltz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmltz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.8h, v0.8h, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmltz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmltz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmltz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmltz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.4s, v0.4s, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmltz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmltz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmlt v0.2d, v0.2d, #0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmltz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp slt <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmneqz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmneqz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmneqz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmneqz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.16b, v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmneqz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmneqz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4h, v0.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmneqz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmneqz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.8h, v0.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmneqz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmneqz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2s, v0.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmneqz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmneqz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.4s, v0.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmneqz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmneqz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmtst v0.2d, v0.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmneqz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ne <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <8 x i8> @cmhsz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmhsz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8b, #2
 ; CHECK-NEXT:    cmhs v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI126_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI126_0]
+; GISEL-NEXT:    cmhs v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmhsz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmhsz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #2
 ; CHECK-NEXT:    cmhs v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI127_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI127_0]
+; GISEL-NEXT:    cmhs v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <16 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmhsz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmhsz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4h, #2
 ; CHECK-NEXT:    cmhs v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI128_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI128_0]
+; GISEL-NEXT:    cmhs v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i16> %A, <i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmhsz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmhsz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #2
 ; CHECK-NEXT:    cmhs v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI129_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI129_0]
+; GISEL-NEXT:    cmhs v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <8 x i16> %A, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmhsz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmhsz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2s, #2
 ; CHECK-NEXT:    cmhs v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI130_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI130_0]
+; GISEL-NEXT:    cmhs v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i32> %A, <i32 2, i32 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmhsz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmhsz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #2
 ; CHECK-NEXT:    cmhs v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI131_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI131_0]
+; GISEL-NEXT:    cmhs v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <4 x i32> %A, <i32 2, i32 2, i32 2, i32 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmhsz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmhsz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhs v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhsz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI132_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI132_0]
+; GISEL-NEXT:    cmhs v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp uge <2 x i64> %A, <i64 2, i64 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 
 define <8 x i8> @cmhiz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmhiz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8b, #1
 ; CHECK-NEXT:    cmhi v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI133_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI133_0]
+; GISEL-NEXT:    cmhi v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i8> %A, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 define <16 x i8> @cmhiz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmhiz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #1
 ; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI134_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI134_0]
+; GISEL-NEXT:    cmhi v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <16 x i8> %A, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 define <4 x i16> @cmhiz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmhiz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4h, #1
 ; CHECK-NEXT:    cmhi v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI135_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI135_0]
+; GISEL-NEXT:    cmhi v0.4h, v0.4h, v1.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i16> %A, <i16 1, i16 1, i16 1, i16 1>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 define <8 x i16> @cmhiz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmhiz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #1
 ; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI136_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI136_0]
+; GISEL-NEXT:    cmhi v0.8h, v0.8h, v1.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <8 x i16> %A, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 define <2 x i32> @cmhiz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmhiz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2s, #1
 ; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI137_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI137_0]
+; GISEL-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i32> %A, <i32 1, i32 1>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @cmhiz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmhiz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #1
 ; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI138_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI138_0]
+; GISEL-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <4 x i32> %A, <i32 1, i32 1, i32 1, i32 1>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @cmhiz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmhiz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #1
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmhiz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI139_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI139_0]
+; GISEL-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ugt <2 x i64> %A, <i64 1, i64 1>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <8 x i8> @cmlsz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmlsz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i8> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <16 x i8> @cmlsz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmlsz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <16 x i8> %A, zeroinitializer;
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <4 x i16> @cmlsz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmlsz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i16> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <8 x i16> @cmlsz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmlsz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <8 x i16> %A, zeroinitializer;
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <2 x i32> @cmlsz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmlsz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i32> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <4 x i32> @cmlsz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmlsz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <4 x i32> %A, zeroinitializer;
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LS implemented as HS, so check reversed operands.
 define <2 x i64> @cmlsz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmlsz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    cmhs v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmlsz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    movi v1.2d, #0000000000000000
+; GISEL-NEXT:    cmhs v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ule <2 x i64> %A, zeroinitializer;
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <8 x i8> @cmloz8xi8(<8 x i8> %A) {
 ; CHECK-LABEL: cmloz8xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8b, #2
 ; CHECK-NEXT:    cmhi v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz8xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI147_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI147_0]
+; GISEL-NEXT:    cmhi v0.8b, v1.8b, v0.8b
+; GISEL-NEXT:    shl v0.8b, v0.8b, #7
+; GISEL-NEXT:    sshr v0.8b, v0.8b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i8>
   ret <8 x i8> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <16 x i8> @cmloz16xi8(<16 x i8> %A) {
 ; CHECK-LABEL: cmloz16xi8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #2
 ; CHECK-NEXT:    cmhi v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz16xi8:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI148_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI148_0]
+; GISEL-NEXT:    cmhi v0.16b, v1.16b, v0.16b
+; GISEL-NEXT:    shl v0.16b, v0.16b, #7
+; GISEL-NEXT:    sshr v0.16b, v0.16b, #7
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <16 x i8> %A, <i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2, i8 2>
   %tmp4 = sext <16 x i1> %tmp3 to <16 x i8>
   ret <16 x i8> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <4 x i16> @cmloz4xi16(<4 x i16> %A) {
 ; CHECK-LABEL: cmloz4xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4h, #2
 ; CHECK-NEXT:    cmhi v0.4h, v1.4h, v0.4h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz4xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI149_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI149_0]
+; GISEL-NEXT:    cmhi v0.4h, v1.4h, v0.4h
+; GISEL-NEXT:    shl v0.4h, v0.4h, #15
+; GISEL-NEXT:    sshr v0.4h, v0.4h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i16> %A, <i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i16>
   ret <4 x i16> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <8 x i16> @cmloz8xi16(<8 x i16> %A) {
 ; CHECK-LABEL: cmloz8xi16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #2
 ; CHECK-NEXT:    cmhi v0.8h, v1.8h, v0.8h
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz8xi16:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI150_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI150_0]
+; GISEL-NEXT:    cmhi v0.8h, v1.8h, v0.8h
+; GISEL-NEXT:    shl v0.8h, v0.8h, #15
+; GISEL-NEXT:    sshr v0.8h, v0.8h, #15
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <8 x i16> %A, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
   %tmp4 = sext <8 x i1> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <2 x i32> @cmloz2xi32(<2 x i32> %A) {
 ; CHECK-LABEL: cmloz2xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.2s, #2
 ; CHECK-NEXT:    cmhi v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz2xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI151_0
+; GISEL-NEXT:    ldr d1, [x8, :lo12:.LCPI151_0]
+; GISEL-NEXT:    cmhi v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i32> %A, <i32 2, i32 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <4 x i32> @cmloz4xi32(<4 x i32> %A) {
 ; CHECK-LABEL: cmloz4xi32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #2
 ; CHECK-NEXT:    cmhi v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz4xi32:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI152_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI152_0]
+; GISEL-NEXT:    cmhi v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <4 x i32> %A, <i32 2, i32 2, i32 2, i32 2>
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; LO implemented as HI, so check reversed operands.
 define <2 x i64> @cmloz2xi64(<2 x i64> %A) {
 ; CHECK-LABEL: cmloz2xi64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #2
 ; CHECK-NEXT:    dup v1.2d, x8
 ; CHECK-NEXT:    cmhi v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: cmloz2xi64:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    adrp x8, .LCPI153_0
+; GISEL-NEXT:    ldr q1, [x8, :lo12:.LCPI153_0]
+; GISEL-NEXT:    cmhi v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = icmp ult <2 x i64> %A, <i64 2, i64 2>
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmoeq2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmoeq2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmoeq4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmoeq4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmoeq2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmoeq2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeq2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmoge2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmoge2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmoge4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmoge4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmoge2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmoge2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoge2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmogt2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmogt2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmogt4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmogt4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmogt2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmogt2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <2 x i32> @fcmole2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmole2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <4 x i32> @fcmole4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmole4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <2 x i64> @fcmole2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmole2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmole2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <2 x i32> @fcmolt2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmolt2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <4 x i32> @fcmolt4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmolt4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; OLE implemented as OGE, so check reversed operands.
 define <2 x i64> @fcmolt2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmolt2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ONE = OGT | OLT, OLT implemented as OGT so check reversed operands
 define <2 x i32> @fcmone2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmone2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ONE = OGT | OLT, OLT implemented as OGT so check reversed operands
 define <4 x i32> @fcmone4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmone4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ONE = OGT | OLT, OLT implemented as OGT so check reversed operands
 ; todo check reversed operands
 define <2 x i64> @fcmone2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmone2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmone2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ORD = OGE | OLT, OLT implemented as OGT, so check reversed operands.
 define <2 x i32> @fcmord2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmord2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ORD = OGE | OLT, OLT implemented as OGT, so check reversed operands.
 define <4 x i32> @fcmord4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmord4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ORD = OGE | OLT, OLT implemented as OGT, so check reversed operands.
 define <2 x i64> @fcmord2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmord2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmord2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 
 ; UNO = !(OGE | OLT), OLT implemented as OGT, so check reversed operands.
 define <2 x i32> @fcmuno2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmuno2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UNO = !(OGE | OLT), OLT implemented as OGT, so check reversed operands.
 define <4 x i32> @fcmuno4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmuno4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UNO = !(OGE | OLT), OLT implemented as OGT, so check reversed operands.
 define <2 x i64> @fcmuno2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmuno2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v2.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuno2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UEQ = !ONE = !(OGT | OLT), OLT implemented as OGT so check reversed operands
 define <2 x i32> @fcmueq2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmueq2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v2.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2s, v0.2s, v1.2s
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    orr v0.8b, v0.8b, v2.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UEQ = !ONE = !(OGT | OLT), OLT implemented as OGT so check reversed operands
 define <4 x i32> @fcmueq4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmueq4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.4s, v0.4s, v1.4s
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UEQ = !ONE = !(OGT | OLT), OLT implemented as OGT so check reversed operands
 define <2 x i64> @fcmueq2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmueq2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueq2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v2.2d, v0.2d, v1.2d
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    orr v0.16b, v0.16b, v2.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UGE = ULE with swapped operands, ULE implemented as !OGT.
 define <2 x i32> @fcmuge2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmuge2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UGE = ULE with swapped operands, ULE implemented as !OGT.
 define <4 x i32> @fcmuge4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmuge4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UGE = ULE with swapped operands, ULE implemented as !OGT.
 define <2 x i64> @fcmuge2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmuge2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmuge2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UGT = ULT with swapped operands, ULT implemented as !OGE.
 define <2 x i32> @fcmugt2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmugt2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v1.2s, v0.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v1.2s, v0.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UGT = ULT with swapped operands, ULT implemented as !OGE.
 define <4 x i32> @fcmugt4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmugt4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v1.4s, v0.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v1.4s, v0.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @fcmugt2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmugt2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v1.2d, v0.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugt2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v1.2d, v0.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ULE implemented as !OGT.
 define <2 x i32> @fcmule2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmule2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ULE implemented as !OGT.
 define <4 x i32> @fcmule4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmule4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ULE implemented as !OGT.
 define <2 x i64> @fcmule2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmule2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmule2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ULT implemented as !OGE.
 define <2 x i32> @fcmult2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmult2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ULT implemented as !OGE.
 define <4 x i32> @fcmult4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmult4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ULT implemented as !OGE.
 define <2 x i64> @fcmult2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmult2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmult2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UNE = !OEQ.
 define <2 x i32> @fcmune2xfloat(<2 x float> %A, <2 x float> %B) {
 ; CHECK-LABEL: fcmune2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v1.2s
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x float> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UNE = !OEQ.
 define <4 x i32> @fcmune4xfloat(<4 x float> %A, <4 x float> %B) {
 ; CHECK-LABEL: fcmune4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v1.4s
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <4 x float> %A, %B
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UNE = !OEQ.
 define <2 x i64> @fcmune2xdouble(<2 x double> %A, <2 x double> %B) {
 ; CHECK-LABEL: fcmune2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmune2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v1.2d
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x double> %A, %B
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmoeqz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmoeqz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmoeqz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmoeqz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmoeqz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmoeqz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoeqz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oeq <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 
 define <2 x i32> @fcmogez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmogez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmogez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmogez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmogez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmogez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp oge <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmogtz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmogtz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmogtz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmogtz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 define <2 x i64> @fcmogtz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmogtz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmogtz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ogt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmoltz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmoltz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmoltz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmoltz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @fcmoltz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmoltz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmoltz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp olt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmolez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmolez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmolez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmolez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @fcmolez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmolez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmolez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ole <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ONE with zero = OLT | OGT
 define <2 x i32> @fcmonez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmonez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ONE with zero = OLT | OGT
 define <4 x i32> @fcmonez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmonez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ONE with zero = OLT | OGT
 define <2 x i64> @fcmonez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmonez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmonez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp one <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ORD with zero = OLT | OGE
 define <2 x i32> @fcmordz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmordz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ORD with zero = OLT | OGE
 define <4 x i32> @fcmordz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmordz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ORD with zero = OLT | OGE
 define <2 x i64> @fcmordz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmordz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmordz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UEQ with zero = !ONE = !(OLT |OGT)
 define <2 x i32> @fcmueqz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmueqz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UEQ with zero = !ONE = !(OLT |OGT)
 define <4 x i32> @fcmueqz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmueqz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UEQ with zero = !ONE = !(OLT |OGT)
 define <2 x i64> @fcmueqz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmueqz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmueqz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ueq <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UGE with zero = !OLT
 define <2 x i32> @fcmugez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmugez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UGE with zero = !OLT
 define <4 x i32> @fcmugez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmugez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UGE with zero = !OLT
 define <2 x i64> @fcmugez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmugez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uge <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UGT with zero = !OLE
 define <2 x i32> @fcmugtz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmugtz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UGT with zero = !OLE
 define <4 x i32> @fcmugtz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmugtz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UGT with zero = !OLE
 define <2 x i64> @fcmugtz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmugtz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmle v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmugtz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmle v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ugt <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ULT with zero = !OGE
 define <2 x i32> @fcmultz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmultz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmultz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmultz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @fcmultz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmultz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmultz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ult <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; ULE with zero = !OGT
 define <2 x i32> @fcmulez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmulez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ULE with zero = !OGT
 define <4 x i32> @fcmulez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmulez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ULE with zero = !OGT
 define <2 x i64> @fcmulez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmulez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmgt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmulez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmgt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp ule <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UNE with zero = !OEQ with zero
 define <2 x i32> @fcmunez2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmunez2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UNE with zero = !OEQ with zero
 define <4 x i32> @fcmunez4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmunez4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UNE with zero = !OEQ with zero
 define <2 x i64> @fcmunez2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmunez2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmeq v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunez2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp une <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UNO with zero = !ORD = !(OLT | OGE)
 define <2 x i32> @fcmunoz2xfloat(<2 x float> %A) {
 ; CHECK-LABEL: fcmunoz2xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz2xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
+; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
+; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    mvn v0.8b, v0.8b
+; GISEL-NEXT:    shl v0.2s, v0.2s, #31
+; GISEL-NEXT:    sshr v0.2s, v0.2s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; UNO with zero = !ORD = !(OLT | OGE)
 define <4 x i32> @fcmunoz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmunoz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz4xfloat:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
+; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.4s, v0.4s, #31
+; GISEL-NEXT:    sshr v0.4s, v0.4s, #31
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; UNO with zero = !ORD = !(OLT | OGE)
 define <2 x i64> @fcmunoz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmunoz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    ret
+;
+; GISEL-LABEL: fcmunoz2xdouble:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
+; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
+; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    mvn v0.16b, v0.16b
+; GISEL-NEXT:    shl v0.2d, v0.2d, #63
+; GISEL-NEXT:    sshr v0.2d, v0.2d, #63
+; GISEL-NEXT:    ret
   %tmp3 = fcmp uno <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 
 }