[X86] Enable extended comparison predicate support for SETUEQ/SETONE when targeting AVX instructions.

[craig.topper]

Summary

If we are using AVX512 instructions I think we can assume that all the comparison predicates are supported since we don't have an AMD implementation to compare against?

Though maybe we proved that AMD really does implement all 32 comparisons correctly in AVX1/2? We are using 0xF to create all ones on AVX1 right?

[RKSimon]

Though maybe we proved that AMD really does implement all 32 comparisons correctly in AVX1/2? We are using 0xF to create all ones on AVX1 right?

Yes we proved that every AVX capable AMD machine (we tested Jaguar, Bulldozer, Excavator and Ryzen) correctly supported all comparison modes. Its the documentation that hasn't been fixed.

[craig.topper]

Do this for AVX instructions as well.

[RKSimon]

Possibly regenerate fast-isel-select-pseudo-cmov.ll before the patch?

LGTM whatever

lib/Target/X86/X86FastISel.cpp

getX86SSEConditionCode(CmpInst::Predicate Predicate) {
   case CmpInst::FCMP_OLE: CC = 2;          break;
   case CmpInst::FCMP_UNO: CC = 3;          break;
   case CmpInst::FCMP_UNE: CC = 4;          break;
   case CmpInst::FCMP_ULE: NeedSwap = true; LLVM_FALLTHROUGH;
   case CmpInst::FCMP_UGE: CC = 5;          break;
   case CmpInst::FCMP_ULT: NeedSwap = true; LLVM_FALLTHROUGH;
   case CmpInst::FCMP_UGT: CC = 6;          break;
   case CmpInst::FCMP_ORD: CC = 7;          break;
-  case CmpInst::FCMP_UEQ:
-  case CmpInst::FCMP_ONE: CC = 8;          break;
+  case CmpInst::FCMP_UEQ: CC = 8;          break;
+  case CmpInst::FCMP_ONE: CC = 12;         break;
   }
 
   return std::make_pair(CC, NeedSwap);
 }
 
 /// \brief Adds a complex addressing mode to the given machine instr builder.
 /// Note, this will constrain the index register.  If its not possible to
 /// constrain the given index register, then a new one will be created.  The
if (Predicate == CmpInst::FCMP_ORD || Predicate == CmpInst::FCMP_UNO) {
     const auto *CmpRHSC = dyn_cast<ConstantFP>(CmpRHS);
     if (CmpRHSC && CmpRHSC->isNullValue())
       CmpRHS = CmpLHS;
   }
 
   unsigned CC;
   bool NeedSwap;
   std::tie(CC, NeedSwap) = getX86SSEConditionCode(Predicate);
-  if (CC > 7)
+  if (CC > 7 && !Subtarget->hasAVX())
     return false;
 
   if (NeedSwap)
     std::swap(CmpLHS, CmpRHS);
 
   // Choose the SSE instruction sequence based on data type (float or double).
   static const uint16_t OpcTable[2][4] = {
     { X86::CMPSSrr,  X86::ANDPSrr,  X86::ANDNPSrr,  X86::ORPSrr  },

lib/Target/X86/X86ISelLowering.cpp

static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
   if (LHS.getNode())
     return getBitTestCondition(LHS, RHS, CC, dl, DAG);
 
   return SDValue();
 }
 
 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
 /// CMPs.
-static int translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
+static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
                                    SDValue &Op1) {
   unsigned SSECC;
   bool Swap = false;
 
   // SSE Condition code mapping:
   //  0 - EQ
   //  1 - LT
   //  2 - LE
   //  3 - UNORD
static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
   case ISD::SETUO:  SSECC = 3; break;
   case ISD::SETUNE:
   case ISD::SETNE:  SSECC = 4; break;
   case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH;
   case ISD::SETUGE: SSECC = 5; break;
   case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH;
   case ISD::SETUGT: SSECC = 6; break;
   case ISD::SETO:   SSECC = 7; break;
-  case ISD::SETUEQ:
-  case ISD::SETONE: SSECC = 8; break;
+  case ISD::SETUEQ: SSECC = 8; break;
+  case ISD::SETONE: SSECC = 12; break;
   }
   if (Swap)
     std::swap(Op0, Op1);
 
   return SSECC;
 }
 
 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1) {
       // them to work with an SSE1 target (integer vector types are not legal).
       VT = Op0.getSimpleValueType();
     }
 
     // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
     // emit two comparisons and a logic op to tie them together.
     // TODO: This can be avoided if Intel (and only Intel as of 2016) AVX is
     // available.
+    // If we are using an AVX512 instruction we can use the extended predicate.
     SDValue Cmp;
     unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1);
-    if (SSECC == 8) {
+    if (SSECC >= 8 && !Subtarget.hasAVX()) {
       // LLVM predicate is SETUEQ or SETONE.
       unsigned CC0, CC1;
       unsigned CombineOpc;
       if (Cond == ISD::SETUEQ) {
         CC0 = 3; // UNORD
         CC1 = 0; // EQ
         CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FOR) :
                                            static_cast<unsigned>(ISD::OR);
SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
   // are available or VBLENDV if AVX is available.
   // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
   if (Cond.getOpcode() == ISD::SETCC &&
       ((Subtarget.hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) ||
        (Subtarget.hasSSE1() && VT == MVT::f32)) &&
       VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
     SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
-    int SSECC = translateX86FSETCC(
+    unsigned SSECC = translateX86FSETCC(
         cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1);
 
-    if (SSECC != 8) {
-      if (Subtarget.hasAVX512()) {
-        SDValue Cmp = DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0,
-                                  CondOp1, DAG.getConstant(SSECC, DL, MVT::i8));
-        assert(!VT.isVector() && "Not a scalar type?");
-        return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
-      }
 
+    if (SSECC < 8 || Subtarget.hasAVX()) {
       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
                                 DAG.getConstant(SSECC, DL, MVT::i8));
 
       // If we have AVX, we can use a variable vector select (VBLENDV) instead
       // of 3 logic instructions for size savings and potentially speed.
       // Unfortunately, there is no scalar form of VBLENDV.
 
       // If either operand is a constant, don't try this. We can expect to

test/CodeGen/X86/avx512-mov.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
   ret <8 x i64>%res
 }
 
 define <16 x float> @test40(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: test40:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @test41(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: test41:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @test42(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: test42:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <16 x float> @test43(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: test43:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <8 x double> @test44(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: test44:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x57,0xd2]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @test45(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: test45:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x57,0xd2]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @test46(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: test46:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x57,0xc9]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }
 
 define <8 x double> @test47(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: test47:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x57,0xc9]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }

test/CodeGen/X86/avx512-schedule.ll

; CHECK-NEXT:    retq # sched: [2:1.00]
   %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
   ret <8 x i64>%res
 }
 
 define <16 x float> @mov_test40(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test40:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test41(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test41:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test42(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test42:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test43(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test43:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <8 x double> @mov_test44(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test44:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test45(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test45:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test46(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test46:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test47(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test47:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }

test/CodeGen/X86/avx512vl-mov.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = select <4 x i1> %mask, <4 x i64> %r, <4 x i64> zeroinitializer
   ret <4 x i64>%res
 }
 
 define <8 x float> @test_256_25(i8 * %addr, <8 x float> %old, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_25:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; CHECK-NEXT:    vcmpordps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm2, %ymm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 32
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> %old
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_26(i8 * %addr, <8 x float> %old, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_26:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; CHECK-NEXT:    vcmpordps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm2, %ymm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> %old
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_27(i8 * %addr, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_27:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
-; CHECK-NEXT:    vcmpordps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 32
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> zeroinitializer
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_28(i8 * %addr, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_28:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
-; CHECK-NEXT:    vcmpordps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> zeroinitializer
   ret <8 x float>%res
 }

test/CodeGen/X86/commute-fcmp.ll

 ; SSE-NEXT:    cmpeqps %xmm0, %xmm2
 ; SSE-NEXT:    cmpunordps %xmm1, %xmm0
 ; SSE-NEXT:    orps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmpps_ueq:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovaps (%rdi), %xmm1
-; AVX-NEXT:    vcmpeqps %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vcmpunordps %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vorps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vcmpeq_uqps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_ueq:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %xmm1
-; AVX512-NEXT:    vcmpeqps %xmm0, %xmm1, %k0
-; AVX512-NEXT:    vcmpunordps %xmm0, %xmm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqps %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x float>, <4 x float>* %a0
   %2 = fcmp ueq <4 x float> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
   ret <4 x i32> %3
 }
 
 define <4 x i32> @commute_cmpps_one(<4 x float>* %a0, <4 x float> %a1) {
 ; SSE-LABEL: commute_cmpps_one:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    movaps (%rdi), %xmm1
 ; SSE-NEXT:    movaps %xmm1, %xmm2
 ; SSE-NEXT:    cmpneqps %xmm0, %xmm2
 ; SSE-NEXT:    cmpordps %xmm1, %xmm0
 ; SSE-NEXT:    andps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmpps_one:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovaps (%rdi), %xmm1
-; AVX-NEXT:    vcmpneqps %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vcmpordps %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vcmpneq_oqps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_one:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %xmm1
-; AVX512-NEXT:    vcmpordps %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vcmpneqps %xmm0, %xmm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqps %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x float>, <4 x float>* %a0
   %2 = fcmp one <4 x float> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
   ret <4 x i32> %3
 }

 ; SSE-NEXT:    cmpeqps %xmm1, %xmm2
 ; SSE-NEXT:    cmpunordps %xmm3, %xmm1
 ; SSE-NEXT:    orps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmpps_ueq_ymm:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovaps (%rdi), %ymm1
-; AVX-NEXT:    vcmpeqps %ymm0, %ymm1, %ymm2
-; AVX-NEXT:    vcmpunordps %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqps %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_ueq_ymm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %ymm1
-; AVX512-NEXT:    vcmpeqps %ymm0, %ymm1, %k0
-; AVX512-NEXT:    vcmpunordps %ymm0, %ymm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqps %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <8 x float>, <8 x float>* %a0
   %2 = fcmp ueq <8 x float> %1, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
   ret <8 x i32> %3
 }

 ; SSE-NEXT:    cmpneqps %xmm1, %xmm2
 ; SSE-NEXT:    cmpordps %xmm3, %xmm1
 ; SSE-NEXT:    andps %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmpps_one_ymm:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovaps (%rdi), %ymm1
-; AVX-NEXT:    vcmpneqps %ymm0, %ymm1, %ymm2
-; AVX-NEXT:    vcmpordps %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqps %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_one_ymm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %ymm1
-; AVX512-NEXT:    vcmpordps %ymm0, %ymm1, %k1
-; AVX512-NEXT:    vcmpneqps %ymm0, %ymm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqps %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <8 x float>, <8 x float>* %a0
   %2 = fcmp one <8 x float> %1, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
   ret <8 x i32> %3
 }

 ; SSE-NEXT:    cmpeqpd %xmm0, %xmm2
 ; SSE-NEXT:    cmpunordpd %xmm1, %xmm0
 ; SSE-NEXT:    orpd %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmppd_ueq:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovapd (%rdi), %xmm1
-; AVX-NEXT:    vcmpeqpd %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vcmpunordpd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vorpd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vcmpeq_uqpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_ueq:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %xmm1
-; AVX512-NEXT:    vcmpeqpd %xmm0, %xmm1, %k0
-; AVX512-NEXT:    vcmpunordpd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqpd %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <2 x double>, <2 x double>* %a0
   %2 = fcmp ueq <2 x double> %1, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
   ret <2 x i64> %3
 }
 
 define <2 x i64> @commute_cmppd_one(<2 x double>* %a0, <2 x double> %a1) {
 ; SSE-LABEL: commute_cmppd_one:
 ; SSE:       # BB#0:
 ; SSE-NEXT:    movapd (%rdi), %xmm1
 ; SSE-NEXT:    movapd %xmm1, %xmm2
 ; SSE-NEXT:    cmpneqpd %xmm0, %xmm2
 ; SSE-NEXT:    cmpordpd %xmm1, %xmm0
 ; SSE-NEXT:    andpd %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmppd_one:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovapd (%rdi), %xmm1
-; AVX-NEXT:    vcmpneqpd %xmm0, %xmm1, %xmm2
-; AVX-NEXT:    vcmpordpd %xmm0, %xmm1, %xmm0
-; AVX-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vcmpneq_oqpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_one:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %xmm1
-; AVX512-NEXT:    vcmpordpd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vcmpneqpd %xmm0, %xmm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqpd %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <2 x double>, <2 x double>* %a0
   %2 = fcmp one <2 x double> %1, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
   ret <2 x i64> %3
 }

 ; SSE-NEXT:    cmpeqpd %xmm1, %xmm2
 ; SSE-NEXT:    cmpunordpd %xmm3, %xmm1
 ; SSE-NEXT:    orpd %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmppd_ueq_ymmm:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovapd (%rdi), %ymm1
-; AVX-NEXT:    vcmpeqpd %ymm0, %ymm1, %ymm2
-; AVX-NEXT:    vcmpunordpd %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    vorpd %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_ueq_ymmm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %ymm1
-; AVX512-NEXT:    vcmpeqpd %ymm0, %ymm1, %k0
-; AVX512-NEXT:    vcmpunordpd %ymm0, %ymm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x double>, <4 x double>* %a0
   %2 = fcmp ueq <4 x double> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }

 ; SSE-NEXT:    cmpneqpd %xmm1, %xmm2
 ; SSE-NEXT:    cmpordpd %xmm3, %xmm1
 ; SSE-NEXT:    andpd %xmm2, %xmm1
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: commute_cmppd_one_ymmm:
 ; AVX:       # BB#0:
 ; AVX-NEXT:    vmovapd (%rdi), %ymm1
-; AVX-NEXT:    vcmpneqpd %ymm0, %ymm1, %ymm2
-; AVX-NEXT:    vcmpordpd %ymm0, %ymm1, %ymm0
-; AVX-NEXT:    vandpd %ymm2, %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_one_ymmm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %ymm1
-; AVX512-NEXT:    vcmpordpd %ymm0, %ymm1, %k1
-; AVX512-NEXT:    vcmpneqpd %ymm0, %ymm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x double>, <4 x double>* %a0
   %2 = fcmp one <4 x double> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }

test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll

-; RUN: llc < %s -mtriple=x86_64-apple-darwin10                                              | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1                  | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10                             -mcpu=corei7-avx | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 -mcpu=corei7-avx | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10                                              | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1                  | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10                             -mcpu=corei7-avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 -mcpu=corei7-avx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
 
 
 define float @select_fcmp_one_f32(float %a, float %b, float %c, float %d) {
-; CHECK-LABEL: select_fcmp_one_f32
-; CHECK:       ucomiss %xmm1, %xmm0
-; CHECK-NEXT:  jne [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  movaps %xmm2, %xmm0
+; SSE-LABEL: select_fcmp_one_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    ucomiss %xmm1, %xmm0
+; SSE-NEXT:    jne LBB0_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:  LBB0_2:
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_fcmp_one_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    vcmpneq_oqss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm0, %xmm2, %xmm3, %xmm0
+; AVX-NEXT:    retq
   %1 = fcmp one float %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define double @select_fcmp_one_f64(double %a, double %b, double %c, double %d) {
-; CHECK-LABEL: select_fcmp_one_f64
-; CHECK:       ucomisd %xmm1, %xmm0
-; CHECK-NEXT:  jne [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  movaps  %xmm2, %xmm0
+; SSE-LABEL: select_fcmp_one_f64:
+; SSE:       ## BB#0:
+; SSE-NEXT:    ucomisd %xmm1, %xmm0
+; SSE-NEXT:    jne LBB1_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:  LBB1_2:
+; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_fcmp_one_f64:
+; AVX:       ## BB#0:
+; AVX-NEXT:    vcmpneq_oqsd %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vblendvpd %xmm0, %xmm2, %xmm3, %xmm0
+; AVX-NEXT:    retq
   %1 = fcmp one double %a, %b
   %2 = select i1 %1, double %c, double %d
   ret double %2
 }
 
 define float @select_icmp_eq_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_eq_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  je [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_eq_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    je LBB2_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB2_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_eq_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    je LBB2_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB2_2:
+; AVX-NEXT:    retq
   %1 = icmp eq i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_ne_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_ne_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jne [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_ne_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jne LBB3_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB3_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_ne_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jne LBB3_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB3_2:
+; AVX-NEXT:    retq
   %1 = icmp ne i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_ugt_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_ugt_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  ja [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_ugt_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    ja LBB4_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB4_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_ugt_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    ja LBB4_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB4_2:
+; AVX-NEXT:    retq
   %1 = icmp ugt i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_uge_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_uge_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jae [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_uge_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jae LBB5_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB5_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_uge_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jae LBB5_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB5_2:
+; AVX-NEXT:    retq
   %1 = icmp uge i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_ult_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_ult_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jb [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_ult_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jb LBB6_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB6_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_ult_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jb LBB6_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB6_2:
+; AVX-NEXT:    retq
   %1 = icmp ult i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_ule_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_ule_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jbe [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_ule_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jbe LBB7_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB7_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_ule_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jbe LBB7_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB7_2:
+; AVX-NEXT:    retq
   %1 = icmp ule i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_sgt_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_sgt_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jg [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_sgt_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jg LBB8_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB8_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_sgt_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jg LBB8_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB8_2:
+; AVX-NEXT:    retq
   %1 = icmp sgt i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_sge_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_sge_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jge [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_sge_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jge LBB9_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB9_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_sge_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jge LBB9_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB9_2:
+; AVX-NEXT:    retq
   %1 = icmp sge i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_slt_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_slt_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jl [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_slt_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jl LBB10_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB10_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_slt_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jl LBB10_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB10_2:
+; AVX-NEXT:    retq
   %1 = icmp slt i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }
 
 define float @select_icmp_sle_f32(i64 %a, i64 %b, float %c, float %d) {
-; CHECK-LABEL: select_icmp_sle_f32
-; CHECK:       cmpq %rsi, %rdi
-; CHECK-NEXT:  jle [[BB:LBB[0-9]+_2]]
-; CHECK:       [[BB]]
-; CHECK-NEXT:  retq
+; SSE-LABEL: select_icmp_sle_f32:
+; SSE:       ## BB#0:
+; SSE-NEXT:    cmpq %rsi, %rdi
+; SSE-NEXT:    jle LBB11_2
+; SSE-NEXT:  ## BB#1:
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:  LBB11_2:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: select_icmp_sle_f32:
+; AVX:       ## BB#0:
+; AVX-NEXT:    cmpq %rsi, %rdi
+; AVX-NEXT:    jle LBB11_2
+; AVX-NEXT:  ## BB#1:
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:  LBB11_2:
+; AVX-NEXT:    retq
   %1 = icmp sle i64 %a, %b
   %2 = select i1 %1, float %c, float %d
   ret float %2
 }