[X86] Enable extended comparison predicate support for SETUEQ/SETONE when targeting AVX instructions.

[craig.topper]

Summary

If we are using AVX512 instructions I think we can assume that all the comparison predicates are supported since we don't have an AMD implementation to compare against?

Though maybe we proved that AMD really does implement all 32 comparisons correctly in AVX1/2? We are using 0xF to create all ones on AVX1 right?

[RKSimon]

Though maybe we proved that AMD really does implement all 32 comparisons correctly in AVX1/2? We are using 0xF to create all ones on AVX1 right?

Yes we proved that every AVX capable AMD machine (we tested Jaguar, Bulldozer, Excavator and Ryzen) correctly supported all comparison modes. Its the documentation that hasn't been fixed.

[craig.topper]

Do this for AVX instructions as well.

[RKSimon]

Possibly regenerate fast-isel-select-pseudo-cmov.ll before the patch?

LGTM whatever

lib/Target/X86/X86ISelLowering.cpp

static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
   if (LHS.getNode())
     return getBitTestCondition(LHS, RHS, CC, dl, DAG);
 
   return SDValue();
 }
 
 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
 /// CMPs.
-static int translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
+static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
                                    SDValue &Op1) {
   unsigned SSECC;
   bool Swap = false;
 
   // SSE Condition code mapping:
   //  0 - EQ
   //  1 - LT
   //  2 - LE
   //  3 - UNORD
static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
   case ISD::SETUO:  SSECC = 3; break;
   case ISD::SETUNE:
   case ISD::SETNE:  SSECC = 4; break;
   case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH;
   case ISD::SETUGE: SSECC = 5; break;
   case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH;
   case ISD::SETUGT: SSECC = 6; break;
   case ISD::SETO:   SSECC = 7; break;
-  case ISD::SETUEQ:
-  case ISD::SETONE: SSECC = 8; break;
+  case ISD::SETUEQ: SSECC = 8; break;
+  case ISD::SETONE: SSECC = 12; break;
   }
   if (Swap)
     std::swap(Op0, Op1);
 
   return SSECC;
 }
 
 /// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1) {
       // them to work with an SSE1 target (integer vector types are not legal).
       VT = Op0.getSimpleValueType();
     }
 
     // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
     // emit two comparisons and a logic op to tie them together.
     // TODO: This can be avoided if Intel (and only Intel as of 2016) AVX is
     // available.
+    // If we are using an AVX512 instruction we can use the extended predicate.
     SDValue Cmp;
     unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1);
-    if (SSECC == 8) {
+    if (SSECC >= 8 && Opc == X86ISD::CMPP) {
       // LLVM predicate is SETUEQ or SETONE.
       unsigned CC0, CC1;
       unsigned CombineOpc;
       if (Cond == ISD::SETUEQ) {
         CC0 = 3; // UNORD
         CC1 = 0; // EQ
         CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FOR) :
                                            static_cast<unsigned>(ISD::OR);
SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
   // are available or VBLENDV if AVX is available.
   // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
   if (Cond.getOpcode() == ISD::SETCC &&
       ((Subtarget.hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) ||
        (Subtarget.hasSSE1() && VT == MVT::f32)) &&
       VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
     SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
-    int SSECC = translateX86FSETCC(
+    unsigned SSECC = translateX86FSETCC(
         cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1);
 
-    if (SSECC != 8) {
-      if (Subtarget.hasAVX512()) {
-        SDValue Cmp = DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0,
-                                  CondOp1, DAG.getConstant(SSECC, DL, MVT::i8));
-        assert(!VT.isVector() && "Not a scalar type?");
-        return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
-      }
 
+    if (SSECC < 8) {
       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
                                 DAG.getConstant(SSECC, DL, MVT::i8));
 
       // If we have AVX, we can use a variable vector select (VBLENDV) instead
       // of 3 logic instructions for size savings and potentially speed.
       // Unfortunately, there is no scalar form of VBLENDV.
 
       // If either operand is a constant, don't try this. We can expect to

test/CodeGen/X86/avx512-mov.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
   ret <8 x i64>%res
 }
 
 define <16 x float> @test40(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: test40:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @test41(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: test41:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe8,0x57,0xd2]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0x74,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @test42(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: test42:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <16 x float> @test43(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: test43:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf0,0x57,0xc9]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xc9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <8 x double> @test44(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: test44:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x57,0xd2]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @test45(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: test45:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x57,0xd2]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 ## encoding: [0x62,0xf1,0xf5,0x48,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @test46(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: test46:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x57,0xc9]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }
 
 define <8 x double> @test47(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: test47:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0x57,0xc9]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }

test/CodeGen/X86/avx512-schedule.ll

; CHECK-NEXT:    retq # sched: [2:1.00]
   %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
   ret <8 x i64>%res
 }
 
 define <16 x float> @mov_test40(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test40:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test41(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test41:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqps %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test42(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test42:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 64
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <16 x float> @mov_test43(i8 * %addr, <16 x float> %mask1) {
 ; CHECK-LABEL: mov_test43:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordps %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqps %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <16 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <16 x float>*
   %r = load <16 x float>, <16 x float>* %vaddr, align 1
   %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
   ret <16 x float>%res
 }
 
 define <8 x double> @mov_test44(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test44:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test45(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test45:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm2, %zmm1, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm2, %zmm1, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test46(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test46:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 64
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }
 
 define <8 x double> @mov_test47(i8 * %addr, <8 x double> %mask1) {
 ; CHECK-LABEL: mov_test47:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.50]
-; CHECK-NEXT:    vcmpordpd %zmm1, %zmm0, %k1
-; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k1 {%k1}
+; CHECK-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1
 ; CHECK-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} # sched: [5:0.50]
 ; CHECK-NEXT:    retq # sched: [2:1.00]
   %mask = fcmp one <8 x double> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x double>*
   %r = load <8 x double>, <8 x double>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
   ret <8 x double>%res
 }

test/CodeGen/X86/avx512vl-mov.ll

; CHECK-NEXT:    retq ## encoding: [0xc3]
   %res = select <4 x i1> %mask, <4 x i64> %r, <4 x i64> zeroinitializer
   ret <4 x i64>%res
 }
 
 define <8 x float> @test_256_25(i8 * %addr, <8 x float> %old, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_25:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; CHECK-NEXT:    vcmpordps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm2, %ymm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 32
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> %old
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_26(i8 * %addr, <8 x float> %old, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_26:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
-; CHECK-NEXT:    vcmpordps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm2, %ymm1, %k1 {%k1} ## encoding: [0x62,0xf1,0x74,0x29,0xc2,0xca,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm2, %ymm1, %k1 ## encoding: [0x62,0xf1,0x74,0x28,0xc2,0xca,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %ymm0 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> %old
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_27(i8 * %addr, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_27:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
-; CHECK-NEXT:    vcmpordps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x28,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 32
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> zeroinitializer
   ret <8 x float>%res
 }
 
 define <8 x float> @test_256_28(i8 * %addr, <8 x float> %mask1) {
 ; CHECK-LABEL: test_256_28:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
-; CHECK-NEXT:    vcmpordps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x07]
-; CHECK-NEXT:    vcmpneqps %ymm1, %ymm0, %k1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x29,0xc2,0xc9,0x04]
+; CHECK-NEXT:    vcmpneq_oqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x0c]
 ; CHECK-NEXT:    vmovups (%rdi), %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7c,0xa9,0x10,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %mask = fcmp one <8 x float> %mask1, zeroinitializer
   %vaddr = bitcast i8* %addr to <8 x float>*
   %r = load <8 x float>, <8 x float>* %vaddr, align 1
   %res = select <8 x i1> %mask, <8 x float> %r, <8 x float> zeroinitializer
   ret <8 x float>%res
 }

test/CodeGen/X86/commute-fcmp.ll

 ; AVX-NEXT:    vcmpeqps %xmm0, %xmm1, %xmm2
 ; AVX-NEXT:    vcmpunordps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vorps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_ueq:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %xmm1
-; AVX512-NEXT:    vcmpeqps %xmm0, %xmm1, %k0
-; AVX512-NEXT:    vcmpunordps %xmm0, %xmm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqps %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x float>, <4 x float>* %a0
   %2 = fcmp ueq <4 x float> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
   ret <4 x i32> %3
 }

 ; AVX-NEXT:    vcmpneqps %xmm0, %xmm1, %xmm2
 ; AVX-NEXT:    vcmpordps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_one:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %xmm1
-; AVX512-NEXT:    vcmpordps %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vcmpneqps %xmm0, %xmm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqps %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x float>, <4 x float>* %a0
   %2 = fcmp one <4 x float> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i32>
   ret <4 x i32> %3
 }

 ; AVX-NEXT:    vcmpeqps %ymm0, %ymm1, %ymm2
 ; AVX-NEXT:    vcmpunordps %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_ueq_ymm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %ymm1
-; AVX512-NEXT:    vcmpeqps %ymm0, %ymm1, %k0
-; AVX512-NEXT:    vcmpunordps %ymm0, %ymm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqps %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <8 x float>, <8 x float>* %a0
   %2 = fcmp ueq <8 x float> %1, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
   ret <8 x i32> %3
 }

 ; AVX-NEXT:    vcmpneqps %ymm0, %ymm1, %ymm2
 ; AVX-NEXT:    vcmpordps %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmpps_one_ymm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovaps (%rdi), %ymm1
-; AVX512-NEXT:    vcmpordps %ymm0, %ymm1, %k1
-; AVX512-NEXT:    vcmpneqps %ymm0, %ymm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqps %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <8 x float>, <8 x float>* %a0
   %2 = fcmp one <8 x float> %1, %a1
   %3 = sext <8 x i1> %2 to <8 x i32>
   ret <8 x i32> %3
 }

 ; AVX-NEXT:    vcmpeqpd %xmm0, %xmm1, %xmm2
 ; AVX-NEXT:    vcmpunordpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vorpd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_ueq:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %xmm1
-; AVX512-NEXT:    vcmpeqpd %xmm0, %xmm1, %k0
-; AVX512-NEXT:    vcmpunordpd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqpd %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <2 x double>, <2 x double>* %a0
   %2 = fcmp ueq <2 x double> %1, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
   ret <2 x i64> %3
 }

 ; AVX-NEXT:    vcmpneqpd %xmm0, %xmm1, %xmm2
 ; AVX-NEXT:    vcmpordpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vandpd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_one:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %xmm1
-; AVX512-NEXT:    vcmpordpd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vcmpneqpd %xmm0, %xmm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqpd %xmm0, %xmm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <2 x double>, <2 x double>* %a0
   %2 = fcmp one <2 x double> %1, %a1
   %3 = sext <2 x i1> %2 to <2 x i64>
   ret <2 x i64> %3
 }

 ; AVX-NEXT:    vcmpeqpd %ymm0, %ymm1, %ymm2
 ; AVX-NEXT:    vcmpunordpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vorpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_ueq_ymmm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %ymm1
-; AVX512-NEXT:    vcmpeqpd %ymm0, %ymm1, %k0
-; AVX512-NEXT:    vcmpunordpd %ymm0, %ymm1, %k1
-; AVX512-NEXT:    korw %k0, %k1, %k1
+; AVX512-NEXT:    vcmpeq_uqpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x double>, <4 x double>* %a0
   %2 = fcmp ueq <4 x double> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }

 ; AVX-NEXT:    vcmpneqpd %ymm0, %ymm1, %ymm2
 ; AVX-NEXT:    vcmpordpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vandpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: commute_cmppd_one_ymmm:
 ; AVX512:       # BB#0:
 ; AVX512-NEXT:    vmovapd (%rdi), %ymm1
-; AVX512-NEXT:    vcmpordpd %ymm0, %ymm1, %k1
-; AVX512-NEXT:    vcmpneqpd %ymm0, %ymm1, %k1 {%k1}
+; AVX512-NEXT:    vcmpneq_oqpd %ymm0, %ymm1, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    retq
   %1 = load <4 x double>, <4 x double>* %a0
   %2 = fcmp one <4 x double> %1, %a1
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }