[X86] Enable CMOV i16 to be promoted to i32 as much as possible

[craig.topper]

Summary

16-bit cmov instructions are 1 byte longer than their 32-bit equivalents. We should promote them to 32-bit operations when possible like we do for binary operators. To accomplish this requires several steps some of which can and probably should be broken out.

We currently have support for promoting extends of 16-bit cmovs of constants to 32 or 64 bits. I've remove all of that as what I've implemented here should be more general purpose. Though I did lose the extending to 64-bit, but I don't know if that was completely necessary.

I've added X86ISD::CMOV to our target hooks for computeKnownBits and computeNumSignBits. This enables removal of zero extends and sign extends after the cmov is promoted. These parts should be committed separately.

Constant inputs to CMOV are sign extended so that computeNumSignBits will enable removal of later sign extends. I believe this is what bin op promotion does with constant inputs as well.

If we see (and (cmov X, Y)) we try to remove undemanded bits from X and Y. This allows us to turn the default sign extending of constants into zero extending if we don't need the bits. This is most important for ands that represent zext_inreg. This could be done more generically if we had SimplifyDemandedBits for target nodes. Once the unused bits are all zeroed out, computeKnownBits can try to remove the and. This can probably be committed independently as well.

I've blocked the promotion if there is a possibility of folding a load. This is what we do for other operations in IsDesirableToPromoteOp.

[craig.topper]

I've already committed part of this and there's a couple bugs in this. Abandoning and will submit a new patch once I have the kinks worked out.

lib/Target/X86/X86ISelLowering.cpp

case X86ISD::VZEXT: {
 
     Known = KnownBits(InBitWidth);
     APInt DemandedSrcElts = APInt::getLowBitsSet(InNumElts, NumElts);
     DAG.computeKnownBits(N0, Known, DemandedSrcElts, Depth + 1);
     Known = Known.zext(BitWidth);
     Known.Zero.setBitsFrom(InBitWidth);
     break;
   }
+  case X86ISD::CMOV: {
+    DAG.computeKnownBits(Op.getOperand(1), Known, Depth+1);
+    // If we don't know any bits, early out.
+    if (!Known.One && !Known.Zero)
+      break;
+    KnownBits Known2;
+    DAG.computeKnownBits(Op.getOperand(0), Known2, Depth+1);
+
+    // Only known if known in both the LHS and RHS.
+    Known.One &= Known2.One;
+    Known.Zero &= Known2.Zero;
+    break;
+  }
   }
 }
 
 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
     unsigned Depth) const {
   unsigned VTBits = Op.getScalarValueSizeInBits();
   unsigned Opcode = Op.getOpcode();
unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
 
   case X86ISD::PCMPGT:
   case X86ISD::PCMPEQ:
   case X86ISD::CMPP:
   case X86ISD::VPCOM:
   case X86ISD::VPCOMU:
     // Vector compares return zero/all-bits result values.
     return VTBits;
+
+  case X86ISD::CMOV: {
+    unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
+    if (Tmp0 == 1) return 1;  // Early out.
+    unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
+    return std::min(Tmp0, Tmp1);
+  }
   }
 
   // Fallback case.
   return 1;
 }
 
 SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
   if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)

 }
 
 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
 static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
                            TargetLowering::DAGCombinerInfo &DCI,
                            const X86Subtarget &Subtarget) {
   SDLoc DL(N);
 
-  // If the flag operand isn't dead, don't touch this CMOV.
-  if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
-    return SDValue();
-
   SDValue FalseOp = N->getOperand(0);
   SDValue TrueOp = N->getOperand(1);
   X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
   SDValue Cond = N->getOperand(3);
 
+  // Try to promote 16-bit CMOVs.
+  if (N->getValueType(0) == MVT::i16 && !MayFoldLoad(N->getOperand(0)) &&
+      !MayFoldLoad(N->getOperand(1))) {
+    if (isa<ConstantSDNode>(FalseOp))
+      FalseOp = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, FalseOp);
+    else
+      FalseOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, FalseOp);
+    DCI.AddToWorklist(FalseOp.getNode());
+    if (isa<ConstantSDNode>(TrueOp))
+      TrueOp = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, TrueOp);
+    else
+      TrueOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, TrueOp);
+    DCI.AddToWorklist(FalseOp.getNode());
+    SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL,
+                               DAG.getVTList(MVT::i32, MVT::Glue), FalseOp,
+                               TrueOp, N->getOperand(2), Cond);
+    DCI.AddToWorklist(Cmov.getNode());
+    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Cmov);
+    if (N->getNumValues() == 2)
+      return DCI.CombineTo(N, Trunc, Cmov.getValue(1));
+    return Trunc;
+  }
+
+  // If the flag operand isn't dead, don't touch this CMOV.
+  if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
+    return SDValue();
+
   if (CC == X86::COND_E || CC == X86::COND_NE) {
     switch (Cond.getOpcode()) {
     default: break;
     case X86ISD::BSR:
     case X86ISD::BSF:
       // If operand of BSR / BSF are proven never zero, then ZF cannot be set.
       if (DAG.isKnownNeverZero(Cond.getOperand(0)))
         return (CC == X86::COND_E) ? FalseOp : TrueOp;
static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
     SDValue Op(N, 0);
     if (combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 1,
                                       /*HasVarMask*/ false, DAG, DCI,
                                       Subtarget))
       return SDValue(); // This routine will use CombineTo to replace N.
   }
 
+  // Try to removed unneeded cmov input bits if possible. This is to work
+  // around not having SimplifyDemandedBits for target nodes.
+  if (N->getOperand(0).getOpcode() == X86ISD::CMOV &&
+      isa<ConstantSDNode>(N->getOperand(1))) {
+    SDValue N0 = N->getOperand(0);
+    APInt DemandedMask =
+      cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
+    if (SDValue DemandedLHS = DAG.GetDemandedBits(N0.getOperand(0),
+                                                  DemandedMask))
+      return DAG.getNode(X86ISD::CMOV, SDLoc(N0), N0->getVTList(), DemandedLHS,
+                         N0.getOperand(1), N0.getOperand(2), N0.getOperand(3));
+    if (SDValue DemandedRHS = DAG.GetDemandedBits(N0.getOperand(1),
+                                                  DemandedMask))
+      return DAG.getNode(X86ISD::CMOV, SDLoc(N0), N0->getVTList(),
+                         N0.getOperand(0), DemandedRHS, N0.getOperand(2),
+                         N0.getOperand(3));
+  }
+
   return SDValue();
 }
 
 // Try to fold:
 //   (or (and (m, y), (pandn m, x)))
 // into:
 //   (vselect m, x, y)
 // As a special case, try to fold:
static SDValue getDivRem8(SDNode *N, SelectionDAG &DAG) {
   auto DivRemOpcode = OpcodeN0 == ISD::SDIVREM ? X86ISD::SDIVREM8_SEXT_HREG
                                                : X86ISD::UDIVREM8_ZEXT_HREG;
   SDValue R = DAG.getNode(DivRemOpcode, SDLoc(N), NodeTys, N0.getOperand(0),
                           N0.getOperand(1));
   DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), R.getValue(0));
   return R.getValue(1);
 }
 
-// If we face {ANY,SIGN,ZERO}_EXTEND that is applied to a CMOV with constant
-// operands and the result of CMOV is not used anywhere else - promote CMOV
-// itself instead of promoting its result. This could be beneficial, because:
-//     1) X86TargetLowering::EmitLoweredSelect later can do merging of two
-//        (or more) pseudo-CMOVs only when they go one-after-another and
-//        getting rid of result extension code after CMOV will help that.
-//     2) Promotion of constant CMOV arguments is free, hence the
-//        {ANY,SIGN,ZERO}_EXTEND will just be deleted.
-//     3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
-//        promotion is also good in terms of code-size.
-//        (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
-//         promotion).
-static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
-  SDValue CMovN = Extend->getOperand(0);
-  if (CMovN.getOpcode() != X86ISD::CMOV)
-    return SDValue();
-
-  EVT TargetVT = Extend->getValueType(0);
-  unsigned ExtendOpcode = Extend->getOpcode();
-  SDLoc DL(Extend);
-
-  EVT VT = CMovN.getValueType();
-  SDValue CMovOp0 = CMovN.getOperand(0);
-  SDValue CMovOp1 = CMovN.getOperand(1);
-
-  bool DoPromoteCMOV =
-      (VT == MVT::i16 && (TargetVT == MVT::i32 || TargetVT == MVT::i64)) &&
-      CMovN.hasOneUse() &&
-      (isa<ConstantSDNode>(CMovOp0.getNode()) &&
-       isa<ConstantSDNode>(CMovOp1.getNode()));
-
-  if (!DoPromoteCMOV)
-    return SDValue();
-
-  CMovOp0 = DAG.getNode(ExtendOpcode, DL, TargetVT, CMovOp0);
-  CMovOp1 = DAG.getNode(ExtendOpcode, DL, TargetVT, CMovOp1);
-
-  return DAG.getNode(X86ISD::CMOV, DL, TargetVT, CMovOp0, CMovOp1,
-                     CMovN.getOperand(2), CMovN.getOperand(3));
-}
-
 // Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
 // This is more or less the reverse of combineBitcastvxi1.
 static SDValue
 combineToExtendBoolVectorInReg(SDNode *N, SelectionDAG &DAG,
                                TargetLowering::DAGCombinerInfo &DCI,
                                const X86Subtarget &Subtarget) {
   unsigned Opcode = N->getOpcode();
   if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND &&
static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT InVT = N0.getValueType();
   SDLoc DL(N);
 
   if (SDValue DivRem8 = getDivRem8(N, DAG))
     return DivRem8;
 
-  if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
-    return NewCMov;
-
   if (!DCI.isBeforeLegalizeOps()) {
     if (InVT == MVT::i1) {
       SDValue Zero = DAG.getConstant(0, DL, VT);
       SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
       return DAG.getSelect(DL, VT, N0, AllOnes, Zero);
     }
     return SDValue();
   }
if (N0.getOpcode() == ISD::TRUNCATE &&
     if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
       return DAG.getNode(ISD::AND, dl, VT,
                          DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
                                      N00.getOperand(0), N00.getOperand(1)),
                          DAG.getConstant(1, dl, VT));
     }
   }
 
-  if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
-    return NewCMov;
-
   if (SDValue V = combineToExtendVectorInReg(N, DAG, DCI, Subtarget))
     return V;
 
   if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
     return V;
 
   if (VT.is256BitVector())
     if (SDValue R = WidenMaskArithmetic(N, DAG, DCI, Subtarget))

test/CodeGen/X86/avx512-insert-extract.ll

 }
 
 define i16 @test15(i1 *%addr) {
 ; CHECK-LABEL: test15:
 ; CHECK:       ## BB#0:
 ; CHECK-NEXT:    movb (%rdi), %al
 ; CHECK-NEXT:    xorl %ecx, %ecx
 ; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    movw $-1, %ax
-; CHECK-NEXT:    cmovew %cx, %ax
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovel %ecx, %eax
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
 ; CHECK-NEXT:    retq
   %x = load i1 , i1 * %addr, align 1
   %x1 = insertelement <16 x i1> undef, i1 %x, i32 10
   %x2 = bitcast <16 x i1>%x1 to i16
   ret i16 %x2
 }
 
 define i16 @test16(i1 *%addr, i16 %a) {

test/CodeGen/X86/avx512-mask-op.ll

; CHECK-NEXT:    retq
   %c = insertelement <4 x i1> <i1 true, i1 false, i1 false, i1 true>, i1 %b, i32 1
   ret <4 x i1> %c
 }
 
 define <16 x i1> @test15(i32 %x, i32 %y)  {
 ; KNL-LABEL: test15:
 ; KNL:       ## BB#0:
 ; KNL-NEXT:    cmpl %esi, %edi
-; KNL-NEXT:    movw $21845, %ax ## imm = 0x5555
-; KNL-NEXT:    movw $1, %cx
-; KNL-NEXT:    cmovgw %ax, %cx
+; KNL-NEXT:    movl $21845, %eax ## imm = 0x5555
+; KNL-NEXT:    movl $1, %ecx
+; KNL-NEXT:    cmovgl %eax, %ecx
 ; KNL-NEXT:    kmovw %ecx, %k1
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test15:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    cmpl %esi, %edi
-; SKX-NEXT:    movw $21845, %ax ## imm = 0x5555
-; SKX-NEXT:    movw $1, %cx
-; SKX-NEXT:    cmovgw %ax, %cx
+; SKX-NEXT:    movl $21845, %eax ## imm = 0x5555
+; SKX-NEXT:    movl $1, %ecx
+; SKX-NEXT:    cmovgl %eax, %ecx
 ; SKX-NEXT:    kmovd %ecx, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test15:
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    cmpl %esi, %edi
-; AVX512BW-NEXT:    movw $21845, %ax ## imm = 0x5555
-; AVX512BW-NEXT:    movw $1, %cx
-; AVX512BW-NEXT:    cmovgw %ax, %cx
+; AVX512BW-NEXT:    movl $21845, %eax ## imm = 0x5555
+; AVX512BW-NEXT:    movl $1, %ecx
+; AVX512BW-NEXT:    cmovgl %eax, %ecx
 ; AVX512BW-NEXT:    kmovd %ecx, %k0
 ; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512BW-NEXT:    ## kill: %XMM0<def> %XMM0<kill> %ZMM0<kill>
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test15:
 ; AVX512DQ:       ## BB#0:
 ; AVX512DQ-NEXT:    cmpl %esi, %edi
-; AVX512DQ-NEXT:    movw $21845, %ax ## imm = 0x5555
-; AVX512DQ-NEXT:    movw $1, %cx
-; AVX512DQ-NEXT:    cmovgw %ax, %cx
+; AVX512DQ-NEXT:    movl $21845, %eax ## imm = 0x5555
+; AVX512DQ-NEXT:    movl $1, %ecx
+; AVX512DQ-NEXT:    cmovgl %eax, %ecx
 ; AVX512DQ-NEXT:    kmovw %ecx, %k0
 ; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
   %a = bitcast i16 21845 to <16 x i1>
   %b = bitcast i16 1 to <16 x i1>
   %mask = icmp sgt i32 %x, %y

test/CodeGen/X86/avx512-schedule.ll

; CHECK-NEXT:    retq # sched: [2:1.00]
   %c = insertelement <4 x i1> <i1 true, i1 false, i1 false, i1 true>, i1 %b, i32 1
   ret <4 x i1> %c
 }
 
 define <16 x i1> @vmov_test15(i32 %x, i32 %y)  {
 ; CHECK-LABEL: vmov_test15:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; CHECK-NEXT:    movw $21845, %ax # imm = 0x5555
+; CHECK-NEXT:    movl $21845, %eax # imm = 0x5555
 ; CHECK-NEXT:    # sched: [1:0.25]
-; CHECK-NEXT:    movw $1, %cx # sched: [1:0.25]
-; CHECK-NEXT:    cmovgw %ax, %cx # sched: [1:1.00]
+; CHECK-NEXT:    movl $1, %ecx # sched: [1:0.25]
+; CHECK-NEXT:    cmovgl %eax, %ecx # sched: [1:1.00]
 ; CHECK-NEXT:    kmovd %ecx, %k0
 ; CHECK-NEXT:    vpmovm2b %k0, %xmm0
 ; CHECK-NEXT:    retq # sched: [2:1.00]
 ; SKX-LABEL: vmov_test15:
 ; SKX:       ## BB#0:
 ; SKX-NEXT:    cmpl %esi, %edi
 ; SKX-NEXT:    movw $21845, %ax ## imm = 0x5555
 ; SKX-NEXT:    movw $1, %cx

test/CodeGen/X86/bool-simplify.ll

; CHECK-NEXT:    retq
   ret i32 %t3
 }
 
 define i16 @rnd16(i16 %arg) nounwind {
 ; CHECK-LABEL: rnd16:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    rdrandw %cx
-; CHECK-NEXT:    cmovbw %di, %ax
+; CHECK-NEXT:    cmovbl %edi, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:    # kill: %AX<def> %AX<kill> %EAX<kill>
 ; CHECK-NEXT:    retq
   %1 = tail call { i16, i32 } @llvm.x86.rdrand.16() nounwind
   %2 = extractvalue { i16, i32 } %1, 0
   %3 = extractvalue { i16, i32 } %1, 1
   %4 = icmp eq i32 %3, 0
   %5 = select i1 %4, i16 0, i16 %arg
; CHECK-NEXT:    retq
   ret i64 %6
 }
 
 define i16 @seed16(i16 %arg) nounwind {
 ; CHECK-LABEL: seed16:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    rdseedw %cx
-; CHECK-NEXT:    cmovbw %di, %ax
+; CHECK-NEXT:    cmovbl %edi, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
 ; CHECK-NEXT:    # kill: %AX<def> %AX<kill> %EAX<kill>
 ; CHECK-NEXT:    retq
   %1 = tail call { i16, i32 } @llvm.x86.rdseed.16() nounwind
   %2 = extractvalue { i16, i32 } %1, 0
   %3 = extractvalue { i16, i32 } %1, 1
   %4 = icmp eq i32 %3, 0
   %5 = select i1 %4, i16 0, i16 %arg

test/CodeGen/X86/cmov-promotion.ll

 }
 
 define i64 @cmov_zpromotion_16_to_64(i1 %c) {
 ; CMOV-LABEL: cmov_zpromotion_16_to_64:
 ; CMOV:       # BB#0:
 ; CMOV-NEXT:    testb $1, %dil
 ; CMOV-NEXT:    movl $12414, %ecx # imm = 0x307E
 ; CMOV-NEXT:    movl $65535, %eax # imm = 0xFFFF
-; CMOV-NEXT:    cmovneq %rcx, %rax
+; CMOV-NEXT:    cmovnel %ecx, %eax
 ; CMOV-NEXT:    retq
 ;
 ; NO_CMOV-LABEL: cmov_zpromotion_16_to_64:
 ; NO_CMOV:       # BB#0:
 ; NO_CMOV-NEXT:    testb $1, {{[0-9]+}}(%esp)
 ; NO_CMOV-NEXT:    movl $12414, %eax # imm = 0x307E
 ; NO_CMOV-NEXT:    jne .LBB4_2
 ; NO_CMOV-NEXT:  # BB#1:
; NO_CMOV-NEXT:    retl
   %ret = sext i16 %t0 to i32
   ret i32 %ret
 }
 
 define i64 @cmov_spromotion_16_to_64(i1 %c) {
 ; CMOV-LABEL: cmov_spromotion_16_to_64:
 ; CMOV:       # BB#0:
 ; CMOV-NEXT:    testb $1, %dil
-; CMOV-NEXT:    movl $12414, %ecx # imm = 0x307E
-; CMOV-NEXT:    movq $-1, %rax
-; CMOV-NEXT:    cmovneq %rcx, %rax
+; CMOV-NEXT:    movl $12414, %eax # imm = 0x307E
+; CMOV-NEXT:    movl $-1, %ecx
+; CMOV-NEXT:    cmovnel %eax, %ecx
+; CMOV-NEXT:    movslq %ecx, %rax
 ; CMOV-NEXT:    retq
 ;
 ; NO_CMOV-LABEL: cmov_spromotion_16_to_64:
 ; NO_CMOV:       # BB#0:
 ; NO_CMOV-NEXT:    testb $1, {{[0-9]+}}(%esp)
 ; NO_CMOV-NEXT:    movl $12414, %eax # imm = 0x307E
 ; NO_CMOV-NEXT:    jne .LBB10_2
 ; NO_CMOV-NEXT:  # BB#1:

test/CodeGen/X86/cmovcmov.ll

 ; NOCMOV-NEXT:   fnstsw  %ax
 ; NOCMOV-NEXT:   sahf
 ; NOCMOV-NEXT:   leal  20(%esp), %ecx
 ; NOCMOV-NEXT:   jne  [[TBB:.LBB[0-9_]+]]
 ; NOCMOV-NEXT:   jp  [[TBB]]
 ; NOCMOV-NEXT:   leal  12(%esp), %ecx
 ; NOCMOV-NEXT: [[TBB]]:
 ; NOCMOV-NEXT:   movl  (%ecx), %eax
-; NOCMOV-NEXT:   orl  $4, %ecx
-; NOCMOV-NEXT:   movl  (%ecx), %edx
+; NOCMOV-NEXT:   movl  4(%ecx), %edx
 ; NOCMOV-NEXT:   retl
 define i64 @test_select_fcmp_oeq_i64(float %a, float %b, i64 %c, i64 %d) #0 {
 entry:
   %cmp = fcmp oeq float %a, %b
   %r = select i1 %cmp, i64 %c, i64 %d
   ret i64 %r
 }
 

 ; NOCMOV-NEXT:   fnstsw  %ax
 ; NOCMOV-NEXT:   sahf
 ; NOCMOV-NEXT:   leal  12(%esp), %ecx
 ; NOCMOV-NEXT:   jne  [[TBB:.LBB[0-9_]+]]
 ; NOCMOV-NEXT:   jp  [[TBB]]
 ; NOCMOV-NEXT:   leal  20(%esp), %ecx
 ; NOCMOV-NEXT: [[TBB]]:
 ; NOCMOV-NEXT:   movl  (%ecx), %eax
-; NOCMOV-NEXT:   orl  $4, %ecx
-; NOCMOV-NEXT:   movl  (%ecx), %edx
+; NOCMOV-NEXT:   movl  4(%ecx), %edx
 ; NOCMOV-NEXT:   retl
 define i64 @test_select_fcmp_une_i64(float %a, float %b, i64 %c, i64 %d) #0 {
 entry:
   %cmp = fcmp une float %a, %b
   %r = select i1 %cmp, i64 %c, i64 %d
   ret i64 %r
 }
 

test/CodeGen/X86/pr32282.ll

 ; X86-NEXT:    andnl %edx, %eax, %edx
 ; X86-NEXT:    shrdl $21, %ecx, %edx
 ; X86-NEXT:    shrl $21, %ecx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    testb %al, %al
 ; X86-NEXT:    cmovnel %ecx, %edx
 ; X86-NEXT:    cmovnel %eax, %ecx
 ; X86-NEXT:    andl $-2, %edx
-; X86-NEXT:    andl $2147483647, %ecx # imm = 0x7FFFFFFF
 ; X86-NEXT:    addl $7, %edx
 ; X86-NEXT:    adcxl %eax, %ecx
 ; X86-NEXT:    pushl %ecx
 ; X86-NEXT:  .Lcfi1:
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4
 ; X86-NEXT:    pushl %edx
 ; X86-NEXT:  .Lcfi2:
 ; X86-NEXT:    .cfi_adjust_cfa_offset 4

test/CodeGen/X86/select.ll

 ; reproducer for pr29002
 define void @clamp(i32 %src, i16* %dst) {
 ; GENERIC-LABEL: clamp:
 ; GENERIC:       ## BB#0:
 ; GENERIC-NEXT:    cmpl $32767, %edi ## imm = 0x7FFF
 ; GENERIC-NEXT:    movl $32767, %eax ## imm = 0x7FFF
 ; GENERIC-NEXT:    cmovlel %edi, %eax
 ; GENERIC-NEXT:    cmpl $-32768, %eax ## imm = 0x8000
-; GENERIC-NEXT:    movw $-32768, %cx ## imm = 0x8000
-; GENERIC-NEXT:    cmovgew %ax, %cx
+; GENERIC-NEXT:    movl $-32768, %ecx ## imm = 0x8000
+; GENERIC-NEXT:    cmovgel %eax, %ecx
 ; GENERIC-NEXT:    movw %cx, (%rsi)
 ; GENERIC-NEXT:    retq
 ;
 ; ATOM-LABEL: clamp:
 ; ATOM:       ## BB#0:
 ; ATOM-NEXT:    cmpl $32767, %edi ## imm = 0x7FFF
 ; ATOM-NEXT:    movl $32767, %eax ## imm = 0x7FFF
 ; ATOM-NEXT:    cmovlel %edi, %eax
-; ATOM-NEXT:    movw $-32768, %cx ## imm = 0x8000
+; ATOM-NEXT:    movl $-32768, %ecx ## imm = 0x8000
 ; ATOM-NEXT:    cmpl $-32768, %eax ## imm = 0x8000
-; ATOM-NEXT:    cmovgew %ax, %cx
+; ATOM-NEXT:    cmovgel %eax, %ecx
 ; ATOM-NEXT:    movw %cx, (%rsi)
 ; ATOM-NEXT:    retq
 ;
 ; MCU-LABEL: clamp:
 ; MCU:       # BB#0:
 ; MCU-NEXT:    cmpl $32767, %eax # imm = 0x7FFF
 ; MCU-NEXT:    movl $32767, %ecx # imm = 0x7FFF
 ; MCU-NEXT:    jg .LBB23_2
 ; MCU-NEXT:  # BB#1:
 ; MCU-NEXT:    movl %eax, %ecx
 ; MCU-NEXT:  .LBB23_2:
 ; MCU-NEXT:    cmpl $-32768, %ecx # imm = 0x8000
-; MCU-NEXT:    movw $-32768, %ax # imm = 0x8000
+; MCU-NEXT:    movl $-32768, %eax # imm = 0x8000
 ; MCU-NEXT:    jl .LBB23_4
 ; MCU-NEXT:  # BB#3:
 ; MCU-NEXT:    movl %ecx, %eax
 ; MCU-NEXT:  .LBB23_4:
 ; MCU-NEXT:    movw %ax, (%edx)
 ; MCU-NEXT:    retl
   %cmp = icmp sgt i32 %src, 32767
   %sel1 = select i1 %cmp, i32 32767, i32 %src

 }
 
 define i16 @select_xor_1(i16 %A, i8 %cond) {
 ; CHECK-LABEL: select_xor_1:
 ; CHECK:       ## BB#0: ## %entry
 ; CHECK-NEXT:    movl %edi, %eax
 ; CHECK-NEXT:    xorl $43, %eax
 ; CHECK-NEXT:    testb $1, %sil
-; CHECK-NEXT:    cmovnew %ax, %di
-; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cmovel %edi, %eax
+; CHECK-NEXT:    ## kill: %AX<def> %AX<kill> %EAX<kill>
 ; CHECK-NEXT:    retq
 ;
 ; MCU-LABEL: select_xor_1:
 ; MCU:       # BB#0: # %entry
 ; MCU-NEXT:    andl $1, %edx
 ; MCU-NEXT:    negl %edx
 ; MCU-NEXT:    andl $43, %edx
 ; MCU-NEXT:    xorl %edx, %eax

test/CodeGen/X86/select_const.ll

 
 ; 4294967297 = 0x100000001.
 ; This becomes an opaque constant via ConstantHoisting, so we don't fold it into the select.
 
 define i64 @opaque_constant(i1 %cond, i64 %x) {
 ; CHECK-LABEL: opaque_constant:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    testb $1, %dil
-; CHECK-NEXT:    movl $23, %ecx
+; CHECK-NEXT:    movl $1, %ecx
 ; CHECK-NEXT:    movq $-4, %rax
 ; CHECK-NEXT:    cmoveq %rcx, %rax
 ; CHECK-NEXT:    movabsq $4294967297, %rcx # imm = 0x100000001
-; CHECK-NEXT:    andq %rcx, %rax
 ; CHECK-NEXT:    xorl %edx, %edx
 ; CHECK-NEXT:    cmpq %rcx, %rsi
 ; CHECK-NEXT:    sete %dl
 ; CHECK-NEXT:    subq %rdx, %rax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i64 -4, i64 23
   %bo = and i64 %sel, 4294967297
   %cmp = icmp eq i64 %x, 4294967297
   %sext = sext i1 %cmp to i64
   %add = add i64 %bo, %sext
   ret i64 %add
 }
 

test/CodeGen/X86/setcc-lowering.ll

 ; KNL-32-NEXT:    .cfi_def_cfa_offset 8
 ; KNL-32-NEXT:  .Lcfi1:
 ; KNL-32-NEXT:    .cfi_offset %esi, -8
 ; KNL-32-NEXT:    vpmovsxbd %xmm0, %zmm0
 ; KNL-32-NEXT:    vpslld $31, %zmm0, %zmm0
 ; KNL-32-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; KNL-32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; KNL-32-NEXT:    movw $-1, %dx
+; KNL-32-NEXT:    movl $-1, %edx
 ; KNL-32-NEXT:    .p2align 4, 0x90
 ; KNL-32-NEXT:  .LBB1_1: # %for_loop599
 ; KNL-32-NEXT:    # =>This Inner Loop Header: Depth=1
 ; KNL-32-NEXT:    cmpl $65536, %ecx # imm = 0x10000
 ; KNL-32-NEXT:    movl %eax, %esi
 ; KNL-32-NEXT:    sbbl $0, %esi
 ; KNL-32-NEXT:    movl $0, %esi
-; KNL-32-NEXT:    cmovlw %dx, %si
+; KNL-32-NEXT:    cmovll %edx, %esi
 ; KNL-32-NEXT:    kmovw %esi, %k1
 ; KNL-32-NEXT:    kandw %k0, %k1, %k1
 ; KNL-32-NEXT:    kmovw %k1, %esi
 ; KNL-32-NEXT:    testw %si, %si
 ; KNL-32-NEXT:    jne .LBB1_1
 ; KNL-32-NEXT:  # BB#2: # %for_exit600
 ; KNL-32-NEXT:    popl %esi
 ; KNL-32-NEXT:    retl

test/CodeGen/X86/vector-compare-results.ll

 ; AVX512BW-LABEL: test_cmp_v64i16:
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vextracti32x4 $3, %zmm2, %xmm4
 ; AVX512BW-NEXT:    vpextrw $1, %xmm4, %ecx
 ; AVX512BW-NEXT:    vextracti32x4 $3, %zmm0, %xmm5
 ; AVX512BW-NEXT:    vpextrw $1, %xmm5, %edx
 ; AVX512BW-NEXT:    xorl %eax, %eax
 ; AVX512BW-NEXT:    cmpw %cx, %dx
-; AVX512BW-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; AVX512BW-NEXT:    movl $-1, %ecx
 ; AVX512BW-NEXT:    movl $0, %edx
 ; AVX512BW-NEXT:    cmovgl %ecx, %edx
 ; AVX512BW-NEXT:    vmovd %xmm4, %esi
 ; AVX512BW-NEXT:    vmovd %xmm5, %edi
 ; AVX512BW-NEXT:    cmpw %si, %di
 ; AVX512BW-NEXT:    movl $0, %esi
 ; AVX512BW-NEXT:    cmovgl %ecx, %esi
 ; AVX512BW-NEXT:    vmovd %esi, %xmm6