[RFC][DAG] Match select of constant equivalents in foldBinOpIntoSelect.

[deadalnix]

Summary

This makes the combiner more robust in case where the select is optimized away because the binary op is processed. This effect is visible in D127115 .

The diff as this has a couple of regression, but I want to get feedback on the apporach to know if tthis is worth going throught he trouble of nailing them down.

[RKSimon]

llvm/test/CodeGen/VE/Scalar/atomic.ll

23

regenerate and NFC commit to reduce the diff

[foad]

The AMDGPU diff looks nice!

[RKSimon]

I've not strong objection, but I'd like to see how far you can get fixing the x86 combineSelectOfTwoConstants combines

[deadalnix]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

@RKSimon Is this an improvement or a regression? Or it doesn't matter?

[RKSimon]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

Avoiding cmov for similar codesize is usually a win - they can't be predicted and have tight eflags dependencies

[gchatelet]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

The codegen of llvm-libc memcmp and bcmp should directly benefit from it :-) Thx!

[gchatelet]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

Ha sorry I read it in the opposite direction. So this would be a regression...

[spatel]

I don't have a problem with the direction, but as noted earlier, all of the x86 tests that changed to cmov would likely be viewed as regressions. Maybe we can predicate with the phase of combining to limit those diffs?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

2256

Does this need to use TLI.getBooleanContents() for non i1 types?

[deadalnix]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

I'm working on a fix for this :) unfortunately, it creates some regressions in the powerpc target, so it's a bit like playing wack a mole.

[deadalnix]

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

2256

Is here a predicate to check if something is a "not"?

[deadalnix]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

So D130765 fixes this, but in turn seems to introduce some regressions for PowerPC.

[deadalnix]

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

165

The regression for PowerPC are gone, I'm going to try to rebase this.

[RKSimon]

@deadalnix reverse-ping - rebase this?

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
                                       SDValue N1);
     SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
                            SDValue N1, SDNodeFlags Flags);
 
     SDValue visitShiftByConstant(SDNode *N);
 
     SDValue foldSelectOfConstants(SDNode *N);
     SDValue foldVSelectOfConstants(SDNode *N);
+    bool matchSelectOfConstants(SDValue Sel, SDValue &Cond, SDValue &CT,
+                                SDValue &CF);
     SDValue foldBinOpIntoSelect(SDNode *BO);
     bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
     SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
     SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
     SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
                              SDValue N2, SDValue N3, ISD::CondCode CC,
                              bool NotExtCompare = false);
     SDValue convertSelectOfFPConstantsToLoadOffset(
if (isIdentityConstantForOpcode(Opcode, FVal)) {
     SDValue F0 = DAG.getFreeze(N0);
     SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
     return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
   }
 
   return SDValue();
 }
 
+static bool matchCondition(SDValue V) {
+  while (true) {
+    if (V.getValueType() == MVT::i1)
+      return true;
+
+    if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1)))
+      return true;
+
+    if (V.getOpcode() == ISD::XOR && isOneConstant(V.getOperand(1))) {
+      V = V.getOperand(0);
+      continue;
+    }
+
+    // Peek through truncate and zext.
+    if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
+      V = V.getOperand(0);
+      continue;
+    }
+
+    break;
+  }
+
+  return false;
+}
+
+bool DAGCombiner::matchSelectOfConstants(SDValue V, SDValue &Cond, SDValue &CT,
+                                         SDValue &CF) {
+  if (!V.hasOneUse())
+    return false;
+
+  if (V.getOpcode() == ISD::SELECT) {
+    Cond = V.getOperand(0);
+    CT = V.getOperand(1);
+    CF = V.getOperand(2);
+
+    return (isConstantOrConstantVector(CT, true) ||
+            DAG.isConstantFPBuildVectorOrConstantFP(CT)) &&
+           (isConstantOrConstantVector(CF, true) ||
+            DAG.isConstantFPBuildVectorOrConstantFP(CF));
+  }
+
+  // op Cond, C -> select Cond, (op 1, C), (op 0, C)
+  unsigned opcode = V->getOpcode();
+  if (!LegalTypes && TLI.isBinOp(opcode) && V->getNumValues() == 1) {
+    Cond = V.getOperand(0);
+    auto C = V.getOperand(1);
+    if ((isConstantOrConstantVector(C, true) ||
+         DAG.isConstantFPBuildVectorOrConstantFP(C)) &&
+        matchCondition(Cond)) {
+      auto DL = SDLoc(V);
+      auto VT = V.getValueType();
+
+      SDValue One = DAG.getConstant(1, DL, VT);
+      CT = DAG.getNode(opcode, DL, VT, One, C);
+      if (!isConstantOrConstantVector(CT, true) &&
+          !DAG.isConstantFPBuildVectorOrConstantFP(CT))
+        return false;
+
+      SDValue Zero = DAG.getConstant(0, DL, VT);
+      CF = DAG.getNode(opcode, DL, VT, Zero, C);
+      if (!isConstantOrConstantVector(CF, true) &&
+          !DAG.isConstantFPBuildVectorOrConstantFP(CF))
+        return false;
+
+      return true;
+    }
+  }
+
+  return false;
+}
+
 SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
   assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&
          "Unexpected binary operator");
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   auto BinOpcode = BO->getOpcode();
   EVT VT = BO->getValueType(0);
   if (TLI.shouldFoldSelectWithIdentityConstant(BinOpcode, VT)) {
     if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false))
       return Sel;
 
     if (TLI.isCommutativeBinOp(BO->getOpcode()))
       if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true))
         return Sel;
   }
 
+  SDValue Cond, CT, CF;
+
   // Don't do this unless the old select is going away. We want to eliminate the
   // binary operator, not replace a binop with a select.
   // TODO: Handle ISD::SELECT_CC.
   unsigned SelOpNo = 0;
   SDValue Sel = BO->getOperand(0);
-  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
+  if (!matchSelectOfConstants(Sel, Cond, CT, CF)) {
     SelOpNo = 1;
     Sel = BO->getOperand(1);
-  }
-
-  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
-    return SDValue();
-
-  SDValue CT = Sel.getOperand(1);
-  if (!isConstantOrConstantVector(CT, true) &&
-      !DAG.isConstantFPBuildVectorOrConstantFP(CT))
-    return SDValue();
-
-  SDValue CF = Sel.getOperand(2);
-  if (!isConstantOrConstantVector(CF, true) &&
-      !DAG.isConstantFPBuildVectorOrConstantFP(CF))
+    if (!matchSelectOfConstants(Sel, Cond, CT, CF))
       return SDValue();
+  }
 
   // Bail out if any constants are opaque because we can't constant fold those.
   // The exception is "and" and "or" with either 0 or -1 in which case we can
   // propagate non constant operands into select. I.e.:
   // and (select Cond, 0, -1), X --> select Cond, 0, X
   // or X, (select Cond, -1, 0) --> select Cond, -1, X
   bool CanFoldNonConst =
+      (Sel.getOpcode() == ISD::SELECT) &&
       (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
       (isNullOrNullSplat(CT) || isAllOnesOrAllOnesSplat(CT)) &&
       (isNullOrNullSplat(CF) || isAllOnesOrAllOnesSplat(CF));
 
   SDValue CBO = BO->getOperand(SelOpNo ^ 1);
   if (!CanFoldNonConst &&
       !isConstantOrConstantVector(CBO, true) &&
       !DAG.isConstantFPBuildVectorOrConstantFP(CBO))
SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
 
   SDValue NewCF = SelOpNo ? DAG.getNode(BinOpcode, DL, VT, CBO, CF)
                           : DAG.getNode(BinOpcode, DL, VT, CF, CBO);
   if (!CanFoldNonConst && !NewCF.isUndef() &&
       !isConstantOrConstantVector(NewCF, true) &&
       !DAG.isConstantFPBuildVectorOrConstantFP(NewCF))
     return SDValue();
 
-  SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF);
+  SDValue SelectOp = DAG.getSelect(DL, VT, Cond, NewCT, NewCF);
   SelectOp->setFlags(BO->getFlags());
+
+  // It is likely that we get a slect of constant here, so we premptively try to
+  // fold.
+  if (SelectOp.getOpcode() == ISD::SELECT) {
+    if (SDValue Combined = foldSelectOfConstants(SelectOp.getNode()))
+      return Combined.getNode() == BO ? SDValue() : Combined;
+  }
+
   return SelectOp;
 }
 
 static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) {
   assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
          "Expecting add or sub");
 
   // Match a constant operand and a zext operand for the math instruction:
if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) {
     SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
     SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
     return DAG.getNode(ISD::AND, DL, VT, Sra, C1);
   }
   return SDValue();
 }
 
 SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
+  assert(N->getOpcode() == ISD::SELECT && "Expecting select");
+
   SDValue Cond = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
   SDValue N2 = N->getOperand(2);
   EVT VT = N->getValueType(0);
   EVT CondVT = Cond.getValueType();
   SDLoc DL(N);
 
   if (!VT.isInteger())

llvm/test/CodeGen/AMDGPU/anyext.ll

 ;
 ; GFX8-LABEL: anyext_i1_i32:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX8-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX8-NEXT:    s_mov_b32 s2, -1
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_cmp_eq_u32 s4, 0
+; GFX8-NEXT:    s_cmp_lg_u32 s4, 0
 ; GFX8-NEXT:    s_cselect_b64 s[4:5], -1, 0
 ; GFX8-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX8-NEXT:    v_not_b32_e32 v0, v0
-; GFX8-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX9-LABEL: anyext_i1_i32:
 ; GFX9:       ; %bb.0: ; %entry
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
 ; GFX9-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX9-NEXT:    s_mov_b32 s6, -1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX9-NEXT:    s_cmp_lg_u32 s2, 0
 ; GFX9-NEXT:    s_cselect_b64 s[0:1], -1, 0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX9-NEXT:    v_not_b32_e32 v0, v0
-; GFX9-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GFX9-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; GFX9-NEXT:    s_endpgm
 entry:
   %tmp = icmp eq i32 %cond, 0
   %tmp1 = zext i1 %tmp to i8
   %tmp2 = xor i8 %tmp1, -1
   %tmp3 = and i8 %tmp2, 1
   %tmp4 = zext i8 %tmp3 to i32

llvm/test/CodeGen/ARM/pr36577.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple armv6t2 %s -o - | FileCheck %s
 ; RUN: llc -mtriple thumbv6t2 %s -o - | FileCheck %s --check-prefix=CHECK-T2
 ; RUN: llc -mtriple armv7 %s -o - | FileCheck %s
 ; RUN: llc -mtriple thumbv7 %s -o - | FileCheck %s --check-prefix=CHECK-T2
 ; RUN: llc -mtriple thumbv7m %s -o - | FileCheck %s --check-prefix=CHECK-T2
 ; RUN: llc -mtriple thumbv8m.main %s -o - | FileCheck %s --check-prefix=CHECK-T2
 
 @a = common dso_local local_unnamed_addr global i16 0, align 2
 
 define dso_local arm_aapcscc i32** @pr36577() {
 ; CHECK-LABEL: pr36577:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    movw r0, :lower16:a
-; CHECK-NEXT:    mvn r1, #7
+; CHECK-NEXT:    mov r2, #1
 ; CHECK-NEXT:    movt r0, :upper16:a
-; CHECK-NEXT:    ldrh r0, [r0]
-; CHECK-NEXT:    mvn r0, r0, lsr #7
-; CHECK-NEXT:    orr r0, r1, r0, lsl #2
+; CHECK-NEXT:    ldrh r1, [r0]
+; CHECK-NEXT:    mvn r0, #3
+; CHECK-NEXT:    tst r2, r1, lsr #7
+; CHECK-NEXT:    mvnne r0, #7
 ; CHECK-NEXT:    bx lr
 ;
 ; CHECK-T2-LABEL: pr36577:
 ; CHECK-T2:       @ %bb.0: @ %entry
 ; CHECK-T2-NEXT:    movw r0, :lower16:a
-; CHECK-T2-NEXT:    mvn r1, #7
 ; CHECK-T2-NEXT:    movt r0, :upper16:a
 ; CHECK-T2-NEXT:    ldrh r0, [r0]
-; CHECK-T2-NEXT:    mvn.w r0, r0, lsr #7
-; CHECK-T2-NEXT:    orr.w r0, r1, r0, lsl #2
+; CHECK-T2-NEXT:    lsrs r0, r0, #7
+; CHECK-T2-NEXT:    lsls r0, r0, #31
+; CHECK-T2-NEXT:    mvn r0, #3
+; CHECK-T2-NEXT:    it ne
+; CHECK-T2-NEXT:    mvnne r0, #7
 ; CHECK-T2-NEXT:    bx lr
 entry:
   %0 = load i16, i16* @a, align 2
   %1 = lshr i16 %0, 7
   %2 = and i16 %1, 1
   %3 = zext i16 %2 to i32
   %4 = xor i32 %3, -1
   %add.ptr = getelementptr inbounds i32*, i32** null, i32 %4
   ret i32** %add.ptr
 }
 

llvm/test/CodeGen/VE/Scalar/atomic.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=ve-unknown-unknown | FileCheck %s
 
 ;;; Test atomicrmw operations
 
 @c = common global i8 0, align 4
 @s = common global i16 0, align 4
 @i = common global i32 0, align 4
 @l = common global i64 0, align 4
 
 ; Function Attrs: norecurse nounwind
 define signext i8 @test_atomic_fetch_add_1() {
 ; CHECK-LABEL: test_atomic_fetch_add_1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, c@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, c@hi(, %s0)
 ; CHECK-NEXT:    and %s0, -4, %s0
 ; CHECK-NEXT:    ldl.sx %s2, (, %s0)
 ; CHECK-NEXT:    lea %s1, -256
 ; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s2
 ; CHECK-NEXT:    adds.w.sx %s2, 1, %s2
 ; CHECK-NEXT:    and %s2, %s2, (56)0
 ; CHECK-NEXT:    and %s4, %s3, %s1
 ; CHECK-NEXT:    or %s2, %s4, %s2
 ; CHECK-NEXT:    cas.w %s2, (%s0), %s3
-; CHECK-NEXT:    brne.w %s2, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s2, %s3, .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    sll %s0, %s2, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw add i8* @c, i8 1 seq_cst
   ret i8 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i16 @test_atomic_fetch_sub_2() {
 ; CHECK-LABEL: test_atomic_fetch_sub_2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, s@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, s@hi(, %s0)
 ; CHECK-NEXT:    and %s0, -4, %s0
 ; CHECK-NEXT:    ldl.sx %s2, (, %s0)
 ; CHECK-NEXT:    lea %s1, -65536
 ; CHECK-NEXT:    and %s1, %s1, (32)0
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s2
 ; CHECK-NEXT:    adds.w.sx %s2, -1, %s2
 ; CHECK-NEXT:    and %s2, %s2, (48)0
 ; CHECK-NEXT:    and %s4, %s3, %s1
 ; CHECK-NEXT:    or %s2, %s4, %s2
 ; CHECK-NEXT:    cas.w %s2, (%s0), %s3
-; CHECK-NEXT:    brne.w %s2, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s2, %s3, .LBB1_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    sll %s0, %s2, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw sub i16* @s, i16 1 seq_cst
   ret i16 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @test_atomic_fetch_and_4() {
 ; CHECK-LABEL: test_atomic_fetch_and_4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, i@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, i@hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s1, (, %s0)
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s2, 0, %s1
 ; CHECK-NEXT:    and %s1, 1, %s2
 ; CHECK-NEXT:    cas.w %s1, (%s0), %s2
-; CHECK-NEXT:    brne.w %s1, %s2, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s1, %s2, .LBB2_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw and i32* @i, i32 1 seq_cst
   ret i32 %0
 }
 ; Function Attrs: norecurse nounwind
 define i64 @test_atomic_fetch_or_8() {
 ; CHECK-LABEL: test_atomic_fetch_or_8:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, l@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s1, l@hi(, %s0)
 ; CHECK-NEXT:    ld %s0, (, %s1)
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s2, 0, %s0
 ; CHECK-NEXT:    or %s0, 1, %s0
 ; CHECK-NEXT:    cas.l %s0, (%s1), %s2
-; CHECK-NEXT:    brne.l %s0, %s2, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.l %s0, %s2, .LBB3_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw or i64* @l, i64 1 seq_cst
   ret i64 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i8 @test_atomic_fetch_xor_1() {
 ; CHECK-LABEL: test_atomic_fetch_xor_1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, c@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, c@hi(, %s0)
 ; CHECK-NEXT:    and %s1, -4, %s0
 ; CHECK-NEXT:    ldl.sx %s0, (, %s1)
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s2, 0, %s0
 ; CHECK-NEXT:    xor %s0, 1, %s2
 ; CHECK-NEXT:    cas.w %s0, (%s1), %s2
-; CHECK-NEXT:    brne.w %s0, %s2, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s0, %s2, .LBB4_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    sll %s0, %s0, 56
 ; CHECK-NEXT:    sra.l %s0, %s0, 56
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw xor i8* @c, i8 1 seq_cst
   ret i8 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i16 @test_atomic_fetch_nand_2() {
 ; CHECK-LABEL: test_atomic_fetch_nand_2:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, s@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, s@hi(, %s0)
 ; CHECK-NEXT:    and %s0, -4, %s0
 ; CHECK-NEXT:    ldl.sx %s2, (, %s0)
-; CHECK-NEXT:    lea %s1, 65534
-; CHECK-NEXT:    lea %s3, -65536
-; CHECK-NEXT:    and %s3, %s3, (32)0
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:    lea %s1, -65536
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or %s4, 0, %s2
-; CHECK-NEXT:    xor %s2, -1, %s4
-; CHECK-NEXT:    or %s2, %s2, %s1
-; CHECK-NEXT:    and %s2, %s2, (48)0
-; CHECK-NEXT:    and %s5, %s4, %s3
-; CHECK-NEXT:    or %s2, %s5, %s2
-; CHECK-NEXT:    cas.w %s2, (%s0), %s4
-; CHECK-NEXT:    brne.w %s2, %s4, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    or %s3, 0, %s2
+; CHECK-NEXT:    and %s2, 1, %s3
+; CHECK-NEXT:    and %s4, %s3, %s1
+; CHECK-NEXT:    lea %s5, 65534
+; CHECK-NEXT:    cmov.w.eq %s5, (48)0, %s2
+; CHECK-NEXT:    or %s2, %s4, %s5
+; CHECK-NEXT:    cas.w %s2, (%s0), %s3
+; CHECK-NEXT:    brne.w %s2, %s3, .LBB5_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    sll %s0, %s2, 48
 ; CHECK-NEXT:    sra.l %s0, %s0, 48
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw nand i16* @s, i16 1 seq_cst
   ret i16 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @test_atomic_fetch_max_4() {
 ; CHECK-LABEL: test_atomic_fetch_max_4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, i@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s1, i@hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s0, (, %s1)
 ; CHECK-NEXT:    or %s2, 1, (0)1
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s0
 ; CHECK-NEXT:    maxs.w.sx %s0, %s0, %s2
 ; CHECK-NEXT:    cas.w %s0, (%s1), %s3
-; CHECK-NEXT:    brne.w %s0, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s0, %s3, .LBB6_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw max i32* @i, i32 1 seq_cst
   ret i32 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @test_atomic_fetch_min_4() {
 ; CHECK-LABEL: test_atomic_fetch_min_4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, i@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, i@hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s1, (, %s0)
 ; CHECK-NEXT:    or %s2, 2, (0)1
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s1
 ; CHECK-NEXT:    cmps.w.sx %s4, %s1, %s2
 ; CHECK-NEXT:    or %s1, 1, (0)1
 ; CHECK-NEXT:    cmov.w.lt %s1, %s3, %s4
 ; CHECK-NEXT:    cas.w %s1, (%s0), %s3
-; CHECK-NEXT:    brne.w %s1, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s1, %s3, .LBB7_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw min i32* @i, i32 1 seq_cst
   ret i32 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @test_atomic_fetch_umax_4() {
 ; CHECK-LABEL: test_atomic_fetch_umax_4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, i@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, i@hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s1, (, %s0)
 ; CHECK-NEXT:    or %s2, 1, (0)1
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB8_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s1
 ; CHECK-NEXT:    cmpu.w %s4, %s1, %s2
 ; CHECK-NEXT:    or %s1, 1, (0)1
 ; CHECK-NEXT:    cmov.w.gt %s1, %s3, %s4
 ; CHECK-NEXT:    cas.w %s1, (%s0), %s3
-; CHECK-NEXT:    brne.w %s1, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s1, %s3, .LBB8_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw umax i32* @i, i32 1 seq_cst
   ret i32 %0
 }
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @test_atomic_fetch_umin_4() {
 ; CHECK-LABEL: test_atomic_fetch_umin_4:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    lea %s0, i@lo
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, i@hi(, %s0)
 ; CHECK-NEXT:    ldl.sx %s1, (, %s0)
 ; CHECK-NEXT:    or %s2, 2, (0)1
-; CHECK-NEXT:  .LBB{{[0-9]+}}_1: # %atomicrmw.start
+; CHECK-NEXT:  .LBB9_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    or %s3, 0, %s1
 ; CHECK-NEXT:    cmpu.w %s4, %s1, %s2
 ; CHECK-NEXT:    or %s1, 1, (0)1
 ; CHECK-NEXT:    cmov.w.lt %s1, %s3, %s4
 ; CHECK-NEXT:    cas.w %s1, (%s0), %s3
-; CHECK-NEXT:    brne.w %s1, %s3, .LBB{{[0-9]+}}_1
+; CHECK-NEXT:    brne.w %s1, %s3, .LBB9_1
 ; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
 ; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
 ; CHECK-NEXT:    fencem 3
 ; CHECK-NEXT:    b.l.t (, %s10)
 entry:
   %0 = atomicrmw umin i32* @i, i32 1 seq_cst
   ret i32 %0
 }

llvm/test/CodeGen/X86/dagcombine-select.ll

; CHECK-NEXT:    retq
   %bo = urem i32 120, %sel
   ret i32 %bo
 }
 
 define i32 @sel_constants_shl_constant(i1 %cond) {
 ; CHECK-LABEL: sel_constants_shl_constant:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    notb %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    andl $1, %eax
-; CHECK-NEXT:    orl $2, %eax
-; CHECK-NEXT:    shll $8, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    movl $768, %ecx # imm = 0x300
+; CHECK-NEXT:    movl $512, %eax # imm = 0x200
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 2, i32 3
   %bo = shl i32 %sel, 8
   ret i32 %bo
 }
 
 define i32 @shl_constant_sel_constants(i1 %cond) {
 ; CHECK-LABEL: shl_constant_sel_constants:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    notb %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    setne %al
 ; CHECK-NEXT:    leal 4(,%rax,4), %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 2, i32 3
   %bo = shl i32 1, %sel
   ret i32 %bo
 }
 
 define i32 @lshr_constant_sel_constants(i1 %cond) {
 ; CHECK-LABEL: lshr_constant_sel_constants:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    andl $1, %edi
-; CHECK-NEXT:    leal 8(,%rdi,8), %eax
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    leal 8(,%rax,8), %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 2, i32 3
   %bo = lshr i32 64, %sel
   ret i32 %bo
 }
 
 define i32 @ashr_constant_sel_constants(i1 %cond) {
 ; CHECK-LABEL: ashr_constant_sel_constants:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    andl $1, %edi
-; CHECK-NEXT:    shll $4, %edi
-; CHECK-NEXT:    leal 16(%rdi), %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    movl $32, %ecx
+; CHECK-NEXT:    movl $16, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 2, i32 3
   %bo = ashr i32 128, %sel
   ret i32 %bo
 }
 
 define double @fsub_constant_sel_constants(i1 %cond) {
 ; CHECK-LABEL: fsub_constant_sel_constants:

llvm/test/CodeGen/X86/memcmp-more-load-pairs-x32.ll

 ; X86-NEXT:    jne .LBB9_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 2(%eax), %eax
 ; X86-NEXT:    movzbl 2(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB9_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind {
 ; X86-LABEL: length3_eq:

 ; X86-NEXT:    jne .LBB16_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB16_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind {
 ; X86-LABEL: length5_eq:

 ; X86-NEXT:    cmpl %esi, %edx
 ; X86-NEXT:    jne .LBB18_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB18_2
 ; X86-NEXT:  .LBB18_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB18_2: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c

 ; X86-NEXT:    movl 3(%esi), %ecx
 ; X86-NEXT:    movl 3(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB19_3
 ; X86-NEXT:  .LBB19_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB19_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 7) nounwind
   ret i32 %m
 }
 
 define i1 @length7_eq(ptr %X, ptr %Y) nounwind {

 ; X86-NEXT:    movl 3(%esi), %ecx
 ; X86-NEXT:    movl 3(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB21_3
 ; X86-NEXT:  .LBB21_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB21_3: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 7) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c

 ; X86-NEXT:    movl 4(%esi), %ecx
 ; X86-NEXT:    movl 4(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB22_3
 ; X86-NEXT:  .LBB22_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB22_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 8) nounwind
   ret i32 %m
 }
 
 define i1 @length8_eq(ptr %X, ptr %Y) nounwind {

 ; X86-NEXT:    movl 8(%esi), %ecx
 ; X86-NEXT:    movl 8(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB29_4
 ; X86-NEXT:  .LBB29_3: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB29_4: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 12) nounwind
   ret i32 %m
 }
 
 define i1 @length13_eq(ptr %X, ptr %Y) nounwind {

 ; X86-NEXT:    movl 12(%esi), %ecx
 ; X86-NEXT:    movl 12(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB33_5
 ; X86-NEXT:  .LBB33_4: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB33_5: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 16) nounwind
   ret i32 %m
 }
 
 define i1 @length16_eq(ptr %x, ptr %y) nounwind {

 ; X86-NEXT:    movl 12(%esi), %ecx
 ; X86-NEXT:    movl 12(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB35_5
 ; X86-NEXT:  .LBB35_4: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB35_5: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i32 16) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp

 ; X86-NEXT:    movl 12(%esi), %eax
 ; X86-NEXT:    movl 12(%edx), %ecx
 ; X86-NEXT:    bswapl %eax
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %ecx, %eax
 ; X86-NEXT:    je .LBB36_5
 ; X86-NEXT:  .LBB36_4: # %res_block
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    setae %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    movl $1, %eax
+; X86-NEXT:    movl $-1, %edx
+; X86-NEXT:    cmovael %eax, %edx
 ; X86-NEXT:  .LBB36_5: # %endblock
 ; X86-NEXT:    testl %edx, %edx
 ; X86-NEXT:    setg %al
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i32 16) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp

llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll

 ; X64-NEXT:    cmpw %cx, %ax
 ; X64-NEXT:    jne .LBB9_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 2(%rdi), %eax
 ; X64-NEXT:    movzbl 2(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB9_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length3_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    cmpl %ecx, %eax
 ; X64-NEXT:    jne .LBB16_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB16_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length5_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB18_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 

 ; X64-NEXT:    movl 3(%rdi), %ecx
 ; X64-NEXT:    movl 3(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
 ; X64-NEXT:    je .LBB19_3
 ; X64-NEXT:  .LBB19_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB19_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 7) nounwind
   ret i32 %m
 }
 
 define i1 @length7_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length7_eq:

 ; X64-NEXT:    movl 3(%rdi), %ecx
 ; X64-NEXT:    movl 3(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
 ; X64-NEXT:    je .LBB21_3
 ; X64-NEXT:  .LBB21_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB21_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 7) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }

 ; X64-NEXT:    movl 8(%rdi), %ecx
 ; X64-NEXT:    movl 8(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB29_3
 ; X64-NEXT:  .LBB29_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB29_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 12) nounwind
   ret i32 %m
 }
 
 define i1 @length13_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length13_eq:

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB33_3
 ; X64-NEXT:  .LBB33_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB33_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 16) nounwind
   ret i32 %m
 }
 
 define i1 @length16_eq(ptr %x, ptr %y) nounwind {
 ; X64-SSE2-LABEL: length16_eq:

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB35_3
 ; X64-NEXT:  .LBB35_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB35_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 16) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 8(%rdi), %rax
 ; X64-NEXT:    movq 8(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
 ; X64-NEXT:    je .LBB36_3
 ; X64-NEXT:  .LBB36_2: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB36_3: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 16) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 16(%rdi), %rcx
 ; X64-NEXT:    movq 16(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB38_4
 ; X64-NEXT:  .LBB38_3: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB38_4: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 24) nounwind
   ret i32 %m
 }
 
 define i1 @length24_eq(ptr %x, ptr %y) nounwind {
 ; X64-SSE2-LABEL: length24_eq:

 ; X64-NEXT:    movq 16(%rdi), %rcx
 ; X64-NEXT:    movq 16(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB40_4
 ; X64-NEXT:  .LBB40_3: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB40_4: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 24) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 16(%rdi), %rax
 ; X64-NEXT:    movq 16(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
 ; X64-NEXT:    je .LBB41_4
 ; X64-NEXT:  .LBB41_3: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB41_4: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 24) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 23(%rdi), %rcx
 ; X64-NEXT:    movq 23(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB43_5
 ; X64-NEXT:  .LBB43_4: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB43_5: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 31) nounwind
   ret i32 %m
 }
 
 define i1 @length31_eq(ptr %x, ptr %y) nounwind {
 ; X64-SSE2-LABEL: length31_eq:

 ; X64-NEXT:    movq 23(%rdi), %rcx
 ; X64-NEXT:    movq 23(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB45_5
 ; X64-NEXT:  .LBB45_4: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB45_5: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 31) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 23(%rdi), %rax
 ; X64-NEXT:    movq 23(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
 ; X64-NEXT:    je .LBB46_5
 ; X64-NEXT:  .LBB46_4: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB46_5: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 31) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 24(%rdi), %rcx
 ; X64-NEXT:    movq 24(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB49_5
 ; X64-NEXT:  .LBB49_4: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB49_5: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 32) nounwind
   ret i32 %m
 }
 
 ; PR33325 - https://bugs.llvm.org/show_bug.cgi?id=33325
 

 ; X64-NEXT:    movq 24(%rdi), %rcx
 ; X64-NEXT:    movq 24(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB51_5
 ; X64-NEXT:  .LBB51_4: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB51_5: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 32) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 24(%rdi), %rax
 ; X64-NEXT:    movq 24(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
 ; X64-NEXT:    je .LBB52_5
 ; X64-NEXT:  .LBB52_4: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB52_5: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 32) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }

llvm/test/CodeGen/X86/memcmp-optsize-x32.ll

 ; X86-NEXT:    cmpw %si, %dx
 ; X86-NEXT:    jne .LBB4_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 2(%eax), %eax
 ; X86-NEXT:    movzbl 2(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB4_2
 ; X86-NEXT:  .LBB4_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    cmpw %si, %dx
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:  .LBB4_2: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind optsize {

 ; X86-NEXT:    cmpl %esi, %edx
 ; X86-NEXT:    jne .LBB9_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB9_2
 ; X86-NEXT:  .LBB9_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:  .LBB9_2: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind optsize {

 ; X86-NEXT:    movl 4(%esi), %ecx
 ; X86-NEXT:    movl 4(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB11_3
 ; X86-NEXT:  .LBB11_2: # %res_block
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    decl %esi
 ; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    cmovbl %esi, %eax
 ; X86-NEXT:  .LBB11_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 8) nounwind
   ret i32 %m
 }
 
 define i1 @length8_eq(ptr %X, ptr %Y) nounwind optsize {

llvm/test/CodeGen/X86/memcmp-optsize.ll

 ; X64-NEXT:    cmpw %cx, %ax
 ; X64-NEXT:    jne .LBB4_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 2(%rdi), %eax
 ; X64-NEXT:    movzbl 2(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB4_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind optsize {
 ; X64-LABEL: length3_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    cmpl %ecx, %eax
 ; X64-NEXT:    jne .LBB9_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB9_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind optsize {
 ; X64-LABEL: length5_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    movl 8(%rdi), %ecx
 ; X64-NEXT:    movl 8(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB15_3
 ; X64-NEXT:  .LBB15_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB15_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 12) nounwind
   ret i32 %m
 }
 
 ; PR33329 - https://bugs.llvm.org/show_bug.cgi?id=33329
 

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB16_3
 ; X64-NEXT:  .LBB16_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB16_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 16) nounwind
   ret i32 %m
 }
 
 define i1 @length16_eq(ptr %x, ptr %y) nounwind optsize {
 ; X64-SSE2-LABEL: length16_eq:

llvm/test/CodeGen/X86/memcmp-pgso-x32.ll

 ; X86-NEXT:    cmpw %si, %dx
 ; X86-NEXT:    jne .LBB4_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 2(%eax), %eax
 ; X86-NEXT:    movzbl 2(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB4_2
 ; X86-NEXT:  .LBB4_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    cmpw %si, %dx
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:  .LBB4_2: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind !prof !14 {

 ; X86-NEXT:    cmpl %esi, %edx
 ; X86-NEXT:    jne .LBB9_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB9_2
 ; X86-NEXT:  .LBB9_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    decl %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
 ; X86-NEXT:  .LBB9_2: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind !prof !14 {

 ; X86-NEXT:    movl 4(%esi), %ecx
 ; X86-NEXT:    movl 4(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB11_3
 ; X86-NEXT:  .LBB11_2: # %res_block
+; X86-NEXT:    xorl %esi, %esi
+; X86-NEXT:    decl %esi
 ; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    incl %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    cmovbl %esi, %eax
 ; X86-NEXT:  .LBB11_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 8) nounwind
   ret i32 %m
 }
 
 define i1 @length8_eq(ptr %X, ptr %Y) nounwind !prof !14 {

llvm/test/CodeGen/X86/memcmp-pgso.ll

 ; X64-NEXT:    cmpw %cx, %ax
 ; X64-NEXT:    jne .LBB4_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 2(%rdi), %eax
 ; X64-NEXT:    movzbl 2(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB4_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind !prof !14 {
 ; X64-LABEL: length3_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    cmpl %ecx, %eax
 ; X64-NEXT:    jne .LBB9_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB9_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind !prof !14 {
 ; X64-LABEL: length5_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    movl 8(%rdi), %ecx
 ; X64-NEXT:    movl 8(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB15_3
 ; X64-NEXT:  .LBB15_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB15_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 12) nounwind
   ret i32 %m
 }
 
 ; PR33329 - https://bugs.llvm.org/show_bug.cgi?id=33329
 

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB16_3
 ; X64-NEXT:  .LBB16_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB16_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 16) nounwind
   ret i32 %m
 }
 
 define i1 @length16_eq(ptr %x, ptr %y) nounwind !prof !14 {
 ; X64-SSE2-LABEL: length16_eq:

llvm/test/CodeGen/X86/memcmp-x32.ll

 ; X86-NEXT:    jne .LBB11_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 2(%eax), %eax
 ; X86-NEXT:    movzbl 2(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB11_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind {
 ; X86-LABEL: length3_eq:

 ; X86-NEXT:    jne .LBB18_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ; X86-NEXT:  .LBB18_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind {
 ; X86-LABEL: length5_eq:

 ; X86-NEXT:    cmpl %esi, %edx
 ; X86-NEXT:    jne .LBB20_3
 ; X86-NEXT:  # %bb.1: # %loadbb1
 ; X86-NEXT:    movzbl 4(%eax), %eax
 ; X86-NEXT:    movzbl 4(%ecx), %ecx
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    jmp .LBB20_2
 ; X86-NEXT:  .LBB20_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB20_2: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c

 ; X86-NEXT:    movl 3(%esi), %ecx
 ; X86-NEXT:    movl 3(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB21_3
 ; X86-NEXT:  .LBB21_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB21_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 7) nounwind
   ret i32 %m
 }
 
 define i1 @length7_lt(ptr %X, ptr %Y) nounwind {

 ; X86-NEXT:    movl 3(%esi), %ecx
 ; X86-NEXT:    movl 3(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB22_3
 ; X86-NEXT:  .LBB22_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB22_3: # %endblock
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 7) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c

 ; X86-NEXT:    movl 4(%esi), %ecx
 ; X86-NEXT:    movl 4(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
 ; X86-NEXT:    je .LBB24_3
 ; X86-NEXT:  .LBB24_2: # %res_block
-; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
 ; X86-NEXT:  .LBB24_3: # %endblock
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i32 8) nounwind
   ret i32 %m
 }
 
 define i1 @length8_eq(ptr %X, ptr %Y) nounwind {

llvm/test/CodeGen/X86/memcmp.ll

 ; X64-NEXT:    cmpw %cx, %ax
 ; X64-NEXT:    jne .LBB11_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 2(%rdi), %eax
 ; X64-NEXT:    movzbl 2(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB11_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 3) nounwind
   ret i32 %m
 }
 
 define i1 @length3_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length3_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:    cmpl %ecx, %eax
 ; X64-NEXT:    jne .LBB18_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB18_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   ret i32 %m
 }
 
 define i1 @length5_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length5_eq:
 ; X64:       # %bb.0:

 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movzbl 4(%rdi), %eax
 ; X64-NEXT:    movzbl 4(%rsi), %ecx
 ; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
 ; X64-NEXT:  .LBB20_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 

 ; X64-NEXT:    movl 3(%rdi), %ecx
 ; X64-NEXT:    movl 3(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
 ; X64-NEXT:    je .LBB21_3
 ; X64-NEXT:  .LBB21_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB21_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 7) nounwind
   ret i32 %m
 }
 
 define i1 @length7_lt(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length7_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %ecx
 ; X64-NEXT:    movl (%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    cmpl %edx, %ecx
 ; X64-NEXT:    jne .LBB22_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movl 3(%rdi), %ecx
 ; X64-NEXT:    movl 3(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
 ; X64-NEXT:    je .LBB22_3
 ; X64-NEXT:  .LBB22_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB22_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 7) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }

 ; X64-NEXT:    movl 8(%rdi), %ecx
 ; X64-NEXT:    movl 8(%rsi), %edx
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    bswapl %edx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB31_3
 ; X64-NEXT:  .LBB31_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB31_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 12) nounwind
   ret i32 %m
 }
 
 define i1 @length13_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length13_eq:

 ; X64-NEXT:    movq 7(%rdi), %rcx
 ; X64-NEXT:    movq 7(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB34_3
 ; X64-NEXT:  .LBB34_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB34_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 15) nounwind
   ret i32 %m
 }
 
 define i1 @length15_lt(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length15_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rcx
 ; X64-NEXT:    movq (%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    jne .LBB35_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movq 7(%rdi), %rcx
 ; X64-NEXT:    movq 7(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB35_3
 ; X64-NEXT:  .LBB35_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB35_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 15) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }

 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movabsq $4051322327650219061, %rcx # imm = 0x3839303132333435
 ; X64-NEXT:    movq 7(%rdi), %rdx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rcx, %rdx
 ; X64-NEXT:    je .LBB36_3
 ; X64-NEXT:  .LBB36_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rcx, %rdx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB36_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i64 15) nounwind
   ret i32 %m
 }
 
 define i1 @length15_eq(ptr %X, ptr %Y) nounwind {
 ; X64-LABEL: length15_eq:

 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movabsq $4051322327650219061, %rax # imm = 0x3839303132333435
 ; X64-NEXT:    movq 7(%rdi), %rcx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rax, %rcx
 ; X64-NEXT:    je .LBB38_3
 ; X64-NEXT:  .LBB38_2: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rax, %rcx
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB38_3: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr getelementptr inbounds ([513 x i8], ptr @.str, i32 0, i32 1), i64 15) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB39_3
 ; X64-NEXT:  .LBB39_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB39_3: # %endblock
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(ptr %X, ptr %Y, i64 16) nounwind
   ret i32 %m
 }
 
 define i1 @length16_eq(ptr %x, ptr %y) nounwind {
 ; X64-SSE2-LABEL: length16_eq:

 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
 ; X64-NEXT:    je .LBB41_3
 ; X64-NEXT:  .LBB41_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %ecx, %eax
 ; X64-NEXT:  .LBB41_3: # %endblock
 ; X64-NEXT:    shrl $31, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 16) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }

 ; X64-NEXT:    movq 8(%rdi), %rax
 ; X64-NEXT:    movq 8(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
 ; X64-NEXT:    je .LBB42_3
 ; X64-NEXT:  .LBB42_2: # %res_block
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovael %eax, %edx
 ; X64-NEXT:  .LBB42_3: # %endblock
 ; X64-NEXT:    testl %edx, %edx
 ; X64-NEXT:    setg %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(ptr %x, ptr %y, i64 16) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }

llvm/test/CodeGen/X86/midpoint-int.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
 ; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
 
 ; These test cases are inspired by C++2a std::midpoint().
 ; See https://bugs.llvm.org/show_bug.cgi?id=40965
 
 ; ---------------------------------------------------------------------------- ;
 ; 32-bit width
 ; ---------------------------------------------------------------------------- ;
 
 ; Values come from regs
 
 define i32 @scalar_i32_signed_reg_reg(i32 %a1, i32 %a2) nounwind {
 ; X64-LABEL: scalar_i32_signed_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %esi, %edi
-; X64-NEXT:    setle %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovlel %ecx, %eax
 ; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    cmovgl %esi, %ecx
 ; X64-NEXT:    cmovgl %edi, %esi
 ; X64-NEXT:    subl %ecx, %esi
 ; X64-NEXT:    shrl %esi
 ; X64-NEXT:    imull %esi, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i32_signed_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB0_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB0_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB0_3
-; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB0_3
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB0_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
   %t6 = select i1 %t3, i32 %a1, i32 %a2
   %t7 = sub i32 %t6, %t5
   %t8 = lshr i32 %t7, 1
   %t9 = mul nsw i32 %t8, %t4 ; signed
   %a10 = add nsw i32 %t9, %a1 ; signed
   ret i32 %a10
 }
 
 define i32 @scalar_i32_unsigned_reg_reg(i32 %a1, i32 %a2) nounwind {
 ; X64-LABEL: scalar_i32_unsigned_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %esi, %edi
-; X64-NEXT:    setbe %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovbel %ecx, %eax
 ; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    cmoval %esi, %ecx
 ; X64-NEXT:    cmoval %edi, %esi
 ; X64-NEXT:    subl %ecx, %esi
 ; X64-NEXT:    shrl %esi
 ; X64-NEXT:    imull %esi, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i32_unsigned_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    setbe %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    ja .LBB1_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jbe .LBB1_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB1_3
-; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB1_3
+; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB1_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %t3 = icmp ugt i32 %a1, %a2
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
   %t6 = select i1 %t3, i32 %a1, i32 %a2
   %t7 = sub i32 %t6, %t5
   %t8 = lshr i32 %t7, 1
   %t9 = mul i32 %t8, %t4
   %a10 = add i32 %t9, %a1
   ret i32 %a10
 }
 
 ; Values are loaded. Only check signed case.
 
 define i32 @scalar_i32_signed_mem_reg(ptr %a1_addr, i32 %a2) nounwind {
 ; X64-LABEL: scalar_i32_signed_mem_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %ecx
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpl %esi, %ecx
-; X64-NEXT:    setle %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovlel %edx, %eax
 ; X64-NEXT:    movl %ecx, %edx
 ; X64-NEXT:    cmovgl %esi, %edx
 ; X64-NEXT:    cmovgl %ecx, %esi
 ; X64-NEXT:    subl %edx, %esi
 ; X64-NEXT:    shrl %esi
 ; X64-NEXT:    imull %esi, %eax
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i32_signed_mem_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB2_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB2_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB2_3
-; X86-NEXT:  .LBB2_1:
+; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB2_3
+; X86-NEXT:  .LBB2_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB2_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %a1 = load i32, ptr %a1_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
   %t6 = select i1 %t3, i32 %a1, i32 %a2
   %t7 = sub i32 %t6, %t5
   %t8 = lshr i32 %t7, 1
   %t9 = mul nsw i32 %t8, %t4 ; signed
   %a10 = add nsw i32 %t9, %a1 ; signed
   ret i32 %a10
 }
 
 define i32 @scalar_i32_signed_reg_mem(i32 %a1, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i32_signed_reg_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rsi), %eax
-; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    cmpl %eax, %edi
-; X64-NEXT:    setle %cl
-; X64-NEXT:    leal -1(%rcx,%rcx), %ecx
-; X64-NEXT:    movl %edi, %edx
-; X64-NEXT:    cmovgl %eax, %edx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $-1, %edx
+; X64-NEXT:    cmovlel %ecx, %edx
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    cmovgl %eax, %ecx
 ; X64-NEXT:    cmovgl %edi, %eax
-; X64-NEXT:    subl %edx, %eax
+; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    shrl %eax
-; X64-NEXT:    imull %ecx, %eax
+; X64-NEXT:    imull %edx, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i32_signed_reg_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl (%eax), %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB3_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB3_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB3_3
-; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB3_3
+; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB3_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %a2 = load i32, ptr %a2_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
   %t6 = select i1 %t3, i32 %a1, i32 %a2
   %t7 = sub i32 %t6, %t5
   %t8 = lshr i32 %t7, 1
   %t9 = mul nsw i32 %t8, %t4 ; signed
   %a10 = add nsw i32 %t9, %a1 ; signed
   ret i32 %a10
 }
 
 define i32 @scalar_i32_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i32_signed_mem_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %ecx
 ; X64-NEXT:    movl (%rsi), %eax
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpl %eax, %ecx
-; X64-NEXT:    setle %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
-; X64-NEXT:    movl %ecx, %esi
-; X64-NEXT:    cmovgl %eax, %esi
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movl $-1, %esi
+; X64-NEXT:    cmovlel %edx, %esi
+; X64-NEXT:    movl %ecx, %edx
+; X64-NEXT:    cmovgl %eax, %edx
 ; X64-NEXT:    cmovgl %ecx, %eax
-; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edx, %eax
 ; X64-NEXT:    shrl %eax
-; X64-NEXT:    imull %edx, %eax
+; X64-NEXT:    imull %esi, %eax
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i32_signed_mem_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
 ; X86-NEXT:    movl (%eax), %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB4_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB4_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB4_3
-; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    movl $-1, %edx
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB4_3
+; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB4_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %a1 = load i32, ptr %a1_addr

 ; 64-bit width
 ; ---------------------------------------------------------------------------- ;
 
 ; Values come from regs
 
 define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X64-LABEL: scalar_i64_signed_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rsi, %rdi
-; X64-NEXT:    setle %al
-; X64-NEXT:    leaq -1(%rax,%rax), %rax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movq $-1, %rax
+; X64-NEXT:    cmovleq %rcx, %rax
 ; X64-NEXT:    movq %rdi, %rcx
 ; X64-NEXT:    cmovgq %rsi, %rcx
 ; X64-NEXT:    cmovgq %rdi, %rsi
 ; X64-NEXT:    subq %rcx, %rsi
 ; X64-NEXT:    shrq %rsi
 ; X64-NEXT:    imulq %rsi, %rax
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq
; X86-NEXT:    retl
   %t9 = mul nsw i64 %t8, %t4 ; signed
   %a10 = add nsw i64 %t9, %a1 ; signed
   ret i64 %a10
 }
 
 define i64 @scalar_i64_unsigned_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X64-LABEL: scalar_i64_unsigned_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rsi, %rdi
-; X64-NEXT:    setbe %al
-; X64-NEXT:    leaq -1(%rax,%rax), %rax
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movq $-1, %rax
+; X64-NEXT:    cmovbeq %rcx, %rax
 ; X64-NEXT:    movq %rdi, %rcx
 ; X64-NEXT:    cmovaq %rsi, %rcx
 ; X64-NEXT:    cmovaq %rdi, %rsi
 ; X64-NEXT:    subq %rcx, %rsi
 ; X64-NEXT:    shrq %rsi
 ; X64-NEXT:    imulq %rsi, %rax
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq

 }
 
 ; Values are loaded. Only check signed case.
 
 define i64 @scalar_i64_signed_mem_reg(ptr %a1_addr, i64 %a2) nounwind {
 ; X64-LABEL: scalar_i64_signed_mem_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rsi, %rcx
-; X64-NEXT:    setle %al
-; X64-NEXT:    leaq -1(%rax,%rax), %rax
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movq $-1, %rax
+; X64-NEXT:    cmovleq %rdx, %rax
 ; X64-NEXT:    movq %rcx, %rdx
 ; X64-NEXT:    cmovgq %rsi, %rdx
 ; X64-NEXT:    cmovgq %rcx, %rsi
 ; X64-NEXT:    subq %rdx, %rsi
 ; X64-NEXT:    shrq %rsi
 ; X64-NEXT:    imulq %rsi, %rax
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    retq
; X86-NEXT:    retl
   %a10 = add nsw i64 %t9, %a1 ; signed
   ret i64 %a10
 }
 
 define i64 @scalar_i64_signed_reg_mem(i64 %a1, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i64_signed_reg_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rsi), %rax
-; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    cmpq %rax, %rdi
-; X64-NEXT:    setle %cl
-; X64-NEXT:    leaq -1(%rcx,%rcx), %rcx
-; X64-NEXT:    movq %rdi, %rdx
-; X64-NEXT:    cmovgq %rax, %rdx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movq $-1, %rdx
+; X64-NEXT:    cmovleq %rcx, %rdx
+; X64-NEXT:    movq %rdi, %rcx
+; X64-NEXT:    cmovgq %rax, %rcx
 ; X64-NEXT:    cmovgq %rdi, %rax
-; X64-NEXT:    subq %rdx, %rax
+; X64-NEXT:    subq %rcx, %rax
 ; X64-NEXT:    shrq %rax
-; X64-NEXT:    imulq %rcx, %rax
+; X64-NEXT:    imulq %rdx, %rax
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i64_signed_reg_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi
; X86-NEXT:    retl
   ret i64 %a10
 }
 
 define i64 @scalar_i64_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i64_signed_mem_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rcx
 ; X64-NEXT:    movq (%rsi), %rax
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpq %rax, %rcx
-; X64-NEXT:    setle %dl
-; X64-NEXT:    leaq -1(%rdx,%rdx), %rdx
-; X64-NEXT:    movq %rcx, %rsi
-; X64-NEXT:    cmovgq %rax, %rsi
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movq $-1, %rsi
+; X64-NEXT:    cmovleq %rdx, %rsi
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    cmovgq %rax, %rdx
 ; X64-NEXT:    cmovgq %rcx, %rax
-; X64-NEXT:    subq %rsi, %rax
+; X64-NEXT:    subq %rdx, %rax
 ; X64-NEXT:    shrq %rax
-; X64-NEXT:    imulq %rdx, %rax
+; X64-NEXT:    imulq %rsi, %rax
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i64_signed_mem_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    pushl %ebx
 ; X86-NEXT:    pushl %edi

 ; 16-bit width
 ; ---------------------------------------------------------------------------- ;
 
 ; Values come from regs
 
 define i16 @scalar_i16_signed_reg_reg(i16 %a1, i16 %a2) nounwind {
 ; X64-LABEL: scalar_i16_signed_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpw %si, %di
-; X64-NEXT:    setle %al
-; X64-NEXT:    leal -1(%rax,%rax), %ecx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; X64-NEXT:    cmovlel %eax, %ecx
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    cmovgl %esi, %eax
 ; X64-NEXT:    cmovgl %edi, %esi
 ; X64-NEXT:    subl %eax, %esi
 ; X64-NEXT:    movzwl %si, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    imull %ecx, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i16_signed_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpw %ax, %cx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB10_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB10_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB10_3
-; X86-NEXT:  .LBB10_1:
+; X86-NEXT:    movl $65535, %edx # imm = 0xFFFF
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB10_3
+; X86-NEXT:  .LBB10_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB10_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
   %t3 = icmp sgt i16 %a1, %a2 ; signed
   %t4 = select i1 %t3, i16 -1, i16 1
   %t5 = select i1 %t3, i16 %a2, i16 %a1
   %t6 = select i1 %t3, i16 %a1, i16 %a2
   %t7 = sub i16 %t6, %t5
   %t8 = lshr i16 %t7, 1
   %t9 = mul nsw i16 %t8, %t4 ; signed
   %a10 = add nsw i16 %t9, %a1 ; signed
   ret i16 %a10
 }
 
 define i16 @scalar_i16_unsigned_reg_reg(i16 %a1, i16 %a2) nounwind {
 ; X64-LABEL: scalar_i16_unsigned_reg_reg:
 ; X64:       # %bb.0:
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpw %si, %di
-; X64-NEXT:    setbe %al
-; X64-NEXT:    leal -1(%rax,%rax), %ecx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; X64-NEXT:    cmovbel %eax, %ecx
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    cmoval %esi, %eax
 ; X64-NEXT:    cmoval %edi, %esi
 ; X64-NEXT:    subl %eax, %esi
 ; X64-NEXT:    movzwl %si, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    imull %ecx, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i16_unsigned_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpw %ax, %cx
-; X86-NEXT:    setbe %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    ja .LBB11_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jbe .LBB11_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB11_3
-; X86-NEXT:  .LBB11_1:
+; X86-NEXT:    movl $65535, %edx # imm = 0xFFFF
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB11_3
+; X86-NEXT:  .LBB11_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB11_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    popl %esi

 }
 
 ; Values are loaded. Only check signed case.
 
 define i16 @scalar_i16_signed_mem_reg(ptr %a1_addr, i16 %a2) nounwind {
 ; X64-LABEL: scalar_i16_signed_mem_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %ecx
-; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpw %si, %cx
-; X64-NEXT:    setle %al
-; X64-NEXT:    leal -1(%rax,%rax), %edx
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; X64-NEXT:    cmovlel %eax, %edx
 ; X64-NEXT:    movl %ecx, %eax
 ; X64-NEXT:    cmovgl %esi, %eax
 ; X64-NEXT:    cmovgl %ecx, %esi
 ; X64-NEXT:    subl %eax, %esi
 ; X64-NEXT:    movzwl %si, %eax
 ; X64-NEXT:    shrl %eax
 ; X64-NEXT:    imull %edx, %eax
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i16_signed_mem_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpw %ax, %cx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB12_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB12_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB12_3
-; X86-NEXT:  .LBB12_1:
+; X86-NEXT:    movl $65535, %edx # imm = 0xFFFF
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB12_3
+; X86-NEXT:  .LBB12_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB12_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    popl %esi
; X86-NEXT:    retl
   %a10 = add nsw i16 %t9, %a1 ; signed
   ret i16 %a10
 }
 
 define i16 @scalar_i16_signed_reg_mem(i16 %a1, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i16_signed_reg_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rsi), %eax
-; X64-NEXT:    xorl %ecx, %ecx
 ; X64-NEXT:    cmpw %ax, %di
-; X64-NEXT:    setle %cl
-; X64-NEXT:    leal -1(%rcx,%rcx), %ecx
-; X64-NEXT:    movl %edi, %edx
-; X64-NEXT:    cmovgl %eax, %edx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; X64-NEXT:    cmovlel %ecx, %edx
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    cmovgl %eax, %ecx
 ; X64-NEXT:    cmovgl %edi, %eax
-; X64-NEXT:    subl %edx, %eax
+; X64-NEXT:    subl %ecx, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    shrl %eax
-; X64-NEXT:    imull %ecx, %eax
+; X64-NEXT:    imull %edx, %eax
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i16_signed_reg_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpw %ax, %cx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB13_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB13_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB13_3
-; X86-NEXT:  .LBB13_1:
+; X86-NEXT:    movl $65535, %edx # imm = 0xFFFF
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB13_3
+; X86-NEXT:  .LBB13_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB13_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    popl %esi
; X86-NEXT:    retl
   ret i16 %a10
 }
 
 define i16 @scalar_i16_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i16_signed_mem_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %ecx
 ; X64-NEXT:    movzwl (%rsi), %eax
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpw %ax, %cx
-; X64-NEXT:    setle %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
-; X64-NEXT:    movl %ecx, %esi
-; X64-NEXT:    cmovgl %eax, %esi
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movl $65535, %esi # imm = 0xFFFF
+; X64-NEXT:    cmovlel %edx, %esi
+; X64-NEXT:    movl %ecx, %edx
+; X64-NEXT:    cmovgl %eax, %edx
 ; X64-NEXT:    cmovgl %ecx, %eax
-; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edx, %eax
 ; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    shrl %eax
-; X64-NEXT:    imull %edx, %eax
+; X64-NEXT:    imull %esi, %eax
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i16_signed_mem_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
 ; X86-NEXT:    movzwl (%eax), %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpw %ax, %cx
-; X86-NEXT:    setle %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
-; X86-NEXT:    jg .LBB14_1
+; X86-NEXT:    movl $1, %edx
+; X86-NEXT:    jle .LBB14_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movl %ecx, %esi
-; X86-NEXT:    jmp .LBB14_3
-; X86-NEXT:  .LBB14_1:
+; X86-NEXT:    movl $65535, %edx # imm = 0xFFFF
 ; X86-NEXT:    movl %eax, %esi
 ; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    jmp .LBB14_3
+; X86-NEXT:  .LBB14_1:
+; X86-NEXT:    movl %ecx, %esi
 ; X86-NEXT:  .LBB14_3:
 ; X86-NEXT:    subl %esi, %eax
 ; X86-NEXT:    movzwl %ax, %eax
 ; X86-NEXT:    shrl %eax
 ; X86-NEXT:    imull %edx, %eax
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    popl %esi

 
 ; Values come from regs
 
 define i8 @scalar_i8_signed_reg_reg(i8 %a1, i8 %a2) nounwind {
 ; X64-LABEL: scalar_i8_signed_reg_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    cmpb %al, %dil
-; X64-NEXT:    setle %cl
-; X64-NEXT:    movl %edi, %edx
-; X64-NEXT:    cmovgl %esi, %edx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    cmovlel %ecx, %edx
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    cmovgl %esi, %ecx
 ; X64-NEXT:    cmovgl %edi, %eax
-; X64-NEXT:    addb %cl, %cl
-; X64-NEXT:    decb %cl
-; X64-NEXT:    subb %dl, %al
+; X64-NEXT:    subb %cl, %al
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    mulb %cl
+; X64-NEXT:    mulb %dl
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i8_signed_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cmpb %al, %cl
-; X86-NEXT:    setle %dl
-; X86-NEXT:    jg .LBB15_1
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    jle .LBB15_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movb %cl, %ah
-; X86-NEXT:    jmp .LBB15_3
-; X86-NEXT:  .LBB15_1:
+; X86-NEXT:    movb $-1, %dl
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    jmp .LBB15_3
+; X86-NEXT:  .LBB15_1:
+; X86-NEXT:    movb %cl, %ah
 ; X86-NEXT:  .LBB15_3:
 ; X86-NEXT:    subb %ah, %al
-; X86-NEXT:    addb %dl, %dl
-; X86-NEXT:    decb %dl
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    mulb %dl
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    retl
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
   %t6 = select i1 %t3, i8 %a1, i8 %a2
   %t7 = sub i8 %t6, %t5
   %t8 = lshr i8 %t7, 1
   %t9 = mul nsw i8 %t8, %t4 ; signed
   %a10 = add nsw i8 %t9, %a1 ; signed
   ret i8 %a10
 }
 
 define i8 @scalar_i8_unsigned_reg_reg(i8 %a1, i8 %a2) nounwind {
 ; X64-LABEL: scalar_i8_unsigned_reg_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    cmpb %al, %dil
-; X64-NEXT:    setbe %cl
-; X64-NEXT:    movl %edi, %edx
-; X64-NEXT:    cmoval %esi, %edx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    cmovbel %ecx, %edx
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    cmoval %esi, %ecx
 ; X64-NEXT:    cmoval %edi, %eax
-; X64-NEXT:    addb %cl, %cl
-; X64-NEXT:    decb %cl
-; X64-NEXT:    subb %dl, %al
+; X64-NEXT:    subb %cl, %al
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    mulb %cl
+; X64-NEXT:    mulb %dl
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i8_unsigned_reg_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cmpb %al, %cl
-; X86-NEXT:    setbe %dl
-; X86-NEXT:    ja .LBB16_1
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    jbe .LBB16_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movb %cl, %ah
-; X86-NEXT:    jmp .LBB16_3
-; X86-NEXT:  .LBB16_1:
+; X86-NEXT:    movb $-1, %dl
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    jmp .LBB16_3
+; X86-NEXT:  .LBB16_1:
+; X86-NEXT:    movb %cl, %ah
 ; X86-NEXT:  .LBB16_3:
 ; X86-NEXT:    subb %ah, %al
-; X86-NEXT:    addb %dl, %dl
-; X86-NEXT:    decb %dl
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    mulb %dl
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    retl
   %t3 = icmp ugt i8 %a1, %a2
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
   %t6 = select i1 %t3, i8 %a1, i8 %a2
   %t7 = sub i8 %t6, %t5
   %t8 = lshr i8 %t7, 1
   %t9 = mul i8 %t8, %t4
   %a10 = add i8 %t9, %a1
   ret i8 %a10
 }
 
 ; Values are loaded. Only check signed case.
 
 define i8 @scalar_i8_signed_mem_reg(ptr %a1_addr, i8 %a2) nounwind {
 ; X64-LABEL: scalar_i8_signed_mem_reg:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl (%rdi), %ecx
 ; X64-NEXT:    cmpb %sil, %cl
-; X64-NEXT:    setle %dl
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    cmovlel %eax, %edx
 ; X64-NEXT:    movl %ecx, %edi
 ; X64-NEXT:    cmovgl %esi, %edi
 ; X64-NEXT:    movl %ecx, %eax
 ; X64-NEXT:    cmovlel %esi, %eax
-; X64-NEXT:    addb %dl, %dl
-; X64-NEXT:    decb %dl
 ; X64-NEXT:    subb %dil, %al
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
 ; X64-NEXT:    mulb %dl
 ; X64-NEXT:    addb %cl, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i8_signed_mem_reg:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl (%ecx), %ecx
 ; X86-NEXT:    cmpb %al, %cl
-; X86-NEXT:    setle %dl
-; X86-NEXT:    jg .LBB17_1
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    jle .LBB17_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movb %cl, %ah
-; X86-NEXT:    jmp .LBB17_3
-; X86-NEXT:  .LBB17_1:
+; X86-NEXT:    movb $-1, %dl
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    jmp .LBB17_3
+; X86-NEXT:  .LBB17_1:
+; X86-NEXT:    movb %cl, %ah
 ; X86-NEXT:  .LBB17_3:
 ; X86-NEXT:    subb %ah, %al
-; X86-NEXT:    addb %dl, %dl
-; X86-NEXT:    decb %dl
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    mulb %dl
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    retl
   %a1 = load i8, ptr %a1_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
   %t6 = select i1 %t3, i8 %a1, i8 %a2
   %t7 = sub i8 %t6, %t5
   %t8 = lshr i8 %t7, 1
   %t9 = mul nsw i8 %t8, %t4 ; signed
   %a10 = add nsw i8 %t9, %a1 ; signed
   ret i8 %a10
 }
 
 define i8 @scalar_i8_signed_reg_mem(i8 %a1, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i8_signed_reg_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl (%rsi), %eax
 ; X64-NEXT:    cmpb %al, %dil
-; X64-NEXT:    setle %cl
-; X64-NEXT:    movl %edi, %edx
-; X64-NEXT:    cmovgl %eax, %edx
+; X64-NEXT:    movl $1, %ecx
+; X64-NEXT:    movl $255, %edx
+; X64-NEXT:    cmovlel %ecx, %edx
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    cmovgl %eax, %ecx
 ; X64-NEXT:    cmovgl %edi, %eax
-; X64-NEXT:    addb %cl, %cl
-; X64-NEXT:    decb %cl
-; X64-NEXT:    subb %dl, %al
+; X64-NEXT:    subb %cl, %al
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    mulb %cl
+; X64-NEXT:    mulb %dl
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i8_signed_reg_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movzbl (%eax), %eax
 ; X86-NEXT:    cmpb %al, %cl
-; X86-NEXT:    setle %dl
-; X86-NEXT:    jg .LBB18_1
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    jle .LBB18_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movb %cl, %ah
-; X86-NEXT:    jmp .LBB18_3
-; X86-NEXT:  .LBB18_1:
+; X86-NEXT:    movb $-1, %dl
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    jmp .LBB18_3
+; X86-NEXT:  .LBB18_1:
+; X86-NEXT:    movb %cl, %ah
 ; X86-NEXT:  .LBB18_3:
 ; X86-NEXT:    subb %ah, %al
-; X86-NEXT:    addb %dl, %dl
-; X86-NEXT:    decb %dl
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    mulb %dl
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    retl
   %a2 = load i8, ptr %a2_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
   %t6 = select i1 %t3, i8 %a1, i8 %a2
   %t7 = sub i8 %t6, %t5
   %t8 = lshr i8 %t7, 1
   %t9 = mul nsw i8 %t8, %t4 ; signed
   %a10 = add nsw i8 %t9, %a1 ; signed
   ret i8 %a10
 }
 
 define i8 @scalar_i8_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
 ; X64-LABEL: scalar_i8_signed_mem_mem:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzbl (%rdi), %ecx
 ; X64-NEXT:    movzbl (%rsi), %eax
 ; X64-NEXT:    cmpb %al, %cl
-; X64-NEXT:    setle %dl
-; X64-NEXT:    movl %ecx, %esi
-; X64-NEXT:    cmovgl %eax, %esi
+; X64-NEXT:    movl $1, %edx
+; X64-NEXT:    movl $255, %esi
+; X64-NEXT:    cmovlel %edx, %esi
+; X64-NEXT:    movl %ecx, %edx
+; X64-NEXT:    cmovgl %eax, %edx
 ; X64-NEXT:    cmovgl %ecx, %eax
-; X64-NEXT:    addb %dl, %dl
-; X64-NEXT:    decb %dl
-; X64-NEXT:    subb %sil, %al
+; X64-NEXT:    subb %dl, %al
 ; X64-NEXT:    shrb %al
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    mulb %dl
+; X64-NEXT:    mulb %sil
 ; X64-NEXT:    addb %cl, %al
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: scalar_i8_signed_mem_mem:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzbl (%ecx), %ecx
 ; X86-NEXT:    movzbl (%eax), %eax
 ; X86-NEXT:    cmpb %al, %cl
-; X86-NEXT:    setle %dl
-; X86-NEXT:    jg .LBB19_1
+; X86-NEXT:    movb $1, %dl
+; X86-NEXT:    jle .LBB19_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    movb %cl, %ah
-; X86-NEXT:    jmp .LBB19_3
-; X86-NEXT:  .LBB19_1:
+; X86-NEXT:    movb $-1, %dl
 ; X86-NEXT:    movb %al, %ah
 ; X86-NEXT:    movb %cl, %al
+; X86-NEXT:    jmp .LBB19_3
+; X86-NEXT:  .LBB19_1:
+; X86-NEXT:    movb %cl, %ah
 ; X86-NEXT:  .LBB19_3:
 ; X86-NEXT:    subb %ah, %al
-; X86-NEXT:    addb %dl, %dl
-; X86-NEXT:    decb %dl
 ; X86-NEXT:    shrb %al
 ; X86-NEXT:    mulb %dl
 ; X86-NEXT:    addb %cl, %al
 ; X86-NEXT:    retl
   %a1 = load i8, ptr %a1_addr
   %a2 = load i8, ptr %a2_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
   %t6 = select i1 %t3, i8 %a1, i8 %a2
   %t7 = sub i8 %t6, %t5
   %t8 = lshr i8 %t7, 1
   %t9 = mul nsw i8 %t8, %t4 ; signed
   %a10 = add nsw i8 %t9, %a1 ; signed
   ret i8 %a10
 }

llvm/test/CodeGen/X86/select-1-or-neg1.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown                      | FileCheck %s --check-prefix=BASE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=slow-3ops-lea | FileCheck %s --check-prefix=SLOWLEA3
 
 ; TODO: Should the 'cmpl' be 'dec' instead?
 ; TODO: What if 'cmov' is 1 uop and full throughput (Ryzen)?
 
 define i32 @PR28968(i32 %x) {
 ; BASE-LABEL: PR28968:
 ; BASE:       # %bb.0:
-; BASE-NEXT:    xorl %eax, %eax
 ; BASE-NEXT:    cmpl $1, %edi
-; BASE-NEXT:    sete %al
-; BASE-NEXT:    leal -1(%rax,%rax), %eax
+; BASE-NEXT:    movl $-1, %eax
+; BASE-NEXT:    cmovel %edi, %eax
 ; BASE-NEXT:    retq
 ;
 ; SLOWLEA3-LABEL: PR28968:
 ; SLOWLEA3:       # %bb.0:
-; SLOWLEA3-NEXT:    xorl %eax, %eax
 ; SLOWLEA3-NEXT:    cmpl $1, %edi
-; SLOWLEA3-NEXT:    sete %al
-; SLOWLEA3-NEXT:    addl %eax, %eax
-; SLOWLEA3-NEXT:    decl %eax
+; SLOWLEA3-NEXT:    movl $-1, %eax
+; SLOWLEA3-NEXT:    cmovel %edi, %eax
 ; SLOWLEA3-NEXT:    retq
   %cmp = icmp eq i32 %x, 1
   %sel = select i1 %cmp, i32 1, i32 -1
   ret i32 %sel
 }
 

llvm/test/CodeGen/X86/select_const.ll

 
 ; If the constants differ by a small multiplier, use LEA.
 ; select Cond, C1, C2 --> add (mul (zext Cond), C1-C2), C2 --> LEA C2(Cond * (C1-C2))
 
 define i32 @select_lea_2(i1 zeroext %cond) {
 ; CHECK-LABEL: select_lea_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 -1, i32 1
   ret i32 %sel
 }
 
 define i64 @select_lea_3(i1 zeroext %cond) {
 ; CHECK-LABEL: select_lea_3:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    leaq -2(%rax,%rax,2), %rax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movq $-2, %rax
+; CHECK-NEXT:    cmovneq %rcx, %rax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i64 -2, i64 1
   ret i64 %sel
 }
 
 define i32 @select_lea_5(i1 zeroext %cond) {
 ; CHECK-LABEL: select_lea_5:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    leal -2(%rax,%rax,4), %eax
+; CHECK-NEXT:    movl $3, %ecx
+; CHECK-NEXT:    movl $-2, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 -2, i32 3
   ret i32 %sel
 }
 
 define i64 @select_lea_9(i1 zeroext %cond) {
 ; CHECK-LABEL: select_lea_9:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    leaq -7(%rax,%rax,8), %rax
+; CHECK-NEXT:    movl $2, %ecx
+; CHECK-NEXT:    movq $-7, %rax
+; CHECK-NEXT:    cmovneq %rcx, %rax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i64 -7, i64 2
   ret i64 %sel
 }
 
 ; Should this be 'sbb x,x' or 'sbb 0,x' with simpler LEA or add?
 
 define i64 @sel_1_2(i64 %x, i64 %y) {

 }
 
 ; No LEA with 8-bit, but this shouldn't need branches or cmov.
 
 define i8 @sel_1_neg1(i32 %x) {
 ; CHECK-LABEL: sel_1_neg1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    cmpl $43, %edi
-; CHECK-NEXT:    setge %al
-; CHECK-NEXT:    shlb $2, %al
-; CHECK-NEXT:    decb %al
+; CHECK-NEXT:    movl $3, %ecx
+; CHECK-NEXT:    movl $255, %eax
+; CHECK-NEXT:    cmovgel %ecx, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
   %cmp = icmp sgt i32 %x, 42
   %sel = select i1 %cmp, i8 3, i8 -1
   ret i8 %sel
 }
 
 ; We get an LEA for 16-bit because we ignore the high-bits.
 
 define i16 @sel_neg1_1(i32 %x) {
 ; CHECK-LABEL: sel_neg1_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl $43, %edi
-; CHECK-NEXT:    setl %al
-; CHECK-NEXT:    leal -1(,%rax,4), %eax
+; CHECK-NEXT:    movl $3, %ecx
+; CHECK-NEXT:    movl $65535, %eax # imm = 0xFFFF
+; CHECK-NEXT:    cmovll %ecx, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    retq
   %cmp = icmp sgt i32 %x, 42
   %sel = select i1 %cmp, i16 -1, i16 3
   ret i16 %sel
 }
 
 ; If the comparison is available, the predicate can be inverted.
 
 define i32 @sel_1_neg1_32(i32 %x) {
 ; CHECK-LABEL: sel_1_neg1_32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl $43, %edi
-; CHECK-NEXT:    setge %al
-; CHECK-NEXT:    leal -1(%rax,%rax,8), %eax
+; CHECK-NEXT:    movl $8, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovgel %ecx, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp sgt i32 %x, 42
   %sel = select i1 %cmp, i32 8, i32 -1
   ret i32 %sel
 }
 
 define i32 @sel_neg1_1_32(i32 %x) {
 ; CHECK-LABEL: sel_neg1_1_32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl $43, %edi
-; CHECK-NEXT:    setl %al
-; CHECK-NEXT:    leal -7(%rax,%rax,8), %eax
+; CHECK-NEXT:    movl $2, %ecx
+; CHECK-NEXT:    movl $-7, %eax
+; CHECK-NEXT:    cmovll %ecx, %eax
 ; CHECK-NEXT:    retq
   %cmp = icmp sgt i32 %x, 42
   %sel = select i1 %cmp, i32 -7, i32 2
   ret i32 %sel
 }
 
 
 ; If the constants differ by a large power-of-2, that can be a shift of the difference plus the smaller constant.
 ; select Cond, C1, C2 --> add (mul (zext Cond), C1-C2), C2
 
 define i8 @select_pow2_diff(i1 zeroext %cond) {
 ; CHECK-LABEL: select_pow2_diff:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    shlb $4, %dil
-; CHECK-NEXT:    leal 3(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    movl $19, %ecx
+; CHECK-NEXT:    movl $3, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i8 19, i8 3
   ret i8 %sel
 }
 
 define i16 @select_pow2_diff_invert(i1 zeroext %cond) {
 ; CHECK-LABEL: select_pow2_diff_invert:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    shll $6, %eax
-; CHECK-NEXT:    orl $7, %eax
+; CHECK-NEXT:    movl $71, %ecx
+; CHECK-NEXT:    movl $7, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i16 7, i16 71
   ret i16 %sel
 }
 
 define i32 @select_pow2_diff_neg(i1 zeroext %cond) {
 ; CHECK-LABEL: select_pow2_diff_neg:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    shll $4, %edi
-; CHECK-NEXT:    leal -25(%rdi), %eax
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    movl $-9, %ecx
+; CHECK-NEXT:    movl $-25, %eax
+; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 -9, i32 -25
   ret i32 %sel
 }
 
 define i64 @select_pow2_diff_neg_invert(i1 zeroext %cond) {
 ; CHECK-LABEL: select_pow2_diff_neg_invert:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xorb $1, %dil
-; CHECK-NEXT:    movzbl %dil, %eax
-; CHECK-NEXT:    shlq $7, %rax
-; CHECK-NEXT:    addq $-99, %rax
+; CHECK-NEXT:    movl $29, %ecx
+; CHECK-NEXT:    movq $-99, %rax
+; CHECK-NEXT:    cmovneq %rcx, %rax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i64 -99, i64 29
   ret i64 %sel
 }
 
 ; This doesn't need a branch, but don't do the wrong thing if subtraction of the constants overflows.
 
 define i8 @sel_67_neg125(i32 %x) {

llvm/test/CodeGen/X86/shift-parts.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
 ; PR4736
 
 %0 = type { i32, i8, [35 x i8] }
 
 @g_144 = external dso_local global %0, align 8              ; <ptr> [#uses=1]
 
 define i32 @int87(i32 %uint64p_8, i1 %cond) nounwind {
 ; CHECK-LABEL: int87:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movq g_144+24(%rip), %rax
 ; CHECK-NEXT:    movq g_144+16(%rip), %rcx
-; CHECK-NEXT:    movzbl %sil, %edx
-; CHECK-NEXT:    shll $6, %edx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_1: # %for.cond
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    testb $64, %dl
-; CHECK-NEXT:    movq %rcx, %rsi
-; CHECK-NEXT:    cmovneq %rax, %rsi
-; CHECK-NEXT:    testl %esi, %esi
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    testb $1, %sil
+; CHECK-NEXT:    setne %dl
+; CHECK-NEXT:    shll $6, %edx
+; CHECK-NEXT:    testb %dl, %dl
+; CHECK-NEXT:    movq %rcx, %rdx
+; CHECK-NEXT:    cmovneq %rax, %rdx
+; CHECK-NEXT:    testl %edx, %edx
 ; CHECK-NEXT:    je .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    movl $1, %eax
 ; CHECK-NEXT:    retq
 entry:
   %srcval4 = load i320, ptr @g_144, align 8 ; <i320> [#uses=1]
   br label %for.cond
 

llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll

 
 define i32 @stack_fold_ucomisd(double %a0, double %a1) {
 ; CHECK-LABEL: stack_fold_ucomisd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
-; CHECK-NEXT:    sete %al
-; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovel %ecx, %eax
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq double %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
   ret i32 %3
 }
 
 define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {

 
 define i32 @stack_fold_ucomiss(float %a0, float %a1) {
 ; CHECK-LABEL: stack_fold_ucomiss:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    vucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; CHECK-NEXT:    sete %al
-; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovel %ecx, %eax
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq float %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
   ret i32 %3
 }
 
 define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {

llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll

 
 define i32 @stack_fold_ucomisd(double %a0, double %a1) {
 ; CHECK-LABEL: stack_fold_ucomisd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movsd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    ucomisd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
-; CHECK-NEXT:    sete %al
-; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovel %ecx, %eax
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq double %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
   ret i32 %3
 }
 
 define i32 @stack_fold_ucomisd_int(<2 x double> %a0, <2 x double> %a1) {

 
 define i32 @stack_fold_ucomiss(float %a0, float %a1) {
 ; CHECK-LABEL: stack_fold_ucomiss:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movss %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    ucomiss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
-; CHECK-NEXT:    sete %al
-; CHECK-NEXT:    leal -1(%rax,%rax), %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovel %ecx, %eax
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
   %2 = fcmp ueq float %a0, %a1
   %3 = select i1 %2, i32 1, i32 -1
   ret i32 %3
 }
 
 define i32 @stack_fold_ucomiss_int(<4 x float> %a0, <4 x float> %a1) {

llvm/test/CodeGen/X86/zext-sext.ll

 ; CHECK-LABEL: func:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movslq (%rsi), %rax
 ; CHECK-NEXT:    movl $4, %esi
 ; CHECK-NEXT:    subq %rax, %rsi
 ; CHECK-NEXT:    movq (%rdx), %rax
 ; CHECK-NEXT:    movswl 8(%rdi), %edx
 ; CHECK-NEXT:    movswl (%rax,%rsi,2), %eax
-; CHECK-NEXT:    movl $1, %esi
+; CHECK-NEXT:    movl $-1, %esi
 ; CHECK-NEXT:    imull %edx, %eax
-; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    movl $1, %edx
 ; CHECK-NEXT:    addl $2138875574, %eax # imm = 0x7F7CA6B6
 ; CHECK-NEXT:    cmpl $-8608074, %eax # imm = 0xFF7CA6B6
 ; CHECK-NEXT:    movslq %eax, %rdi
-; CHECK-NEXT:    setl %dl
+; CHECK-NEXT:    cmovll %edx, %esi
 ; CHECK-NEXT:    cmpl $2138875574, %eax # imm = 0x7F7CA6B6
 ; CHECK-NEXT:    movq %rdi, %r8
-; CHECK-NEXT:    leal -1(%rdx,%rdx), %edx
-; CHECK-NEXT:    cmovll %edx, %esi
+; CHECK-NEXT:    cmovgel %edx, %esi
 ; CHECK-NEXT:    subq %rax, %r8
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpl $1, %esi
 ; CHECK-NEXT:    cmovneq %rax, %r8
 ; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    cmovnsq %rax, %r8
 ; CHECK-NEXT:    movq (%rcx), %rax
 ; CHECK-NEXT:    subq %r8, %rdi