[x86] Fix an amazing goof in the handling of sub, or, and xor lowering.

[chandlerc]

Summary

The comment for this code indicated that it should work similar to our
handling of add lowering above: if we see uses of an instruction other
than flag usage and store usage, it tries to avoid the specialized
X86ISD::* nodes that are designed for flag+op modeling.

Problem is, only the add case actually did this. In all the other cases,
the logic was incomplete and inverted. Any time the value was used by
a store, we bailed on the specialized X86ISD node.

Turns out, we have a *ton* of patterns designed around these nodes. We
should actually form them. I fixed the code to match what we do for add,
and it has quite a positive effect just within some of our test cases.
The only thing close to a regression I see is using:

notl %r
testl %r, %r

instead of:

xorl -1, %r

But we can add a pattern or something to fold that back out. The
improvements seem more than worth this.

Unless I'm missing any context here? I just made what seemed like
a "doh!" bug fix, and got ... much more in the way of generated code
changes than I was expecting....

[craig.topper]

Curious, what happens if you change it to just

if (UI->getOpcode() != ISD::CopyToReg && UI->getOpcode() != ISD::SETCC)

The "!= ISD::STORE" was added when the INC/DEC support was added to X86ISelDAGToDAG.cpp. Prior to that, ADD just had CopyToReg and SETCC. Sometime even earlier than that ADD was == STORE like SUB/XOR/OR/AND are now.

I don't think the comment above ADD was ever properly updated after != STORE was added.

[craig.topper]

We can leave the != ISD::STORE. It doesn't seem to have any visible effect right now. So consistency with ISD::ADD is best.

[chandlerc]

In D37096#852078, @craig.topper wrote:

We can leave the != ISD::STORE. It doesn't seem to have any visible effect right now. So consistency with ISD::ADD is best.

Thanks! and thanks for the detailed help! landing...

[craig.topper]

LGTM

lib/Target/X86/X86ISelLowering.cpp

case ISD::AND:
     LLVM_FALLTHROUGH;
   case ISD::SUB:
   case ISD::OR:
   case ISD::XOR:
     // Due to the ISEL shortcoming noted above, be conservative if this op is
     // likely to be selected as part of a load-modify-store instruction.
     for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
            UE = Op.getNode()->use_end(); UI != UE; ++UI)
-      if (UI->getOpcode() == ISD::STORE)
+      if (UI->getOpcode() != ISD::CopyToReg &&
+          UI->getOpcode() != ISD::SETCC &&
+          UI->getOpcode() != ISD::STORE)
         goto default_case;
 
     // Otherwise use a regular EFLAGS-setting instruction.
     switch (ArithOp.getOpcode()) {
     default: llvm_unreachable("unexpected operator!");
     case ISD::SUB: Opcode = X86ISD::SUB; break;
     case ISD::XOR: Opcode = X86ISD::XOR; break;
     case ISD::AND: Opcode = X86ISD::AND; break;

test/CodeGen/X86/atomic-minmax-i6432.ll

 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    pushl %esi
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    movl $4, %esi
 ; LINUX-NEXT:    .p2align 4, 0x90
 ; LINUX-NEXT:  .LBB0_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
 ; LINUX-NEXT:    cmpl %eax, %esi
+; LINUX-NEXT:    movl $0, %ecx
 ; LINUX-NEXT:    sbbl %edx, %ecx
-; LINUX-NEXT:    setl %cl
-; LINUX-NEXT:    andb $1, %cl
-; LINUX-NEXT:    movl %eax, %ebx
-; LINUX-NEXT:    jne .LBB0_3
-; LINUX-NEXT:  # BB#2: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; LINUX-NEXT:    movl $0, %ecx
+; LINUX-NEXT:    cmovll %edx, %ecx
 ; LINUX-NEXT:    movl $5, %ebx
-; LINUX-NEXT:  .LBB0_3: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; LINUX-NEXT:    testb %cl, %cl
-; LINUX-NEXT:    movl %edx, %ecx
-; LINUX-NEXT:    jne .LBB0_5
-; LINUX-NEXT:  # BB#4: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB0_1 Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
-; LINUX-NEXT:  .LBB0_5: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; LINUX-NEXT:    cmovll %eax, %ebx
 ; LINUX-NEXT:    lock cmpxchg8b sc64
 ; LINUX-NEXT:    jne .LBB0_1
-; LINUX-NEXT:  # BB#6: # %atomicrmw.end
+; LINUX-NEXT:  # BB#2: # %atomicrmw.end
 ; LINUX-NEXT:    popl %esi
 ; LINUX-NEXT:    popl %ebx
 ; LINUX-NEXT:    retl
 ;
 ; PIC-LABEL: atomic_max_i64:
 ; PIC:       ## BB#0: ## %entry
 ; PIC-NEXT:    pushl %ebx
 ; PIC-NEXT:    pushl %edi
 ; PIC-NEXT:    pushl %esi
 ; PIC-NEXT:    calll L0$pb
 ; PIC-NEXT:  L0$pb:
 ; PIC-NEXT:    popl %eax
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L0$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    movl $4, %edi
 ; PIC-NEXT:    .p2align 4, 0x90
 ; PIC-NEXT:  LBB0_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
 ; PIC-NEXT:    cmpl %eax, %edi
+; PIC-NEXT:    movl $0, %ecx
 ; PIC-NEXT:    sbbl %edx, %ecx
-; PIC-NEXT:    setl %cl
-; PIC-NEXT:    andb $1, %cl
-; PIC-NEXT:    movl %eax, %ebx
-; PIC-NEXT:    jne LBB0_3
-; PIC-NEXT:  ## BB#2: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; PIC-NEXT:    movl $0, %ecx
+; PIC-NEXT:    cmovll %edx, %ecx
 ; PIC-NEXT:    movl $5, %ebx
-; PIC-NEXT:  LBB0_3: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB0_1 Depth=1
-; PIC-NEXT:    testb %cl, %cl
-; PIC-NEXT:    movl %edx, %ecx
-; PIC-NEXT:    jne LBB0_5
-; PIC-NEXT:  ## BB#4: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB0_1 Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
-; PIC-NEXT:  LBB0_5: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; PIC-NEXT:    cmovll %eax, %ebx
 ; PIC-NEXT:    lock cmpxchg8b (%esi)
 ; PIC-NEXT:    jne LBB0_1
-; PIC-NEXT:  ## BB#6: ## %atomicrmw.end
+; PIC-NEXT:  ## BB#2: ## %atomicrmw.end
 ; PIC-NEXT:    popl %esi
 ; PIC-NEXT:    popl %edi
 ; PIC-NEXT:    popl %ebx
 ; PIC-NEXT:    retl
 ; PIC-NEXT:    ## -- End function
 entry:
   %max = atomicrmw max i64* @sc64, i64 5 acquire
   ret i64 %max
 }
 
 define i64 @atomic_min_i64() nounwind {
 ; LINUX-LABEL: atomic_min_i64:
 ; LINUX:       # BB#0: # %entry
 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    .p2align 4, 0x90
 ; LINUX-NEXT:  .LBB1_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl $7, %eax
 ; LINUX-NEXT:    movl %edx, %ecx
 ; LINUX-NEXT:    sbbl $0, %ecx
-; LINUX-NEXT:    setl %cl
-; LINUX-NEXT:    andb $1, %cl
-; LINUX-NEXT:    movl %eax, %ebx
-; LINUX-NEXT:    jne .LBB1_3
-; LINUX-NEXT:  # BB#2: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; LINUX-NEXT:    movl $0, %ecx
+; LINUX-NEXT:    cmovll %edx, %ecx
 ; LINUX-NEXT:    movl $6, %ebx
-; LINUX-NEXT:  .LBB1_3: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; LINUX-NEXT:    testb %cl, %cl
-; LINUX-NEXT:    movl %edx, %ecx
-; LINUX-NEXT:    jne .LBB1_5
-; LINUX-NEXT:  # BB#4: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
-; LINUX-NEXT:  .LBB1_5: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; LINUX-NEXT:    cmovll %eax, %ebx
 ; LINUX-NEXT:    lock cmpxchg8b sc64
 ; LINUX-NEXT:    jne .LBB1_1
-; LINUX-NEXT:  # BB#6: # %atomicrmw.end
+; LINUX-NEXT:  # BB#2: # %atomicrmw.end
 ; LINUX-NEXT:    popl %ebx
 ; LINUX-NEXT:    retl
 ;
 ; PIC-LABEL: atomic_min_i64:
 ; PIC:       ## BB#0: ## %entry
 ; PIC-NEXT:    pushl %ebx
 ; PIC-NEXT:    pushl %esi
 ; PIC-NEXT:    calll L1$pb
 ; PIC-NEXT:  L1$pb:
 ; PIC-NEXT:    popl %eax
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L1$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    .p2align 4, 0x90
 ; PIC-NEXT:  LBB1_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl $7, %eax
 ; PIC-NEXT:    movl %edx, %ecx
 ; PIC-NEXT:    sbbl $0, %ecx
-; PIC-NEXT:    setl %cl
-; PIC-NEXT:    andb $1, %cl
-; PIC-NEXT:    movl %eax, %ebx
-; PIC-NEXT:    jne LBB1_3
-; PIC-NEXT:  ## BB#2: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB1_1 Depth=1
+; PIC-NEXT:    movl $0, %ecx
+; PIC-NEXT:    cmovll %edx, %ecx
 ; PIC-NEXT:    movl $6, %ebx
-; PIC-NEXT:  LBB1_3: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB1_1 Depth=1
-; PIC-NEXT:    testb %cl, %cl
-; PIC-NEXT:    movl %edx, %ecx
-; PIC-NEXT:    jne LBB1_5
-; PIC-NEXT:  ## BB#4: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB1_1 Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
-; PIC-NEXT:  LBB1_5: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB1_1 Depth=1
+; PIC-NEXT:    cmovll %eax, %ebx
 ; PIC-NEXT:    lock cmpxchg8b (%esi)
 ; PIC-NEXT:    jne LBB1_1
-; PIC-NEXT:  ## BB#6: ## %atomicrmw.end
+; PIC-NEXT:  ## BB#2: ## %atomicrmw.end
 ; PIC-NEXT:    popl %esi
 ; PIC-NEXT:    popl %ebx
 ; PIC-NEXT:    retl
 ; PIC-NEXT:    ## -- End function
 entry:
   %min = atomicrmw min i64* @sc64, i64 6 acquire
   ret i64 %min
 }
 
 define i64 @atomic_umax_i64() nounwind {
 ; LINUX-LABEL: atomic_umax_i64:
 ; LINUX:       # BB#0: # %entry
 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    pushl %esi
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    movl $7, %esi
 ; LINUX-NEXT:    .p2align 4, 0x90
 ; LINUX-NEXT:  .LBB2_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
 ; LINUX-NEXT:    cmpl %eax, %esi
+; LINUX-NEXT:    movl $0, %ecx
 ; LINUX-NEXT:    sbbl %edx, %ecx
-; LINUX-NEXT:    setb %cl
-; LINUX-NEXT:    andb $1, %cl
-; LINUX-NEXT:    movl %eax, %ebx
-; LINUX-NEXT:    jne .LBB2_3
-; LINUX-NEXT:  # BB#2: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB2_1 Depth=1
+; LINUX-NEXT:    movl $0, %ecx
+; LINUX-NEXT:    cmovbl %edx, %ecx
 ; LINUX-NEXT:    movl $7, %ebx
-; LINUX-NEXT:  .LBB2_3: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB2_1 Depth=1
-; LINUX-NEXT:    testb %cl, %cl
-; LINUX-NEXT:    movl %edx, %ecx
-; LINUX-NEXT:    jne .LBB2_5
-; LINUX-NEXT:  # BB#4: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB2_1 Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
-; LINUX-NEXT:  .LBB2_5: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB2_1 Depth=1
+; LINUX-NEXT:    cmovbl %eax, %ebx
 ; LINUX-NEXT:    lock cmpxchg8b sc64
 ; LINUX-NEXT:    jne .LBB2_1
-; LINUX-NEXT:  # BB#6: # %atomicrmw.end
+; LINUX-NEXT:  # BB#2: # %atomicrmw.end
 ; LINUX-NEXT:    popl %esi
 ; LINUX-NEXT:    popl %ebx
 ; LINUX-NEXT:    retl
 ;
 ; PIC-LABEL: atomic_umax_i64:
 ; PIC:       ## BB#0: ## %entry
 ; PIC-NEXT:    pushl %ebx
 ; PIC-NEXT:    pushl %edi
 ; PIC-NEXT:    pushl %esi
 ; PIC-NEXT:    calll L2$pb
 ; PIC-NEXT:  L2$pb:
 ; PIC-NEXT:    popl %eax
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L2$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    movl $7, %edi
 ; PIC-NEXT:    .p2align 4, 0x90
 ; PIC-NEXT:  LBB2_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
 ; PIC-NEXT:    cmpl %eax, %edi
+; PIC-NEXT:    movl $0, %ecx
 ; PIC-NEXT:    sbbl %edx, %ecx
-; PIC-NEXT:    setb %cl
-; PIC-NEXT:    andb $1, %cl
-; PIC-NEXT:    movl %eax, %ebx
-; PIC-NEXT:    jne LBB2_3
-; PIC-NEXT:  ## BB#2: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB2_1 Depth=1
+; PIC-NEXT:    movl $0, %ecx
+; PIC-NEXT:    cmovbl %edx, %ecx
 ; PIC-NEXT:    movl $7, %ebx
-; PIC-NEXT:  LBB2_3: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB2_1 Depth=1
-; PIC-NEXT:    testb %cl, %cl
-; PIC-NEXT:    movl %edx, %ecx
-; PIC-NEXT:    jne LBB2_5
-; PIC-NEXT:  ## BB#4: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB2_1 Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
-; PIC-NEXT:  LBB2_5: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB2_1 Depth=1
+; PIC-NEXT:    cmovbl %eax, %ebx
 ; PIC-NEXT:    lock cmpxchg8b (%esi)
 ; PIC-NEXT:    jne LBB2_1
-; PIC-NEXT:  ## BB#6: ## %atomicrmw.end
+; PIC-NEXT:  ## BB#2: ## %atomicrmw.end
 ; PIC-NEXT:    popl %esi
 ; PIC-NEXT:    popl %edi
 ; PIC-NEXT:    popl %ebx
 ; PIC-NEXT:    retl
 ; PIC-NEXT:    ## -- End function
 entry:
   %umax = atomicrmw umax i64* @sc64, i64 7 acquire
   ret i64 %umax
 }
 
 define i64 @atomic_umin_i64() nounwind {
 ; LINUX-LABEL: atomic_umin_i64:
 ; LINUX:       # BB#0: # %entry
 ; LINUX-NEXT:    pushl %ebx
 ; LINUX-NEXT:    movl sc64+4, %edx
 ; LINUX-NEXT:    movl sc64, %eax
 ; LINUX-NEXT:    .p2align 4, 0x90
 ; LINUX-NEXT:  .LBB3_1: # %atomicrmw.start
 ; LINUX-NEXT:    # =>This Inner Loop Header: Depth=1
 ; LINUX-NEXT:    cmpl $9, %eax
 ; LINUX-NEXT:    movl %edx, %ecx
 ; LINUX-NEXT:    sbbl $0, %ecx
-; LINUX-NEXT:    setb %cl
-; LINUX-NEXT:    andb $1, %cl
-; LINUX-NEXT:    movl %eax, %ebx
-; LINUX-NEXT:    jne .LBB3_3
-; LINUX-NEXT:  # BB#2: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; LINUX-NEXT:    movl $0, %ecx
+; LINUX-NEXT:    cmovbl %edx, %ecx
 ; LINUX-NEXT:    movl $8, %ebx
-; LINUX-NEXT:  .LBB3_3: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB3_1 Depth=1
-; LINUX-NEXT:    testb %cl, %cl
-; LINUX-NEXT:    movl %edx, %ecx
-; LINUX-NEXT:    jne .LBB3_5
-; LINUX-NEXT:  # BB#4: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB3_1 Depth=1
-; LINUX-NEXT:    xorl %ecx, %ecx
-; LINUX-NEXT:  .LBB3_5: # %atomicrmw.start
-; LINUX-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; LINUX-NEXT:    cmovbl %eax, %ebx
 ; LINUX-NEXT:    lock cmpxchg8b sc64
 ; LINUX-NEXT:    jne .LBB3_1
-; LINUX-NEXT:  # BB#6: # %atomicrmw.end
+; LINUX-NEXT:  # BB#2: # %atomicrmw.end
 ; LINUX-NEXT:    popl %ebx
 ; LINUX-NEXT:    retl
 ;
 ; PIC-LABEL: atomic_umin_i64:
 ; PIC:       ## BB#0: ## %entry
 ; PIC-NEXT:    pushl %ebx
 ; PIC-NEXT:    pushl %esi
 ; PIC-NEXT:    calll L3$pb
 ; PIC-NEXT:  L3$pb:
 ; PIC-NEXT:    popl %eax
 ; PIC-NEXT:    movl L_sc64$non_lazy_ptr-L3$pb(%eax), %esi
 ; PIC-NEXT:    movl (%esi), %eax
 ; PIC-NEXT:    movl 4(%esi), %edx
 ; PIC-NEXT:    .p2align 4, 0x90
 ; PIC-NEXT:  LBB3_1: ## %atomicrmw.start
 ; PIC-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; PIC-NEXT:    cmpl $9, %eax
 ; PIC-NEXT:    movl %edx, %ecx
 ; PIC-NEXT:    sbbl $0, %ecx
-; PIC-NEXT:    setb %cl
-; PIC-NEXT:    andb $1, %cl
-; PIC-NEXT:    movl %eax, %ebx
-; PIC-NEXT:    jne LBB3_3
-; PIC-NEXT:  ## BB#2: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB3_1 Depth=1
+; PIC-NEXT:    movl $0, %ecx
+; PIC-NEXT:    cmovbl %edx, %ecx
 ; PIC-NEXT:    movl $8, %ebx
-; PIC-NEXT:  LBB3_3: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB3_1 Depth=1
-; PIC-NEXT:    testb %cl, %cl
-; PIC-NEXT:    movl %edx, %ecx
-; PIC-NEXT:    jne LBB3_5
-; PIC-NEXT:  ## BB#4: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB3_1 Depth=1
-; PIC-NEXT:    xorl %ecx, %ecx
-; PIC-NEXT:  LBB3_5: ## %atomicrmw.start
-; PIC-NEXT:    ## in Loop: Header=BB3_1 Depth=1
+; PIC-NEXT:    cmovbl %eax, %ebx
 ; PIC-NEXT:    lock cmpxchg8b (%esi)
 ; PIC-NEXT:    jne LBB3_1
-; PIC-NEXT:  ## BB#6: ## %atomicrmw.end
+; PIC-NEXT:  ## BB#2: ## %atomicrmw.end
 ; PIC-NEXT:    popl %esi
 ; PIC-NEXT:    popl %ebx
 ; PIC-NEXT:    retl
 ; PIC-NEXT:    ## -- End function
 entry:
   %umin = atomicrmw umin i64* @sc64, i64 8 acquire
   ret i64 %umin
 }

test/CodeGen/X86/atomic128.ll

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB5_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setge %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB5_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB5_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB5_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB5_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
+; CHECK-NEXT:    cmovgeq %rdx, %rcx
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    cmovgeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB5_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw min i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB6_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rsi, %rax
 ; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    sbbq %r8, %rcx
-; CHECK-NEXT:    setge %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB6_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB6_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB6_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB6_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
+; CHECK-NEXT:    cmovgeq %rdx, %rcx
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    cmovgeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB6_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw max i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB7_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setae %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB7_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB7_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB7_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB7_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
+; CHECK-NEXT:    cmovaeq %rdx, %rcx
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    cmovaeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB7_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw umin i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB8_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setb %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB8_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB8_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB8_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB8_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
+; CHECK-NEXT:    cmovbq %rdx, %rcx
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    cmovbq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB8_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw umax i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

test/CodeGen/X86/cmov.ll

 ; CHECK-NEXT:    retq
   %d = select i1 %c, i8 %a, i8 %b
   ret i8 %d
 }
 
 define i32 @smin(i32 %x) {
 ; CHECK-LABEL: smin:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    xorl $-1, %edi
+; CHECK-NEXT:    notl %edi
+; CHECK-NEXT:    testl %edi, %edi
 ; CHECK-NEXT:    movl $-1, %eax
 ; CHECK-NEXT:    cmovsl %edi, %eax
 ; CHECK-NEXT:    retq
   %not_x = xor i32 %x, -1
   %1 = icmp slt i32 %not_x, -1
   %sel = select i1 %1, i32 %not_x, i32 -1
   ret i32 %sel
 }
 

test/CodeGen/X86/cmp.ll

 
 }
 
 ; This test failed due to incorrect handling of "shift + icmp" sequence
 define void @test20(i32 %bf.load, i8 %x1, i8* %b_addr) {
 ; CHECK-LABEL: test20:
 ; CHECK:       ## BB#0: ## %entry
 ; CHECK-NEXT:    xorl %eax, %eax ## encoding: [0x31,0xc0]
-; CHECK-NEXT:    andl $16777215, %edi ## encoding: [0x81,0xe7,0xff,0xff,0xff,0x00]
+; CHECK-NEXT:    testl $16777215, %edi ## encoding: [0xf7,0xc7,0xff,0xff,0xff,0x00]
 ; CHECK-NEXT:    ## imm = 0xFFFFFF
 ; CHECK-NEXT:    setne %al ## encoding: [0x0f,0x95,0xc0]
 ; CHECK-NEXT:    movzbl %sil, %ecx ## encoding: [0x40,0x0f,0xb6,0xce]
 ; CHECK-NEXT:    addl %eax, %ecx ## encoding: [0x01,0xc1]
 ; CHECK-NEXT:    setne (%rdx) ## encoding: [0x0f,0x95,0x02]
-; CHECK-NEXT:    testl %edi, %edi ## encoding: [0x85,0xff]
+; CHECK-NEXT:    testl $16777215, %edi ## encoding: [0xf7,0xc7,0xff,0xff,0xff,0x00]
+; CHECK-NEXT:    ## imm = 0xFFFFFF
 ; CHECK-NEXT:    setne {{.*}}(%rip) ## encoding: [0x0f,0x95,0x05,A,A,A,A]
 ; CHECK-NEXT:    ## fixup A - offset: 3, value: _d-4, kind: reloc_riprel_4byte
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %bf.shl = shl i32 %bf.load, 8
   %bf.ashr = ashr exact i32 %bf.shl, 8
   %tobool4 = icmp ne i32 %bf.ashr, 0
   %conv = zext i1 %tobool4 to i32

test/CodeGen/X86/select.ll

 ; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; CHECK-NEXT:    movd %xmm0, (%rsi)
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    ## -- End function
 ;
 ; MCU-LABEL: test5:
 ; MCU:       # BB#0:
 ; MCU-NEXT:    pushl %esi
-; MCU-NEXT:    andb $1, %al
+; MCU-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    testb $1, %al
 ; MCU-NEXT:    jne .LBB4_2
 ; MCU-NEXT:  # BB#1:
+; MCU-NEXT:    movw {{[0-9]+}}(%esp), %cx
 ; MCU-NEXT:    movw {{[0-9]+}}(%esp), %dx
 ; MCU-NEXT:  .LBB4_2:
-; MCU-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    testb %al, %al
-; MCU-NEXT:    jne .LBB4_4
-; MCU-NEXT:  # BB#3:
-; MCU-NEXT:    movw {{[0-9]+}}(%esp), %cx
-; MCU-NEXT:  .LBB4_4:
-; MCU-NEXT:    movw %dx, (%esi)
 ; MCU-NEXT:    movw %cx, 2(%esi)
+; MCU-NEXT:    movw %dx, (%esi)
 ; MCU-NEXT:    popl %esi
 ; MCU-NEXT:    retl
   %x = select i1 %c, <2 x i16> %a, <2 x i16> %b
   store <2 x i16> %x, <2 x i16>* %p
   ret void
 }
 
 ; Verify that the fmul gets sunk into the one part of the diamond where it is needed.
; MCU-NEXT:    retl
   %retval = select i1 %tmp9, x86_fp80 0xK4005B400000000000000, x86_fp80 0xK40078700000000000000
   ret x86_fp80 %retval
 }
 
 ; widening select v6i32 and then a sub
 define void @test8(i1 %c, <6 x i32>* %dst.addr, <6 x i32> %src1,<6 x i32> %src2) nounwind {
 ; GENERIC-LABEL: test8:
 ; GENERIC:       ## BB#0:
-; GENERIC-NEXT:    andb $1, %dil
+; GENERIC-NEXT:    testb $1, %dil
 ; GENERIC-NEXT:    jne LBB7_1
 ; GENERIC-NEXT:  ## BB#2:
-; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    jmp LBB7_3
-; GENERIC-NEXT:  LBB7_1:
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; GENERIC-NEXT:  LBB7_3:
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; GENERIC-NEXT:    testb %dil, %dil
-; GENERIC-NEXT:    jne LBB7_4
-; GENERIC-NEXT:  ## BB#5:
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; GENERIC-NEXT:    jmp LBB7_3
+; GENERIC-NEXT:  LBB7_1:
+; GENERIC-NEXT:    movd %r9d, %xmm0
+; GENERIC-NEXT:    movd %r8d, %xmm1
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; GENERIC-NEXT:    movd %ecx, %xmm2
+; GENERIC-NEXT:    movd %edx, %xmm0
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; GENERIC-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    jmp LBB7_6
-; GENERIC-NEXT:  LBB7_4:
-; GENERIC-NEXT:    movd %r9d, %xmm1
-; GENERIC-NEXT:    movd %r8d, %xmm2
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; GENERIC-NEXT:    movd %ecx, %xmm3
-; GENERIC-NEXT:    movd %edx, %xmm1
-; GENERIC-NEXT:  LBB7_6:
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; GENERIC-NEXT:  LBB7_3:
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; GENERIC-NEXT:    pcmpeqd %xmm2, %xmm2
-; GENERIC-NEXT:    paddd %xmm2, %xmm1
 ; GENERIC-NEXT:    paddd %xmm2, %xmm0
-; GENERIC-NEXT:    movq %xmm0, 16(%rsi)
-; GENERIC-NEXT:    movdqa %xmm1, (%rsi)
+; GENERIC-NEXT:    paddd %xmm2, %xmm1
+; GENERIC-NEXT:    movq %xmm1, 16(%rsi)
+; GENERIC-NEXT:    movdqa %xmm0, (%rsi)
 ; GENERIC-NEXT:    retq
 ; GENERIC-NEXT:    ## -- End function
 ;
 ; ATOM-LABEL: test8:
 ; ATOM:       ## BB#0:
-; ATOM-NEXT:    andb $1, %dil
+; ATOM-NEXT:    testb $1, %dil
 ; ATOM-NEXT:    jne LBB7_1
 ; ATOM-NEXT:  ## BB#2:
-; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ATOM-NEXT:    jmp LBB7_3
-; ATOM-NEXT:  LBB7_1:
-; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ATOM-NEXT:  LBB7_3:
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; ATOM-NEXT:    testb %dil, %dil
-; ATOM-NEXT:    jne LBB7_4
-; ATOM-NEXT:  ## BB#5:
 ; ATOM-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; ATOM-NEXT:    jmp LBB7_6
-; ATOM-NEXT:  LBB7_4:
-; ATOM-NEXT:    movd %r9d, %xmm1
+; ATOM-NEXT:    jmp LBB7_3
+; ATOM-NEXT:  LBB7_1:
+; ATOM-NEXT:    movd %r9d, %xmm0
 ; ATOM-NEXT:    movd %r8d, %xmm2
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; ATOM-NEXT:    movd %ecx, %xmm3
-; ATOM-NEXT:    movd %edx, %xmm1
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; ATOM-NEXT:  LBB7_6:
+; ATOM-NEXT:    movd %edx, %xmm0
+; ATOM-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; ATOM-NEXT:  LBB7_3:
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; ATOM-NEXT:    pcmpeqd %xmm2, %xmm2
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; ATOM-NEXT:    paddd %xmm2, %xmm0
 ; ATOM-NEXT:    paddd %xmm2, %xmm1
-; ATOM-NEXT:    movq %xmm0, 16(%rsi)
-; ATOM-NEXT:    movdqa %xmm1, (%rsi)
+; ATOM-NEXT:    movq %xmm1, 16(%rsi)
+; ATOM-NEXT:    movdqa %xmm0, (%rsi)
 ; ATOM-NEXT:    retq
 ; ATOM-NEXT:    ## -- End function
 ;
 ; MCU-LABEL: test8:
 ; MCU:       # BB#0:
 ; MCU-NEXT:    pushl %ebp
 ; MCU-NEXT:    pushl %ebx
 ; MCU-NEXT:    pushl %edi
 ; MCU-NEXT:    pushl %esi
-; MCU-NEXT:    andb $1, %al
+; MCU-NEXT:    testb $1, %al
 ; MCU-NEXT:    jne .LBB7_1
 ; MCU-NEXT:  # BB#2:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; MCU-NEXT:    movl (%ecx), %ecx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:    movl (%eax), %eax
 ; MCU-NEXT:    je .LBB7_5
 ; MCU-NEXT:  .LBB7_4:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    movl (%esi), %esi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; MCU-NEXT:    movl (%ecx), %ecx
 ; MCU-NEXT:    je .LBB7_8
 ; MCU-NEXT:  .LBB7_7:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
-; MCU-NEXT:    movl (%edi), %edi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    movl (%esi), %esi
 ; MCU-NEXT:    je .LBB7_11
 ; MCU-NEXT:  .LBB7_10:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
-; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
+; MCU-NEXT:    movl (%edi), %edi
 ; MCU-NEXT:    je .LBB7_14
 ; MCU-NEXT:  .LBB7_13:
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
+; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    je .LBB7_17
+; MCU-NEXT:  .LBB7_16:
 ; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
-; MCU-NEXT:    jmp .LBB7_15
+; MCU-NEXT:    jmp .LBB7_18
 ; MCU-NEXT:  .LBB7_1:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; MCU-NEXT:    movl (%ecx), %ecx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:    movl (%eax), %eax
 ; MCU-NEXT:    jne .LBB7_4
 ; MCU-NEXT:  .LBB7_5:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    movl (%esi), %esi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; MCU-NEXT:    movl (%ecx), %ecx
 ; MCU-NEXT:    jne .LBB7_7
 ; MCU-NEXT:  .LBB7_8:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
-; MCU-NEXT:    movl (%edi), %edi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    movl (%esi), %esi
 ; MCU-NEXT:    jne .LBB7_10
 ; MCU-NEXT:  .LBB7_11:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
-; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
+; MCU-NEXT:    movl (%edi), %edi
 ; MCU-NEXT:    jne .LBB7_13
 ; MCU-NEXT:  .LBB7_14:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
-; MCU-NEXT:  .LBB7_15:
-; MCU-NEXT:    movl (%ebp), %ebp
-; MCU-NEXT:    testb %al, %al
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
+; MCU-NEXT:    movl (%ebx), %ebx
 ; MCU-NEXT:    jne .LBB7_16
-; MCU-NEXT:  # BB#17:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; MCU-NEXT:    jmp .LBB7_18
-; MCU-NEXT:  .LBB7_16:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:  .LBB7_17:
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
 ; MCU-NEXT:  .LBB7_18:
-; MCU-NEXT:    movl (%eax), %eax
-; MCU-NEXT:    decl %eax
+; MCU-NEXT:    movl (%ebp), %ebp
 ; MCU-NEXT:    decl %ebp
 ; MCU-NEXT:    decl %ebx
 ; MCU-NEXT:    decl %edi
 ; MCU-NEXT:    decl %esi
 ; MCU-NEXT:    decl %ecx
-; MCU-NEXT:    movl %ecx, 20(%edx)
-; MCU-NEXT:    movl %esi, 16(%edx)
-; MCU-NEXT:    movl %edi, 12(%edx)
-; MCU-NEXT:    movl %ebx, 8(%edx)
-; MCU-NEXT:    movl %ebp, 4(%edx)
-; MCU-NEXT:    movl %eax, (%edx)
+; MCU-NEXT:    decl %eax
+; MCU-NEXT:    movl %eax, 20(%edx)
+; MCU-NEXT:    movl %ecx, 16(%edx)
+; MCU-NEXT:    movl %esi, 12(%edx)
+; MCU-NEXT:    movl %edi, 8(%edx)
+; MCU-NEXT:    movl %ebx, 4(%edx)
+; MCU-NEXT:    movl %ebp, (%edx)
 ; MCU-NEXT:    popl %esi
 ; MCU-NEXT:    popl %edi
 ; MCU-NEXT:    popl %ebx
 ; MCU-NEXT:    popl %ebp
 ; MCU-NEXT:    retl
   %x = select i1 %c, <6 x i32> %src1, <6 x i32> %src2
   %val = sub <6 x i32> %x, < i32 1, i32 1, i32 1, i32 1, i32 1, i32 1 >
   store <6 x i32> %val, <6 x i32>* %dst.addr

test/CodeGen/X86/tbm_patterns.ll

; CHECK-NEXT:    retq
   %t1 = lshr i32 %t0, 4
   %t2 = and i32 %t1, 4095
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_bextri_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_bextri_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    shrl $4, %edi
-; CHECK-NEXT:    andl $4095, %edi # imm = 0xFFF
-; CHECK-NEXT:    cmovel %esi, %edi
-; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    bextr $3076, %edi, %eax # imm = 0xC04
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = lshr i32 %a, 4
   %t1 = and i32 %t0, 4095
   %t2 = icmp eq i32 %t1, 0
   %t3 = select i1 %t2, i32 %b, i32 %t1
   ret i32 %t3
 }
 
; CHECK-NEXT:    retq
   %t1 = lshr i64 %t0, 4
   %t2 = and i64 %t1, 4095
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_bextri_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_bextri_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    shrl $4, %edi
-; CHECK-NEXT:    andl $4095, %edi # imm = 0xFFF
-; CHECK-NEXT:    cmoveq %rsi, %rdi
-; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    bextr $3076, %edi, %eax # imm = 0xC04
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovneq %rax, %rsi
+; CHECK-NEXT:    movq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = lshr i64 %a, 4
   %t1 = and i64 %t0, 4095
   %t2 = icmp eq i64 %t1, 0
   %t3 = select i1 %t2, i64 %b, i64 %t1
   ret i64 %t3
 }
 
 define i32 @test_x86_tbm_blcfill_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcfill %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = and i32 %t0, %a
   ret i32 %t1
 }
 
 define i32 @test_x86_tbm_blcfill_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal 1(%rdi), %eax
-; CHECK-NEXT:    andl %edi, %eax
+; CHECK-NEXT:    blcfill %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = and i32 %t0, %a
   %t2 = icmp eq i32 %t1, 0
   %t3 = select i1 %t2, i32 %b, i32 %t1
   ret i32 %t3
 }
 
 define i64 @test_x86_tbm_blcfill_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcfill %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = and i64 %t0, %a
   ret i64 %t1
 }
 
 define i64 @test_x86_tbm_blcfill_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq 1(%rdi), %rax
-; CHECK-NEXT:    andq %rdi, %rax
+; CHECK-NEXT:    blcfill %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = and i64 %t0, %a
   %t2 = icmp eq i64 %t1, 0
   %t3 = select i1 %t2, i64 %b, i64 %t1
   ret i64 %t3
 }
 
 define i32 @test_x86_tbm_blci_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blci %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 1, %a
   %t1 = xor i32 %t0, -1
   %t2 = or i32 %t1, %a
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_blci_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal 1(%rdi), %eax
-; CHECK-NEXT:    notl %eax
-; CHECK-NEXT:    orl %edi, %eax
+; CHECK-NEXT:    blci %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 1, %a
   %t1 = xor i32 %t0, -1
   %t2 = or i32 %t1, %a
   %t3 = icmp eq i32 %t2, 0
   %t4 = select i1 %t3, i32 %b, i32 %t2
   ret i32 %t4
 }
 
 define i64 @test_x86_tbm_blci_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blci %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 1, %a
   %t1 = xor i64 %t0, -1
   %t2 = or i64 %t1, %a
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_blci_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq 1(%rdi), %rax
-; CHECK-NEXT:    notq %rax
-; CHECK-NEXT:    orq %rdi, %rax
+; CHECK-NEXT:    blci %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 1, %a
   %t1 = xor i64 %t0, -1
   %t2 = or i64 %t1, %a
   %t3 = icmp eq i64 %t2, 0
   %t4 = select i1 %t3, i64 %b, i64 %t2
   ret i64 %t4
; CHECK-NEXT:    retq
   %t1 = add i32 %a, 1
   %t2 = and i32 %t1, %t0
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_blcic_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcic_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal 1(%rdi), %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    andl %ecx, %eax
+; CHECK-NEXT:    blcic %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, 1
   %t2 = and i32 %t1, %t0
   %t3 = icmp eq i32 %t2, 0
   %t4 = select i1 %t3, i32 %b, i32 %t2
   ret i32 %t4
 }
 
 define i64 @test_x86_tbm_blcic_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcic_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcic %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, 1
   %t2 = and i64 %t1, %t0
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_blcic_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcic_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq 1(%rdi), %rax
-; CHECK-NEXT:    notq %rdi
-; CHECK-NEXT:    andq %rdi, %rax
+; CHECK-NEXT:    blcic %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, 1
   %t2 = and i64 %t1, %t0
   %t3 = icmp eq i64 %t2, 0
   %t4 = select i1 %t3, i64 %b, i64 %t2
   ret i64 %t4
 }
 
 define i32 @test_x86_tbm_blcmsk_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcmsk %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = xor i32 %t0, %a
   ret i32 %t1
 }
 
 define i32 @test_x86_tbm_blcmsk_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal 1(%rdi), %eax
-; CHECK-NEXT:    xorl %edi, %eax
+; CHECK-NEXT:    blcmsk %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = xor i32 %t0, %a
   %t2 = icmp eq i32 %t1, 0
   %t3 = select i1 %t2, i32 %b, i32 %t1
   ret i32 %t3
 }
 
 define i64 @test_x86_tbm_blcmsk_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcmsk %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = xor i64 %t0, %a
   ret i64 %t1
 }
 
 define i64 @test_x86_tbm_blcmsk_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq 1(%rdi), %rax
-; CHECK-NEXT:    xorq %rdi, %rax
+; CHECK-NEXT:    blcmsk %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = xor i64 %t0, %a
   %t2 = icmp eq i64 %t1, 0
   %t3 = select i1 %t2, i64 %b, i64 %t1
   ret i64 %t3
 }
 
 define i32 @test_x86_tbm_blcs_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcs %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = or i32 %t0, %a
   ret i32 %t1
 }
 
 define i32 @test_x86_tbm_blcs_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal 1(%rdi), %eax
-; CHECK-NEXT:    orl %edi, %eax
+; CHECK-NEXT:    blcs %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
   %t1 = or i32 %t0, %a
   %t2 = icmp eq i32 %t1, 0
   %t3 = select i1 %t2, i32 %b, i32 %t1
   ret i32 %t3
 }
 
 define i64 @test_x86_tbm_blcs_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blcs %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = or i64 %t0, %a
   ret i64 %t1
 }
 
 define i64 @test_x86_tbm_blcs_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq 1(%rdi), %rax
-; CHECK-NEXT:    orq %rdi, %rax
+; CHECK-NEXT:    blcs %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
   %t1 = or i64 %t0, %a
   %t2 = icmp eq i64 %t1, 0
   %t3 = select i1 %t2, i64 %b, i64 %t1
   ret i64 %t3
 }
 
 define i32 @test_x86_tbm_blsfill_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blsfill %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, -1
   %t1 = or i32 %t0, %a
   ret i32 %t1
 }
 
 define i32 @test_x86_tbm_blsfill_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    # kill: %EDI<def> %EDI<kill> %RDI<def>
-; CHECK-NEXT:    leal -1(%rdi), %eax
-; CHECK-NEXT:    orl %edi, %eax
+; CHECK-NEXT:    blsfill %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
 ; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, -1
   %t1 = or i32 %t0, %a
   %t2 = icmp eq i32 %t1, 0
   %t3 = select i1 %t2, i32 %b, i32 %t1
   ret i32 %t3
 }
 
 define i64 @test_x86_tbm_blsfill_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blsfill %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, -1
   %t1 = or i64 %t0, %a
   ret i64 %t1
 }
 
 define i64 @test_x86_tbm_blsfill_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    leaq -1(%rdi), %rax
-; CHECK-NEXT:    orq %rdi, %rax
+; CHECK-NEXT:    blsfill %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
 ; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, -1
   %t1 = or i64 %t0, %a
   %t2 = icmp eq i64 %t1, 0
   %t3 = select i1 %t2, i64 %b, i64 %t1
   ret i64 %t3
 }
 
 define i32 @test_x86_tbm_blsic_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blsic %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, -1
   %t2 = or i32 %t0, %t1
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_blsic_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    notl %eax
-; CHECK-NEXT:    decl %edi
-; CHECK-NEXT:    orl %eax, %edi
-; CHECK-NEXT:    cmovel %esi, %edi
-; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    blsic %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, -1
   %t2 = or i32 %t0, %t1
   %t3 = icmp eq i32 %t2, 0
   %t4 = select i1 %t3, i32 %b, i32 %t2
   ret i32 %t4
 }
 
 define i64 @test_x86_tbm_blsic_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    blsic %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, -1
   %t2 = or i64 %t0, %t1
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_blsic_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    notq %rax
-; CHECK-NEXT:    decq %rdi
-; CHECK-NEXT:    orq %rax, %rdi
-; CHECK-NEXT:    cmoveq %rsi, %rdi
-; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    blsic %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, -1
   %t2 = or i64 %t0, %t1
   %t3 = icmp eq i64 %t2, 0
   %t4 = select i1 %t3, i64 %b, i64 %t2
   ret i64 %t4
 }
 
 define i32 @test_x86_tbm_t1mskc_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    t1mskc %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, 1
   %t2 = or i32 %t0, %t1
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_t1mskc_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    notl %eax
-; CHECK-NEXT:    incl %edi
-; CHECK-NEXT:    orl %eax, %edi
-; CHECK-NEXT:    cmovel %esi, %edi
-; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    t1mskc %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, 1
   %t2 = or i32 %t0, %t1
   %t3 = icmp eq i32 %t2, 0
   %t4 = select i1 %t3, i32 %b, i32 %t2
   ret i32 %t4
 }
 
 define i64 @test_x86_tbm_t1mskc_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    t1mskc %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, 1
   %t2 = or i64 %t0, %t1
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_t1mskc_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    notq %rax
-; CHECK-NEXT:    incq %rdi
-; CHECK-NEXT:    orq %rax, %rdi
-; CHECK-NEXT:    cmoveq %rsi, %rdi
-; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    t1mskc %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, 1
   %t2 = or i64 %t0, %t1
   %t3 = icmp eq i64 %t2, 0
   %t4 = select i1 %t3, i64 %b, i64 %t2
   ret i64 %t4
 }
 
 define i32 @test_x86_tbm_tzmsk_u32(i32 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u32:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    tzmsk %edi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, -1
   %t2 = and i32 %t0, %t1
   ret i32 %t2
 }
 
 define i32 @test_x86_tbm_tzmsk_u32_z(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u32_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    notl %eax
-; CHECK-NEXT:    decl %edi
-; CHECK-NEXT:    andl %eax, %edi
-; CHECK-NEXT:    cmovel %esi, %edi
-; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    tzmsk %edi, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    cmovel %esi, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
   %t1 = add i32 %a, -1
   %t2 = and i32 %t0, %t1
   %t3 = icmp eq i32 %t2, 0
   %t4 = select i1 %t3, i32 %b, i32 %t2
   ret i32 %t4
 }
 
 define i64 @test_x86_tbm_tzmsk_u64(i64 %a) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u64:
 ; CHECK:       # BB#0:
 ; CHECK-NEXT:    tzmsk %rdi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, -1
   %t2 = and i64 %t0, %t1
   ret i64 %t2
 }
 
 define i64 @test_x86_tbm_tzmsk_u64_z(i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u64_z:
 ; CHECK:       # BB#0:
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    notq %rax
-; CHECK-NEXT:    decq %rdi
-; CHECK-NEXT:    andq %rax, %rdi
-; CHECK-NEXT:    cmoveq %rsi, %rdi
-; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    tzmsk %rdi, %rax
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    cmoveq %rsi, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
   %t1 = add i64 %a, -1
   %t2 = and i64 %t0, %t1
   %t3 = icmp eq i64 %t2, 0
   %t4 = select i1 %t3, i64 %b, i64 %t2
   ret i64 %t4
 }