[X86] Check for already emitted TEST pattern in EmitTest function.

[aivchenk]

Summary

This will generate this:

       testb   $1, %dil
       movl    $42, %eax
       movl    $31415, %ecx
       cmovnel %eax, %ecx
       movl    $4048, %edx
       movl    $666, %eax
       cmovnel %edx, %eax
       addl    %ecx, %eax
       retq

instead of this:

       andb    $1, %dil
       movl    $4048, %eax
       movl    $666, %ecx
       cmovnel %eax, %ecx
       testb   %dil, %dil
       movl    $42, %edx
       movl    $31415, %eax
       cmovnel %edx, %eax
       addl    %ecx, %eax
       retq

for this example:

define i32 @test(i1 %c) {
entry:
  %0 = select i1 %c, i32 42, i32 31415
  %1 = select i1 %c, i32 4048, i32 666
  %ret = add i32 %0, %1
  ret i32 %ret
}

[aivchenk]

Kind ping :)

[craig.topper]

It looks like Chandler's patch D37096 might also be fixing some of this and that might be a more consistent fix. I'd like to hold off on this until we disposition that patch.

[RKSimon]

In D36612#851598, @craig.topper wrote:

It looks like Chandler's patch D37096 might also be fixing some of this and that might be a more consistent fix. I'd like to hold off on this until we disposition that patch.

What's happening with this? Chandler's patch landed at rL311737.

[aivchenk]

Chandler's patch fixed the issue in more general way.

lib/Target/X86/X86ISelLowering.cpp

if (Op.getOpcode() == ISD::BITCAST) {
     MVT Op0VT = Op0.getValueType().getSimpleVT();
     if (Op0VT.isVector() && Op0VT.getVectorElementType() == MVT::i1 &&
         hasKTEST(Op0VT))
       return DAG.getNode(X86ISD::KTEST, SDLoc(Op), Op0VT, Op0, Op0);
   }
   return SDValue();
 }
 
+// Look up CMP with 0 (which is the TEST pattern) among uses of OP.
+static SDValue lookupCmpWithZeroUse(SDValue Op) {
+  for (SDNode *U : Op->uses()) {
+    if (U->getOpcode() == X86ISD::CMP) {
+      auto *CN = dyn_cast<ConstantSDNode>(U->getOperand(1));
+      if (CN->getAPIntValue() == 0)
+        return SDValue(U, 0);
+    }
+  }
+  return SDValue();
+}
+
 /// Emit nodes that will be selected as "test Op0,Op0", or something
 /// equivalent.
 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
                                     SelectionDAG &DAG) const {
   if (Op.getValueType() == MVT::i1) {
     SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op);
     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp,
                        DAG.getConstant(0, dl, MVT::i8));
if (!hasNonFlagsUse(Op)) {
           MVT ShTy = getScalarShiftAmountTy(DAG.getDataLayout(), VT);
           SDValue ShAmt = DAG.getConstant(LeadingZeros, dl, ShTy);
           Op = DAG.getNode(ISD::SHL, dl, VT, Op0, ShAmt);
           break;
         }
 
         break;
       }
+    } else if (SDValue V = lookupCmpWithZeroUse(Op)) {
+      // If we found that AND instruction already has a use in form of
+      // CMP with 0 (TEST pattern) - use this as the condition. The
+      // common example of that is when we lower two selects that
+      // use the same AND as the condition:
+      //    %c = and i8 %cond, 1
+      //    %0 = select i1 %c, i32 a, i32 b
+      //    %1 = select i1 %c, i32 c, i32 d
+      // - When we lower first select, we do emit TEST pattern, which we
+      // will use for the second select here.
+      return V;
     }
     LLVM_FALLTHROUGH;
   case ISD::SUB:
   case ISD::OR:
   case ISD::XOR:
     // Due to the ISEL shortcoming noted above, be conservative if this op is
     // likely to be selected as part of a load-modify-store instruction.
     for (SDNode::use_iterator UI = Op.getNode()->use_begin(),

test/CodeGen/X86/atomic-minmax-i6432.ll

 ; RUN: llc -mattr=+cmov,cx16 -mtriple=i386-pc-linux -verify-machineinstrs < %s | FileCheck %s -check-prefix=LINUX
 ; RUN: llc -mattr=cx16 -mtriple=i386-macosx -relocation-model=pic -verify-machineinstrs < %s | FileCheck %s -check-prefix=PIC
 
 @sc64 = external global i64
 
 define void @atomic_maxmin_i6432() {
 ; LINUX: atomic_maxmin_i6432
   %1 = atomicrmw max  i64* @sc64, i64 5 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
 ; LINUX: cmpl
 ; LINUX: sbbl
-; LINUX: jne
-; LINUX: jne
+; LINUX: cmovll
+; LINUX: cmovll
 ; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %2 = atomicrmw min  i64* @sc64, i64 6 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
 ; LINUX: cmpl
 ; LINUX: sbbl
-; LINUX: jne
-; LINUX: jne
+; LINUX: cmovll
+; LINUX: cmovll
 ; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %3 = atomicrmw umax i64* @sc64, i64 7 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
 ; LINUX: cmpl
 ; LINUX: sbbl
-; LINUX: jne
-; LINUX: jne
+; LINUX: cmovbl
+; LINUX: cmovbl
 ; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   %4 = atomicrmw umin i64* @sc64, i64 8 acquire
 ; LINUX: [[LABEL:.LBB[0-9]+_[0-9]+]]
 ; LINUX: cmpl
 ; LINUX: sbbl
-; LINUX: jne
-; LINUX: jne
+; LINUX: cmovbl
+; LINUX: cmovbl
 ; LINUX: lock cmpxchg8b
 ; LINUX: jne [[LABEL]]
   ret void
 }
 
 ; rdar://12453106
 @id = internal global i64 0, align 8
 

test/CodeGen/X86/atomic128.ll

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB5_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setge %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB5_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB5_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB5_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
-; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB5_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB5_1 Depth=1
+; CHECK-NEXT:    movq    %r8, %rcx
+; CHECK-NEXT:    cmovgeq %rdx, %rcx
+; CHECK-NEXT:    movq    %rsi, %rbx
+; CHECK-NEXT:    cmovgeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB5_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw min i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB6_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rsi, %rax
 ; CHECK-NEXT:    movq %rdx, %rcx
 ; CHECK-NEXT:    sbbq %r8, %rcx
-; CHECK-NEXT:    setge %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB6_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB6_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB6_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB6_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB6_1 Depth=1
+; CHECK-NEXT:    cmovgeq %rdx, %rcx
+; CHECK-NEXT:    movq    %rsi, %rbx
+; CHECK-NEXT:    cmovgeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB6_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw max i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB7_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setae %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB7_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB7_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB7_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB7_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB7_1 Depth=1
+; CHECK-NEXT:    cmovaeq %rdx, %rcx
+; CHECK-NEXT:    movq    %rsi, %rbx
+; CHECK-NEXT:    cmovaeq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB7_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw umin i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

 ; CHECK-NEXT:    movq (%rdi), %rax
 ; CHECK-NEXT:    movq 8(%rdi), %rdx
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  LBB8_1: ## %atomicrmw.start
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    movq %r8, %rcx
 ; CHECK-NEXT:    sbbq %rdx, %rcx
-; CHECK-NEXT:    setb %cl
-; CHECK-NEXT:    andb $1, %cl
-; CHECK-NEXT:    movq %rax, %rbx
-; CHECK-NEXT:    jne LBB8_3
-; CHECK-NEXT:  ## BB#2: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
-; CHECK-NEXT:    movq %rsi, %rbx
-; CHECK-NEXT:  LBB8_3: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
-; CHECK-NEXT:    testb %cl, %cl
-; CHECK-NEXT:    movq %rdx, %rcx
-; CHECK-NEXT:    jne LBB8_5
-; CHECK-NEXT:  ## BB#4: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
 ; CHECK-NEXT:    movq %r8, %rcx
-; CHECK-NEXT:  LBB8_5: ## %atomicrmw.start
-; CHECK-NEXT:    ## in Loop: Header=BB8_1 Depth=1
+; CHECK-NEXT:    cmovbq %rdx, %rcx
+; CHECK-NEXT:    movq %rsi, %rbx
+; CHECK-NEXT:    cmovbq %rax, %rbx
 ; CHECK-NEXT:    lock cmpxchg16b (%rdi)
 ; CHECK-NEXT:    jne LBB8_1
-; CHECK-NEXT:  ## BB#6: ## %atomicrmw.end
+; CHECK-NEXT:  ## BB#2: ## %atomicrmw.end
 ; CHECK-NEXT:    movq %rax, {{.*}}(%rip)
 ; CHECK-NEXT:    movq %rdx, _var+{{.*}}(%rip)
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    retq
   %val = atomicrmw umax i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }

test/CodeGen/X86/cmov.ll

 ; CHECK-NEXT:    cmovsl %edi, %eax
 ; CHECK-NEXT:    retq
   %not_x = xor i32 %x, -1
   %1 = icmp slt i32 %not_x, -1
   %sel = select i1 %1, i32 %not_x, i32 -1
   ret i32 %sel
 }
 
+; CHECK-LABEL: test8:
+; CHECK:       # BB#0:
+; CHECK-NEXT:  testb   $1, %dil
+; CHECK-NEXT:  movl    $42, %eax
+; CHECK-NEXT:  movl    $31415, %ecx
+; CHECK-NEXT:  cmovnel %eax, %ecx
+; CHECK-NEXT:  movl    $4048, %edx
+; CHECK-NEXT:  movl    $666, %eax
+; CHECK-NEXT:  cmovnel %edx, %eax
+; CHECK-NEXT:  addl    %ecx, %eax
+; CHECK-NEXT:  retq
+
+define i32 @test8(i1 %c) {
+entry:
+  %0 = select i1 %c, i32 42, i32 31415
+  %1 = select i1 %c, i32 4048, i32 666
+  %ret = add i32 %0, %1
+  ret i32 %ret
+}

test/CodeGen/X86/cmp.ll

define void @test20(i32 %bf.load, i8 %x1, i8* %b_addr) {
   %frombool = zext i1 %tobool7 to i8
   store i8 %frombool, i8* %b_addr, align 1
   %tobool14 = icmp ne i32 %bf.shl, 0
   %frombool15 = zext i1 %tobool14 to i8
   store i8 %frombool15, i8* @d, align 1
   ret void
 
 ; CHECK-LABEL: test20
-; CHECK: andl
+; CHECK: testl
 ; CHECK: setne
 ; CHECK: addl
 ; CHECK: setne
 ; CHECK: testl
 ; CHECK: setne
 }
 
 define i32 @test21(i64 %val) {

test/CodeGen/X86/select.ll

 ; CHECK-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
 ; CHECK-NEXT:    movd %xmm0, (%rsi)
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:    ## -- End function
 ;
 ; MCU-LABEL: test5:
 ; MCU:       # BB#0:
 ; MCU-NEXT:    pushl %esi
-; MCU-NEXT:    andb $1, %al
+; MCU-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    testb $1, %al
 ; MCU-NEXT:    jne .LBB4_2
 ; MCU-NEXT:  # BB#1:
+; MCU-NEXT:    movw {{[0-9]+}}(%esp), %cx
 ; MCU-NEXT:    movw {{[0-9]+}}(%esp), %dx
 ; MCU-NEXT:  .LBB4_2:
-; MCU-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    testb %al, %al
-; MCU-NEXT:    jne .LBB4_4
-; MCU-NEXT:  # BB#3:
-; MCU-NEXT:    movw {{[0-9]+}}(%esp), %cx
-; MCU-NEXT:  .LBB4_4:
-; MCU-NEXT:    movw %dx, (%esi)
 ; MCU-NEXT:    movw %cx, 2(%esi)
+; MCU-NEXT:    movw %dx, (%esi)
 ; MCU-NEXT:    popl %esi
 ; MCU-NEXT:    retl
   %x = select i1 %c, <2 x i16> %a, <2 x i16> %b
   store <2 x i16> %x, <2 x i16>* %p
   ret void
 }
 
 ; Verify that the fmul gets sunk into the one part of the diamond where it is needed.
; MCU-NEXT:    retl
   %retval = select i1 %tmp9, x86_fp80 0xK4005B400000000000000, x86_fp80 0xK40078700000000000000
   ret x86_fp80 %retval
 }
 
 ; widening select v6i32 and then a sub
 define void @test8(i1 %c, <6 x i32>* %dst.addr, <6 x i32> %src1,<6 x i32> %src2) nounwind {
 ; GENERIC-LABEL: test8:
 ; GENERIC:       ## BB#0:
-; GENERIC-NEXT:    andb $1, %dil
+; GENERIC-NEXT:    testb $1, %dil
 ; GENERIC-NEXT:    jne LBB7_1
 ; GENERIC-NEXT:  ## BB#2:
-; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    jmp LBB7_3
-; GENERIC-NEXT:  LBB7_1:
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; GENERIC-NEXT:  LBB7_3:
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; GENERIC-NEXT:    testb %dil, %dil
-; GENERIC-NEXT:    jne LBB7_4
-; GENERIC-NEXT:  ## BB#5:
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; GENERIC-NEXT:    jmp LBB7_3
+; GENERIC-NEXT:  LBB7_1:
+; GENERIC-NEXT:    movd %r9d, %xmm0
+; GENERIC-NEXT:    movd %r8d, %xmm1
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; GENERIC-NEXT:    movd %ecx, %xmm2
+; GENERIC-NEXT:    movd %edx, %xmm0
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; GENERIC-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; GENERIC-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; GENERIC-NEXT:    jmp LBB7_6
-; GENERIC-NEXT:  LBB7_4:
-; GENERIC-NEXT:    movd %r9d, %xmm1
-; GENERIC-NEXT:    movd %r8d, %xmm2
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; GENERIC-NEXT:    movd %ecx, %xmm3
-; GENERIC-NEXT:    movd %edx, %xmm1
-; GENERIC-NEXT:  LBB7_6:
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; GENERIC-NEXT:  LBB7_3:
+; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; GENERIC-NEXT:    pcmpeqd %xmm2, %xmm2
-; GENERIC-NEXT:    paddd %xmm2, %xmm1
 ; GENERIC-NEXT:    paddd %xmm2, %xmm0
-; GENERIC-NEXT:    movq %xmm0, 16(%rsi)
-; GENERIC-NEXT:    movdqa %xmm1, (%rsi)
+; GENERIC-NEXT:    paddd %xmm2, %xmm1
+; GENERIC-NEXT:    movq %xmm1, 16(%rsi)
+; GENERIC-NEXT:    movdqa %xmm0, (%rsi)
 ; GENERIC-NEXT:    retq
 ; GENERIC-NEXT:    ## -- End function
 ;
 ; ATOM-LABEL: test8:
 ; ATOM:       ## BB#0:
-; ATOM-NEXT:    andb $1, %dil
+; ATOM-NEXT:    testb $1, %dil
 ; ATOM-NEXT:    jne LBB7_1
 ; ATOM-NEXT:  ## BB#2:
-; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ATOM-NEXT:    jmp LBB7_3
-; ATOM-NEXT:  LBB7_1:
-; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ATOM-NEXT:  LBB7_3:
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; ATOM-NEXT:    testb %dil, %dil
-; ATOM-NEXT:    jne LBB7_4
-; ATOM-NEXT:  ## BB#5:
 ; ATOM-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; ATOM-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; ATOM-NEXT:    jmp LBB7_6
-; ATOM-NEXT:  LBB7_4:
-; ATOM-NEXT:    movd %r9d, %xmm1
+; ATOM-NEXT:    jmp LBB7_3
+; ATOM-NEXT:  LBB7_1:
+; ATOM-NEXT:    movd %r9d, %xmm0
 ; ATOM-NEXT:    movd %r8d, %xmm2
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
 ; ATOM-NEXT:    movd %ecx, %xmm3
-; ATOM-NEXT:    movd %edx, %xmm1
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; ATOM-NEXT:  LBB7_6:
+; ATOM-NEXT:    movd %edx, %xmm0
+; ATOM-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; ATOM-NEXT:  LBB7_3:
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
 ; ATOM-NEXT:    pcmpeqd %xmm2, %xmm2
+; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; ATOM-NEXT:    paddd %xmm2, %xmm0
 ; ATOM-NEXT:    paddd %xmm2, %xmm1
-; ATOM-NEXT:    movq %xmm0, 16(%rsi)
-; ATOM-NEXT:    movdqa %xmm1, (%rsi)
+; ATOM-NEXT:    movq %xmm1, 16(%rsi)
+; ATOM-NEXT:    movdqa %xmm0, (%rsi)
 ; ATOM-NEXT:    retq
 ; ATOM-NEXT:    ## -- End function
 ;
 ; MCU-LABEL: test8:
 ; MCU:       # BB#0:
 ; MCU-NEXT:    pushl %ebp
 ; MCU-NEXT:    pushl %ebx
 ; MCU-NEXT:    pushl %edi
 ; MCU-NEXT:    pushl %esi
-; MCU-NEXT:    andb $1, %al
+; MCU-NEXT:    testb $1, %al
 ; MCU-NEXT:    jne .LBB7_1
 ; MCU-NEXT:  # BB#2:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; MCU-NEXT:    movl (%ecx), %ecx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:    movl (%eax), %eax
 ; MCU-NEXT:    je .LBB7_5
 ; MCU-NEXT:  .LBB7_4:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    movl (%esi), %esi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; MCU-NEXT:    movl (%ecx), %ecx
 ; MCU-NEXT:    je .LBB7_8
 ; MCU-NEXT:  .LBB7_7:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
-; MCU-NEXT:    movl (%edi), %edi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    movl (%esi), %esi
 ; MCU-NEXT:    je .LBB7_11
 ; MCU-NEXT:  .LBB7_10:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
-; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
+; MCU-NEXT:    movl (%edi), %edi
 ; MCU-NEXT:    je .LBB7_14
 ; MCU-NEXT:  .LBB7_13:
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
+; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    je .LBB7_17
+; MCU-NEXT:  .LBB7_16:
 ; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
-; MCU-NEXT:    jmp .LBB7_15
+; MCU-NEXT:    jmp .LBB7_18
 ; MCU-NEXT:  .LBB7_1:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
-; MCU-NEXT:    movl (%ecx), %ecx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:    movl (%eax), %eax
 ; MCU-NEXT:    jne .LBB7_4
 ; MCU-NEXT:  .LBB7_5:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
-; MCU-NEXT:    movl (%esi), %esi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ecx
+; MCU-NEXT:    movl (%ecx), %ecx
 ; MCU-NEXT:    jne .LBB7_7
 ; MCU-NEXT:  .LBB7_8:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
-; MCU-NEXT:    movl (%edi), %edi
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; MCU-NEXT:    movl (%esi), %esi
 ; MCU-NEXT:    jne .LBB7_10
 ; MCU-NEXT:  .LBB7_11:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
-; MCU-NEXT:    movl (%ebx), %ebx
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %edi
+; MCU-NEXT:    movl (%edi), %edi
 ; MCU-NEXT:    jne .LBB7_13
 ; MCU-NEXT:  .LBB7_14:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
-; MCU-NEXT:  .LBB7_15:
-; MCU-NEXT:    movl (%ebp), %ebp
-; MCU-NEXT:    testb %al, %al
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebx
+; MCU-NEXT:    movl (%ebx), %ebx
 ; MCU-NEXT:    jne .LBB7_16
-; MCU-NEXT:  # BB#17:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
-; MCU-NEXT:    jmp .LBB7_18
-; MCU-NEXT:  .LBB7_16:
-; MCU-NEXT:    leal {{[0-9]+}}(%esp), %eax
+; MCU-NEXT:  .LBB7_17:
+; MCU-NEXT:    leal {{[0-9]+}}(%esp), %ebp
 ; MCU-NEXT:  .LBB7_18:
-; MCU-NEXT:    movl (%eax), %eax
-; MCU-NEXT:    decl %eax
+; MCU-NEXT:    movl (%ebp), %ebp
 ; MCU-NEXT:    decl %ebp
 ; MCU-NEXT:    decl %ebx
 ; MCU-NEXT:    decl %edi
 ; MCU-NEXT:    decl %esi
 ; MCU-NEXT:    decl %ecx
-; MCU-NEXT:    movl %ecx, 20(%edx)
-; MCU-NEXT:    movl %esi, 16(%edx)
-; MCU-NEXT:    movl %edi, 12(%edx)
-; MCU-NEXT:    movl %ebx, 8(%edx)
-; MCU-NEXT:    movl %ebp, 4(%edx)
-; MCU-NEXT:    movl %eax, (%edx)
+; MCU-NEXT:    decl %eax
+; MCU-NEXT:    movl %eax, 20(%edx)
+; MCU-NEXT:    movl %ecx, 16(%edx)
+; MCU-NEXT:    movl %esi, 12(%edx)
+; MCU-NEXT:    movl %edi, 8(%edx)
+; MCU-NEXT:    movl %ebx, 4(%edx)
+; MCU-NEXT:    movl %ebp, (%edx)
 ; MCU-NEXT:    popl %esi
 ; MCU-NEXT:    popl %edi
 ; MCU-NEXT:    popl %ebx
 ; MCU-NEXT:    popl %ebp
 ; MCU-NEXT:    retl
   %x = select i1 %c, <6 x i32> %src1, <6 x i32> %src2
   %val = sub <6 x i32> %x, < i32 1, i32 1, i32 1, i32 1, i32 1, i32 1 >
   store <6 x i32> %val, <6 x i32>* %dst.addr