[X86] Combine (cmov (and/or (setcc) (setcc))) into (cmov (cmov))

[ab]

Summary

Fold and/or of setcc's to double CMOV:

(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)

In practice, when we need to lower to control flow using a custom inserter,
we represent the double CMOV using a special node, CMOV2:

(CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV2 F, T, cc1, cc2)
(CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV2 T, F, !cc1, !cc2)

This enables the custom inserter to only need to insert one PHI, instead
of two if it looked at two independent CMOVs.

This combine lets us generate:

cmovcc1 (jcc1 if we don't have CMOV)
cmovcc2 (same)

instead of:

setcc1
setcc2
and/or
cmovne (jne if we don't have CMOV)

When we can't use the CMOV instruction, it might increase branch mispredicts.
When we can, or when there is no mispredict, this improves throughput and reduces register pressure.

---

This is pretty much the same thing as D7622, except here the pattern is exposed during ops legalization. By the time we get to the combiner, we already have X86ISD nodes, so even an independent combine wouldn't have been useful.

[EDIT: dropped the CMOV2 for now, I'll put that in a separate patch]
The CMOV2 pseudo is yucky, but I can't see an alternative. If you do it the obvious way (just form two CMOVs all the time) the custom inserter (which definitely should not look at other instructions) adds a PHI between the two jumps, which ends up creating a few copies all around. For instance, for the (sitofp (zext (fcmp une))) testcase:

        ucomiss %xmm1, %xmm0
        movss  <1.0f>, %xmm0
        movaps  %xmm0, %xmm1
        jne     .LBB5_2
        xorps   %xmm1, %xmm1
.LBB5_2:
        jp      .LBB5_4
        movaps  %xmm1, %xmm0
.LBB5_4:
        retq

We could avoid the problem by optimizing something like this:

  A
  | \
  |  B
  | /
  C
  | \
  |  D
  | /
  E

A: X = ...; Y = ...
B: empty
C: Z = PHI [X, A], [Y, B]
D: empty
E: PHI [X, C], [Z, D]

To this:

  A
  | \
  |  C
  | /|
  |/ |
  |  |
  |  D
  | /
  E

A: X = ...; Y = ...
D: empty
E: PHI [X, A], [X, C], [Z, D]

That sounds awfully specific though, and, having looked into it, I'm not sure which more general pass should do it.

Of course, we could just ignore the problem, and have a few extra MOVs around.

[ab]

• Add missing CMOV2 documentation.
• Isolate CMOV pseudo multiclass refactoring

[ab]

Add back the test file..

[ab]

• Split out CMOV2 changes

[MatzeB]

I've got some small nitpicks, but LGTM.

lib/Target/X86/X86ISelLowering.cpp

21016

I would move this before the switch statement to be closer to the def.

21240

I would not initialize this var here and rather make sure that checkBoolTestAndOrSetCCCombine() always sets isAndSetCC if it returns true.

[ab]

r231045.

I guess phabricator isn't content with a link to the revision to auto-close.

-Ahmed

lib/Target/X86/X86ISelLowering.cpp

if (needOppositeCond)
       CC = X86::GetOppositeBranchCondition(CC);
     return SetCC.getOperand(3);
   }
   }
 
   return SDValue();
 }
 
+/// Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
+/// Match:
+///   (X86or (X86setcc) (X86setcc))
+///   (X86cmp (and (X86setcc) (X86setcc)), 0)
+static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
+                                           X86::CondCode &CC1, SDValue &Flags,
+                                           bool &isAnd) {
+  SDValue SetCC0, SetCC1;
+
+  if (Cond->getOpcode() == X86ISD::CMP) {
+    ConstantSDNode *CondOp1C = dyn_cast<ConstantSDNode>(Cond->getOperand(1));
+    if (!CondOp1C || !CondOp1C->isNullValue())
+      return false;
+
+    Cond = Cond->getOperand(0);
+  }
+
+  switch (Cond->getOpcode()) {
+  default: return false;
+  case ISD::AND:
+  case X86ISD::AND:
+    isAnd = true;
+    // fallthru
+  case ISD::OR:
+  case X86ISD::OR:
+    SetCC0 = Cond->getOperand(0);
+    SetCC1 = Cond->getOperand(1);
+    break;
+  };
+
+  // Make sure we have SETCC nodes, using the same flags value.
+  if (SetCC0.getOpcode() != X86ISD::SETCC ||
+      SetCC1.getOpcode() != X86ISD::SETCC ||
+      SetCC0->getOperand(1) != SetCC1->getOperand(1))
+    return false;
+
+  CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
+  CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
+  Flags = SetCC0->getOperand(1);
+  return true;
+}
+
 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
                                   TargetLowering::DAGCombinerInfo &DCI,
                                   const X86Subtarget *Subtarget) {
   SDLoc DL(N);
 
   // If the flag operand isn't dead, don't touch this CMOV.
   if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
         SDValue Ops[] = { FalseOp, Cond.getOperand(0),
                           DAG.getConstant(CC, MVT::i8), Cond };
         return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
       }
     }
   }
 
+  // Fold and/or of setcc's to double CMOV:
+  //   (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
+  //   (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
+  //
+  // This combine lets us generate:
+  //   cmovcc1 (jcc1 if we don't have CMOV)
+  //   cmovcc2 (same)
+  // instead of:
+  //   setcc1
+  //   setcc2
+  //   and/or
+  //   cmovne (jne if we don't have CMOV)
+  // When we can't use the CMOV instruction, it might increase branch
+  // mispredicts.
+  // When we can use CMOV, or when there is no mispredict, this improves
+  // throughput and reduces register pressure.
+  //
+  if (CC == X86::COND_NE) {
+    SDValue Flags;
+    X86::CondCode CC0, CC1;
+    bool isAndSetCC = false;
+    if (checkBoolTestAndOrSetCCCombine(Cond, CC0, CC1, Flags, isAndSetCC)) {
+      if (isAndSetCC) {
+        std::swap(FalseOp, TrueOp);
+        CC0 = X86::GetOppositeBranchCondition(CC0);
+        CC1 = X86::GetOppositeBranchCondition(CC1);
+      }
+
+      SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, MVT::i8),
+        Flags};
+      SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), LOps);
+      SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, MVT::i8), Flags};
+      SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
+      DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(CMOV.getNode(), 1));
+      return CMOV;
+    }
+  }
+
   return SDValue();
 }
 
 static SDValue PerformINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG,
                                                 const X86Subtarget *Subtarget) {
   unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
   switch (IntNo) {
   default: return SDValue();

test/CodeGen/X86/cmovcmov.ll

+; RUN: llc < %s -asm-verbose=false -mtriple=x86_64-unknown-linux | FileCheck %s --check-prefix=CHECK --check-prefix=CMOV
+; RUN: llc < %s -asm-verbose=false -mtriple=i686-unknown-linux | FileCheck %s --check-prefix=CHECK --check-prefix=NOCMOV
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test 2xCMOV patterns exposed after legalization.
+; One way to do that is with (select (fcmp une/oeq)), which gets
+; legalized to setp/setne.
+
+; CHECK-LABEL: test_select_fcmp_oeq_i32:
+
+; CMOV-NEXT: ucomiss  %xmm1, %xmm0
+; CMOV-NEXT: cmovnel  %esi, %edi
+; CMOV-NEXT: cmovpl  %esi, %edi
+; CMOV-NEXT: movl  %edi, %eax
+; CMOV-NEXT: retq
+
+; NOCMOV-NEXT:   flds  8(%esp)
+; NOCMOV-NEXT:   flds  4(%esp)
+; NOCMOV-NEXT:   fucompp
+; NOCMOV-NEXT:   fnstsw  %ax
+; NOCMOV-NEXT:   sahf
+; NOCMOV-NEXT:   leal  16(%esp), %eax
+; NOCMOV-NEXT:   movl  %eax, %ecx
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  12(%esp), %ecx
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %ecx, %eax
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%eax), %eax
+; NOCMOV-NEXT:   retl
+define i32 @test_select_fcmp_oeq_i32(float %a, float %b, i32 %c, i32 %d) #0 {
+entry:
+  %cmp = fcmp oeq float %a, %b
+  %r = select i1 %cmp, i32 %c, i32 %d
+  ret i32 %r
+}
+
+; CHECK-LABEL: test_select_fcmp_oeq_i64:
+
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   cmovneq  %rsi, %rdi
+; CMOV-NEXT:   cmovpq  %rsi, %rdi
+; CMOV-NEXT:   movq  %rdi, %rax
+; CMOV-NEXT:   retq
+
+; NOCMOV-NEXT:   flds  8(%esp)
+; NOCMOV-NEXT:   flds  4(%esp)
+; NOCMOV-NEXT:   fucompp
+; NOCMOV-NEXT:   fnstsw  %ax
+; NOCMOV-NEXT:   sahf
+; NOCMOV-NEXT:   leal  20(%esp), %ecx
+; NOCMOV-NEXT:   movl  %ecx, %eax
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  12(%esp), %eax
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %eax, %ecx
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%ecx), %eax
+; NOCMOV-NEXT:   orl  $4, %ecx
+; NOCMOV-NEXT:   movl  (%ecx), %edx
+; NOCMOV-NEXT:   retl
+define i64 @test_select_fcmp_oeq_i64(float %a, float %b, i64 %c, i64 %d) #0 {
+entry:
+  %cmp = fcmp oeq float %a, %b
+  %r = select i1 %cmp, i64 %c, i64 %d
+  ret i64 %r
+}
+
+; CHECK-LABEL: test_select_fcmp_une_i64:
+
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   cmovneq  %rdi, %rsi
+; CMOV-NEXT:   cmovpq  %rdi, %rsi
+; CMOV-NEXT:   movq  %rsi, %rax
+; CMOV-NEXT:   retq
+
+; NOCMOV-NEXT:   flds  8(%esp)
+; NOCMOV-NEXT:   flds  4(%esp)
+; NOCMOV-NEXT:   fucompp
+; NOCMOV-NEXT:   fnstsw  %ax
+; NOCMOV-NEXT:   sahf
+; NOCMOV-NEXT:   leal  12(%esp), %ecx
+; NOCMOV-NEXT:   movl  %ecx, %eax
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  20(%esp), %eax
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %eax, %ecx
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%ecx), %eax
+; NOCMOV-NEXT:   orl  $4, %ecx
+; NOCMOV-NEXT:   movl  (%ecx), %edx
+; NOCMOV-NEXT:   retl
+define i64 @test_select_fcmp_une_i64(float %a, float %b, i64 %c, i64 %d) #0 {
+entry:
+  %cmp = fcmp une float %a, %b
+  %r = select i1 %cmp, i64 %c, i64 %d
+  ret i64 %r
+}
+
+; CHECK-LABEL: test_select_fcmp_oeq_f64:
+
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   movaps  %xmm3, %xmm0
+; CMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm2, %xmm0
+; CMOV-NEXT: [[TBB1]]:
+; CMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm0, %xmm3
+; CMOV-NEXT: [[TBB2]]:
+; CMOV-NEXT:   movaps  %xmm3, %xmm0
+; CMOV-NEXT:   retq
+
+; NOCMOV-NEXT:   flds  8(%esp)
+; NOCMOV-NEXT:   flds  4(%esp)
+; NOCMOV-NEXT:   fucompp
+; NOCMOV-NEXT:   fnstsw  %ax
+; NOCMOV-NEXT:   sahf
+; NOCMOV-NEXT:   leal  20(%esp), %eax
+; NOCMOV-NEXT:   movl  %eax, %ecx
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  12(%esp), %ecx
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %ecx, %eax
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   fldl  (%eax)
+; NOCMOV-NEXT:   retl
+define double @test_select_fcmp_oeq_f64(float %a, float %b, double %c, double %d) #0 {
+entry:
+  %cmp = fcmp oeq float %a, %b
+  %r = select i1 %cmp, double %c, double %d
+  ret double %r
+}
+
+; CHECK-LABEL: test_select_fcmp_oeq_v4i32:
+
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   movaps  %xmm3, %xmm0
+; CMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm2, %xmm0
+; CMOV-NEXT: [[TBB1]]:
+; CMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm0, %xmm3
+; CMOV-NEXT: [[TBB2]]:
+; CMOV-NEXT:   movaps  %xmm3, %xmm0
+; CMOV-NEXT:   retq
+
+; NOCMOV-NEXT:   pushl  %ebx
+; NOCMOV-NEXT:   pushl  %edi
+; NOCMOV-NEXT:   pushl  %esi
+; NOCMOV-NEXT:   flds  24(%esp)
+; NOCMOV-NEXT:   flds  20(%esp)
+; NOCMOV-NEXT:   fucompp
+; NOCMOV-NEXT:   fnstsw  %ax
+; NOCMOV-NEXT:   sahf
+; NOCMOV-NEXT:   leal  44(%esp), %eax
+; NOCMOV-NEXT:   movl  %eax, %ecx
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  28(%esp), %ecx
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %ecx, %eax
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%eax), %eax
+; NOCMOV-NEXT:   leal  48(%esp), %ecx
+; NOCMOV-NEXT:   movl  %ecx, %edx
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  32(%esp), %edx
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %edx, %ecx
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%ecx), %ecx
+; NOCMOV-NEXT:   leal  52(%esp), %edx
+; NOCMOV-NEXT:   movl  %edx, %esi
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  36(%esp), %esi
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %esi, %edx
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%edx), %edx
+; NOCMOV-NEXT:   leal  56(%esp), %esi
+; NOCMOV-NEXT:   movl  %esi, %ebx
+; NOCMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   leal  40(%esp), %ebx
+; NOCMOV-NEXT: [[TBB1]]:
+; NOCMOV-NEXT:   movl  16(%esp), %edi
+; NOCMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; NOCMOV-NEXT:   movl  %ebx, %esi
+; NOCMOV-NEXT: [[TBB2]]:
+; NOCMOV-NEXT:   movl  (%esi), %esi
+; NOCMOV-NEXT:   movl  %esi, 12(%edi)
+; NOCMOV-NEXT:   movl  %edx, 8(%edi)
+; NOCMOV-NEXT:   movl  %ecx, 4(%edi)
+; NOCMOV-NEXT:   movl  %eax, (%edi)
+; NOCMOV-NEXT:   popl  %esi
+; NOCMOV-NEXT:   popl  %edi
+; NOCMOV-NEXT:   popl  %ebx
+; NOCMOV-NEXT:   retl  $4
+define <4 x i32> @test_select_fcmp_oeq_v4i32(float %a, float %b, <4 x i32> %c, <4 x i32> %d) #0 {
+entry:
+  %cmp = fcmp oeq float %a, %b
+  %r = select i1 %cmp, <4 x i32> %c, <4 x i32> %d
+  ret <4 x i32> %r
+}
+
+; Also make sure we catch the original code-sequence of interest:
+
+; CMOV: [[ONE_F32_LCPI:.LCPI.*]]:
+; CMOV-NEXT:   .long  1065353216
+
+; CHECK-LABEL: test_zext_fcmp_une:
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   movss  [[ONE_F32_LCPI]](%rip), %xmm0
+; CMOV-NEXT:   movaps  %xmm0, %xmm1
+; CMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; CMOV-NEXT:   xorps  %xmm1, %xmm1
+; CMOV-NEXT: [[TBB1]]:
+; CMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm1, %xmm0
+; CMOV-NEXT: [[TBB2]]:
+; CMOV-NEXT:   retq
+
+; NOCMOV:     jne
+; NOCMOV:     jp
+define float @test_zext_fcmp_une(float %a, float %b) #0 {
+entry:
+  %cmp = fcmp une float %a, %b
+  %conv1 = zext i1 %cmp to i32
+  %conv2 = sitofp i32 %conv1 to float
+  ret float %conv2
+}
+
+; CMOV: [[ONE_F32_LCPI:.LCPI.*]]:
+; CMOV-NEXT:   .long  1065353216
+
+; CHECK-LABEL: test_zext_fcmp_oeq:
+; CMOV-NEXT:   ucomiss  %xmm1, %xmm0
+; CMOV-NEXT:   xorps  %xmm0, %xmm0
+; CMOV-NEXT:   xorps  %xmm1, %xmm1
+; CMOV-NEXT:   jne  [[TBB1:.LBB[0-9_]+]]
+; CMOV-NEXT:   movss  [[ONE_F32_LCPI]](%rip), %xmm1
+; CMOV-NEXT: [[TBB1]]:
+; CMOV-NEXT:   jp  [[TBB2:.LBB[0-9_]+]]
+; CMOV-NEXT:   movaps  %xmm1, %xmm0
+; CMOV-NEXT: [[TBB2]]:
+; CMOV-NEXT:   retq
+
+; NOCMOV:   jne
+; NOCMOV:   jp
+define float @test_zext_fcmp_oeq(float %a, float %b) #0 {
+entry:
+  %cmp = fcmp oeq float %a, %b
+  %conv1 = zext i1 %cmp to i32
+  %conv2 = sitofp i32 %conv1 to float
+  ret float %conv2
+}
+
+attributes #0 = { nounwind }

test/CodeGen/X86/fast-isel-select-cmov2.ll

-; RUN: llc < %s -mtriple=x86_64-apple-darwin10                             | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10                             | FileCheck %s --check-prefix=CHECK --check-prefix=SDAG
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefix=CHECK --check-prefix=FAST
 
 ; Test all the cmp predicates that can feed an integer conditional move.
 
 define i64 @select_fcmp_false_cmov(double %a, double %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: select_fcmp_false_cmov
 ; CHECK:       movq %rsi, %rax
 ; CHECK-NEXT:  retq
   %1 = fcmp false double %a, %b
   %2 = select i1 %1, i64 %c, i64 %d
   ret i64 %2
 }
 
 define i64 @select_fcmp_oeq_cmov(double %a, double %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: select_fcmp_oeq_cmov
 ; CHECK:       ucomisd %xmm1, %xmm0
-; CHECK-NEXT:  setnp %al
-; CHECK-NEXT:  sete %cl
-; CHECK-NEXT:  testb %al, %cl
-; CHECK-NEXT:  cmoveq %rsi, %rdi
+; SDAG-NEXT:   cmovneq %rsi, %rdi
+; SDAG-NEXT:   cmovpq %rsi, %rdi
+; SDAG-NEXT:   movq %rdi, %rax
+; FAST-NEXT:   setnp %al
+; FAST-NEXT:   sete %cl
+; FAST-NEXT:   testb %al, %cl
+; FAST-NEXT:   cmoveq %rsi, %rdi
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, i64 %c, i64 %d
   ret i64 %2
 }
 
 define i64 @select_fcmp_ogt_cmov(double %a, double %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: select_fcmp_ogt_cmov
 ; CHECK:       ucomisd %xmm1, %xmm0
; CHECK-NEXT:  cmovaq %rsi, %rdi
   %1 = fcmp ule double %a, %b
   %2 = select i1 %1, i64 %c, i64 %d
   ret i64 %2
 }
 
 define i64 @select_fcmp_une_cmov(double %a, double %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: select_fcmp_une_cmov
 ; CHECK:       ucomisd %xmm1, %xmm0
-; CHECK-NEXT:  setp %al
-; CHECK-NEXT:  setne %cl
-; CHECK-NEXT:  orb %al, %cl
-; CHECK-NEXT:  cmoveq %rsi, %rdi
+; SDAG-NEXT:   cmovneq %rdi, %rsi
+; SDAG-NEXT:   cmovpq %rdi, %rsi
+; SDAG-NEXT:   movq %rsi, %rax
+; FAST-NEXT:   setp %al
+; FAST-NEXT:   setne %cl
+; FAST-NEXT:   orb %al, %cl
+; FAST-NEXT:   cmoveq %rsi, %rdi
   %1 = fcmp une double %a, %b
   %2 = select i1 %1, i64 %c, i64 %d
   ret i64 %2
 }
 
 define i64 @select_fcmp_true_cmov(double %a, double %b, i64 %c, i64 %d) {
 ; CHECK-LABEL: select_fcmp_true_cmov
 ; CHECK:       movq %rdi, %rax