AArch64: implement canonical `!isnan(...)` with single compare.

[t.p.northover]

Summary

LLVM canonicalizes a check for whether a number is NaN or not to fcmp ord %in, zeroinitializer. An ord check normally needs to be lowered to two separate comparisons on AArch64 but in this case it's equivalent to fcmp oeq %in, %in which we have a single fcmeq instruction for.

[t.p.northover]

Ping.

[paquette]

LGTM

[t.p.northover]

Thanks. Committed as 6b98824a587a.

llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp

static bool lowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
   unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits();
   if (EltSize != 32 && EltSize != 64)
     return false;
   Register RHS = MI.getOperand(3).getReg();
   auto Splat = getAArch64VectorSplat(*MRI.getVRegDef(RHS), MRI);
 
   // Compares against 0 have special target-specific pseudos.
   bool IsZero = Splat && Splat->isCst() && Splat->getCst() == 0;
-  bool Invert;
-  AArch64CC::CondCode CC, CC2;
+
+
+  bool Invert = false;
+  AArch64CC::CondCode CC, CC2 = AArch64CC::AL;
+  if (Pred == CmpInst::Predicate::FCMP_ORD && IsZero) {
+    // The special case "fcmp ord %a, 0" is the canonical check that LHS isn't
+    // NaN, so equivalent to a == a and doesn't need the two comparisons an
+    // "ord" normally would.
+    RHS = LHS;
+    IsZero = false;
+    CC = AArch64CC::EQ;
+  } else
     changeVectorFCMPPredToAArch64CC(Pred, CC, CC2, Invert);
+
   bool NoNans = ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
 
   // Instead of having an apply function, just build here to simplify things.
   MIB.setInstrAndDebugLoc(MI);
   auto Cmp = getVectorFCMP(CC, LHS, RHS, IsZero, NoNans, MRI);
   Register CmpRes;
   if (CC2 == AArch64CC::AL)
     CmpRes = Cmp(MIB);

llvm/test/CodeGen/AArch64/GlobalISel/lower-neon-vector-fcmp.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple aarch64 -run-pass=aarch64-postlegalizer-lowering -verify-machineinstrs %s -o - | FileCheck %s
 ...
 ---
 name:            oeq
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: oeq
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
-    ; CHECK: $q0 = COPY [[FCMEQ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMEQ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            oeq_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: oeq_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
-    ; CHECK: $q0 = COPY [[FCMEQZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK-NEXT: $q0 = COPY [[FCMEQZ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(oeq), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 
 ...
 ---
 name:            ogt
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: ogt
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
-    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ogt_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: ogt_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
-    ; CHECK: $q0 = COPY [[FCMGTZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK-NEXT: $q0 = COPY [[FCMGTZ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ogt), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            oge
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: oge
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
-    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            oge_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: oge_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
-    ; CHECK: $q0 = COPY [[FCMGEZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK-NEXT: $q0 = COPY [[FCMGEZ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(oge), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 
 ...
 ---
 name:            olt
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: olt
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
-    ; CHECK: $q0 = COPY [[FCMGT]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMGT]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            olt_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: olt_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
-    ; CHECK: $q0 = COPY [[FCMLTZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK-NEXT: $q0 = COPY [[FCMLTZ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(olt), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ole
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: ole
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
-    ; CHECK: $q0 = COPY [[FCMGE]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMGE]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ole_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
     ; CHECK-LABEL: name: ole_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
-    ; CHECK: $q0 = COPY [[FCMLEZ]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK-NEXT: $q0 = COPY [[FCMLEZ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ole), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            one
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Two compares.
 
     ; CHECK-LABEL: name: one
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[FCMGT1:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT1]], [[FCMGT]]
-    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[FCMGT1:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT1]], [[FCMGT]]
+    ; CHECK-NEXT: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            one_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Two compares.
 
     ; CHECK-LABEL: name: one_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
-    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGTZ]]
-    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK-NEXT: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGTZ]]
+    ; CHECK-NEXT: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(one), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            uno
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: uno
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            uno_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: uno_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
-    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK-NEXT: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[OR]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(uno), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ord
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Needs two compares. No invert.
 
     ; CHECK-LABEL: name: ord
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
-    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMGT]], [[FCMGE]]
+    ; CHECK-NEXT: $q0 = COPY [[OR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ord_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Needs two compares. No invert.
 
     ; CHECK-LABEL: name: ord_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
-    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
-    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s64>) = G_OR [[FCMLTZ]], [[FCMGEZ]]
-    ; CHECK: $q0 = COPY [[OR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMEQ]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ord), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ult
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ult
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ueq_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ueq_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGEZ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGEZ %lhs
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGEZ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ult), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ule
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ule
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ule_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ule_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGTZ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMGTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMGTZ %lhs
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGTZ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ule), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ugt
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ugt
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGE:%[0-9]+]]:_(<2 x s64>) = G_FCMGE %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGE]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            ugt_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: ugt_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLEZ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMLEZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLEZ %lhs
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLEZ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(ugt), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            uge
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: uge
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMGT:%[0-9]+]]:_(<2 x s64>) = G_FCMGT %rhs, %lhs(<2 x s64>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMGT]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            uge_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Should be inverted. Needs two compares.
 
     ; CHECK-LABEL: name: uge_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLTZ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMLTZ:%[0-9]+]]:_(<2 x s64>) = G_FCMLTZ %lhs
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMLTZ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(uge), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            une
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Negated EQ.
 
     ; CHECK-LABEL: name: une
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %rhs:_(<2 x s64>) = COPY $q1
-    ; CHECK: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<2 x s64>) = COPY $q1
+    ; CHECK-NEXT: [[FCMEQ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQ %lhs, %rhs(<2 x s64>)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %rhs:_(<2 x s64>) = COPY $q1
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %rhs
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            une_zero
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; Negated EQ.
 
     ; CHECK-LABEL: name: une_zero
-    ; CHECK: %lhs:_(<2 x s64>) = COPY $q0
-    ; CHECK: %zero:_(s64) = G_CONSTANT i64 0
-    ; CHECK: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
-    ; CHECK: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQZ]], [[BUILD_VECTOR]]
-    ; CHECK: $q0 = COPY [[XOR]](<2 x s64>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<2 x s64>) = COPY $q0
+    ; CHECK-NEXT: %zero:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero(s64), %zero(s64)
+    ; CHECK-NEXT: [[FCMEQZ:%[0-9]+]]:_(<2 x s64>) = G_FCMEQZ %lhs
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
+    ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<2 x s64>) = G_XOR [[FCMEQZ]], [[BUILD_VECTOR]]
+    ; CHECK-NEXT: $q0 = COPY [[XOR]](<2 x s64>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<2 x s64>) = COPY $q0
     %zero:_(s64) = G_CONSTANT i64 0
     %zero_vec:_(<2 x s64>) = G_BUILD_VECTOR %zero, %zero
     %fcmp:_(<2 x s64>) = G_FCMP floatpred(une), %lhs(<2 x s64>), %zero_vec
     $q0 = COPY %fcmp(<2 x s64>)
     RET_ReallyLR implicit $q0
 
 ...
 ---
 name:            dont_lower_s16
 alignment:       4
 legalized:       true
 body:             |
   bb.0:
     liveins: $q0, $q1
 
     ; CHECK-LABEL: name: dont_lower_s16
-    ; CHECK: %lhs:_(<8 x s16>) = COPY $q0
-    ; CHECK: %rhs:_(<8 x s16>) = COPY $q1
-    ; CHECK: %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
-    ; CHECK: $q0 = COPY %fcmp(<8 x s16>)
-    ; CHECK: RET_ReallyLR implicit $q0
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<8 x s16>) = COPY $q0
+    ; CHECK-NEXT: %rhs:_(<8 x s16>) = COPY $q1
+    ; CHECK-NEXT: %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
+    ; CHECK-NEXT: $q0 = COPY %fcmp(<8 x s16>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %lhs:_(<8 x s16>) = COPY $q0
     %rhs:_(<8 x s16>) = COPY $q1
     %fcmp:_(<8 x s16>) = G_FCMP floatpred(oeq), %lhs(<8 x s16>), %rhs
     $q0 = COPY %fcmp(<8 x s16>)
     RET_ReallyLR implicit $q0
+
+...
+---
+name:            is_not_nan
+alignment:       4
+legalized:       true
+body:             |
+  bb.0:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: is_not_nan
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %lhs:_(<4 x s32>) = COPY $q0
+    ; CHECK-NEXT: [[FCMEQ:%[0-9]+]]:_(<4 x s32>) = G_FCMEQ %lhs, %lhs(<4 x s32>)
+    ; CHECK-NEXT: $q0 = COPY [[FCMEQ]](<4 x s32>)
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %lhs:_(<4 x s32>) = COPY $q0
+    %zero:_(s32) = G_FCONSTANT float 0.000000e+00
+    %veczero:_(<4 x s32>) = G_BUILD_VECTOR %zero, %zero, %zero, %zero
+    %fcmp:_(<4 x s32>) = G_FCMP floatpred(ord), %lhs(<4 x s32>), %veczero
+    $q0 = COPY %fcmp(<4 x s32>)
+    RET_ReallyLR implicit $q0

llvm/test/CodeGen/AArch64/neon-compare-instructions.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz2xfloat:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
-; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
-; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v0.2s
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 ; ORD with zero = OLT | OGE
 define <4 x i32> @fcmordz4xfloat(<4 x float> %A) {
 ; CHECK-LABEL: fcmordz4xfloat:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz4xfloat:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
-; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
-; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v0.4s
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; ORD with zero = OLT | OGE
 define <2 x i64> @fcmordz2xdouble(<2 x double> %A) {
 ; CHECK-LABEL: fcmordz2xdouble:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz2xdouble:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
-; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
-; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v0.2d
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp ord <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; UEQ with zero = !ONE = !(OLT |OGT)
 define <2 x i32> @fcmueqz2xfloat(<2 x float> %A) {

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2s, v0.2s, #0.0
 ; CHECK-NEXT:    fcmlt v0.2s, v0.2s, #0.0
 ; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz2xfloat_fast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.2s, v0.2s, #0.0
-; GISEL-NEXT:    fcmlt v0.2s, v0.2s, #0.0
-; GISEL-NEXT:    orr v0.8b, v0.8b, v1.8b
+; GISEL-NEXT:    fcmeq v0.2s, v0.2s, v0.2s
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp fast ord <2 x float> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
   ret <2 x i32> %tmp4
 }
 
 define <4 x i32> @fcmordz4xfloat_fast(<4 x float> %A) {
 ; CHECK-LABEL: fcmordz4xfloat_fast:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.4s, v0.4s, #0.0
 ; CHECK-NEXT:    fcmlt v0.4s, v0.4s, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz4xfloat_fast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.4s, v0.4s, #0.0
-; GISEL-NEXT:    fcmlt v0.4s, v0.4s, #0.0
-; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    fcmeq v0.4s, v0.4s, v0.4s
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp fast ord <4 x float> %A, zeroinitializer
   %tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 define <2 x i64> @fcmordz2xdouble_fast(<2 x double> %A) {
 ; CHECK-LABEL: fcmordz2xdouble_fast:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcmge v1.2d, v0.2d, #0.0
 ; CHECK-NEXT:    fcmlt v0.2d, v0.2d, #0.0
 ; CHECK-NEXT:    orr v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: fcmordz2xdouble_fast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    fcmge v1.2d, v0.2d, #0.0
-; GISEL-NEXT:    fcmlt v0.2d, v0.2d, #0.0
-; GISEL-NEXT:    orr v0.16b, v0.16b, v1.16b
+; GISEL-NEXT:    fcmeq v0.2d, v0.2d, v0.2d
 ; GISEL-NEXT:    ret
   %tmp3 = fcmp fast ord <2 x double> %A, zeroinitializer
   %tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 define <2 x i32> @fcmueqz2xfloat_fast(<2 x float> %A) {
 ; CHECK-LABEL: fcmueqz2xfloat_fast: