[GlobalISel][AArch64] Fold G_SUB into G_ICMP when it's safe to do so

[paquette]

Summary

Basically porting over the behaviour in AArch64ISelLowering to GISel. See emitComparison for reference.

When we have something like this:

lhs = G_SUB 0, y
...
G_ICMP lhs, rhs

We can fold away the G_SUB and produce a cmn instead, given that we produce the same value in NZCV.

Add a test showing that the transformation works, and also showing that we don't perform the transformation when it's unsafe.

[arsenm]

I thought PeepholeOptimizer took care of this sort of thing?

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

3033–3034 ↗

(On Diff #204148)

No else after return

[aemerson]

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

2967 ↗

(On Diff #204148)

This function generates semantically incorrect GMIR and then relies on the caller to fix and select it. I would rather we also modify the opcode in this function to ANDS[WX]rr and then have the caller check the return value to see if it needs to modify the opc. That way, on entry and exit we have semantically correct, if only partially selected, MIR.

[paquette]

• tryFoldCMN -> tryOptCMN
• Make tryOptCMN single-exit
• Make tryOptCMN not leave the G_ICMP in a half-selected state; now it selects the CMN entirely.
• Factor out cset emission into emitCSet.

[paquette]

In D63163#1538760, @arsenm wrote:

I thought PeepholeOptimizer took care of this sort of thing?

In AArch64, these sorts of things are also explicitly matched and selected in AArch64ISelLowering.cpp.

I guess that it's a pain to match past ISel, so I guess SDAG tries to do some of the work early in.

[aemerson]

LGTM with a further change.

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

3066 ↗

(On Diff #204849)

I think this predicate inversion is always needed when using CSINC right? So it can be folded into the emitCSet() function which can take the predicate.

llvm/trunk/lib/Target/AArch64/AArch64InstructionSelector.cpp

MachineInstr *emitExtractVectorElt(Optional<unsigned> DstReg,
                                      MachineIRBuilder &MIRBuilder) const;
 
   /// Helper function for selecting G_FCONSTANT. If the G_FCONSTANT can be
   /// materialized using a FMOV instruction, then update MI and return it.
   /// Otherwise, do nothing and return a nullptr.
   MachineInstr *emitFMovForFConstant(MachineInstr &MI,
                                      MachineRegisterInfo &MRI) const;
 
+  /// Emit a CSet for a compare.
+  MachineInstr *emitCSetForICMP(unsigned DefReg, unsigned Pred,
+                                MachineIRBuilder &MIRBuilder) const;
+
   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
 
   ComplexRendererFns selectAddrModeUnscaled(MachineOperand &Root,
                                             unsigned Size) const;
 
   ComplexRendererFns selectAddrModeUnscaled8(MachineOperand &Root) const {
     return selectAddrModeUnscaled(Root, 1);
   }
MachineInstr *findMIFromReg(unsigned Reg, unsigned Opc,
     if (!Def || Def->getOpcode() != Opc)
       return nullptr;
     return Def;
   }
 
   bool tryOptVectorShuffle(MachineInstr &I) const;
   bool tryOptVectorDup(MachineInstr &MI) const;
   bool tryOptSelect(MachineInstr &MI) const;
+  bool tryOptCMN(MachineInstr &MI) const;
 
   const AArch64TargetMachine &TM;
   const AArch64Subtarget &STI;
   const AArch64InstrInfo &TII;
   const AArch64RegisterInfo &TRI;
   const AArch64RegisterBankInfo &RBI;
 
 #define GET_GLOBALISEL_PREDICATES_DECL
if (Ty != LLT::scalar(32)) {
       LLVM_DEBUG(dbgs() << "G_ICMP result has type: " << Ty
                         << ", expected: " << LLT::scalar(32) << '\n');
       return false;
     }
 
     unsigned CmpOpc = 0;
     unsigned ZReg = 0;
 
+    // Check if this compare can be represented as a cmn, and perform any
+    // necessary transformations to do so.
+    if (tryOptCMN(I))
+      return true;
+
     LLT CmpTy = MRI.getType(I.getOperand(2).getReg());
     if (CmpTy == LLT::scalar(32)) {
       CmpOpc = AArch64::SUBSWrr;
       ZReg = AArch64::WZR;
     } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) {
       CmpOpc = AArch64::SUBSXrr;
       ZReg = AArch64::XZR;
     } else {
       return false;
     }
 
     // Try to match immediate forms.
     auto ImmFns = selectArithImmed(I.getOperand(3));
     if (ImmFns)
       CmpOpc = CmpOpc == AArch64::SUBSWrr ? AArch64::SUBSWri : AArch64::SUBSXri;
 
-    // CSINC increments the result by one when the condition code is false.
-    // Therefore, we have to invert the predicate to get an increment by 1 when
-    // the predicate is true.
-    const AArch64CC::CondCode invCC =
-        changeICMPPredToAArch64CC(CmpInst::getInversePredicate(
-            (CmpInst::Predicate)I.getOperand(1).getPredicate()));
-
     auto CmpMI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc))
                      .addDef(ZReg)
                      .addUse(I.getOperand(2).getReg());
 
     // If we matched a valid constant immediate, add those operands.
     if (ImmFns) {
       for (auto &RenderFn : *ImmFns)
         RenderFn(CmpMI);
     } else {
       CmpMI.addUse(I.getOperand(3).getReg());
     }
 
-    MachineInstr &CSetMI =
-        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr))
-             .addDef(I.getOperand(0).getReg())
-             .addUse(AArch64::WZR)
-             .addUse(AArch64::WZR)
-             .addImm(invCC);
-
+    MachineIRBuilder MIRBuilder(I);
+    emitCSetForICMP(I.getOperand(0).getReg(), I.getOperand(1).getPredicate(),
+             MIRBuilder);
     constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
-    constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI);
-
     I.eraseFromParent();
     return true;
   }
 
   case TargetOpcode::G_FCMP: {
     if (Ty != LLT::scalar(32)) {
       LLVM_DEBUG(dbgs() << "G_FCMP result has type: " << Ty
                         << ", expected: " << LLT::scalar(32) << '\n');
MachineInstr *AArch64InstructionSelector::emitFMovForFConstant(
   // Update MI to represent the new FMOV instruction, constrain it, and return.
   ImmOp.ChangeToImmediate(Imm);
   unsigned MovOpc = DefSize == 32 ? AArch64::FMOVSi : AArch64::FMOVDi;
   I.setDesc(TII.get(MovOpc));
   constrainSelectedInstRegOperands(I, TII, TRI, RBI);
   return &I;
 }
 
+MachineInstr *
+AArch64InstructionSelector::emitCSetForICMP(unsigned DefReg, unsigned Pred,
+                                     MachineIRBuilder &MIRBuilder) const {
+  // CSINC increments the result when the predicate is false. Invert it.
+  const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
+      CmpInst::getInversePredicate((CmpInst::Predicate)Pred));
+  auto I =
+      MIRBuilder
+          .buildInstr(AArch64::CSINCWr, {DefReg}, {AArch64::WZR, AArch64::WZR})
+          .addImm(InvCC);
+  constrainSelectedInstRegOperands(*I, TII, TRI, RBI);
+  return &*I;
+}
+
 bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const {
   MachineIRBuilder MIB(I);
   MachineRegisterInfo &MRI = *MIB.getMRI();
   const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
 
   // We want to recognize this pattern:
   //
   // $z = G_FCMP pred, $x, $y
auto CSel =
                      {I.getOperand(2).getReg(), I.getOperand(3).getReg()})
           .addImm(CondCode);
   constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
   constrainSelectedInstRegOperands(*CSel, TII, TRI, RBI);
   I.eraseFromParent();
   return true;
 }
 
+bool AArch64InstructionSelector::tryOptCMN(MachineInstr &I) const {
+  assert(I.getOpcode() == TargetOpcode::G_ICMP && "Expected G_ICMP");
+  MachineIRBuilder MIRBuilder(I);
+  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
+  // We want to find this sort of thing:
+  // x = G_SUB 0, y
+  // G_ICMP z, x
+  //
+  // In this case, we can fold the G_SUB into the G_ICMP using a CMN instead.
+  // e.g:
+  //
+  // cmn z, y
+
+  // Helper lambda to find the def.
+  auto FindDef = [&](unsigned VReg) {
+    MachineInstr *Def = MRI.getVRegDef(VReg);
+    while (Def) {
+      if (Def->getOpcode() != TargetOpcode::COPY)
+        break;
+      // Copies can be from physical registers. If we hit this, we're done.
+      if (TargetRegisterInfo::isPhysicalRegister(Def->getOperand(1).getReg()))
+        break;
+      Def = MRI.getVRegDef(Def->getOperand(1).getReg());
+    }
+    return Def;
+  };
+
+  // Helper lambda to detect the subtract followed by the compare.
+  // Takes in the def of the LHS or RHS, and checks if it's a subtract from 0.
+  auto IsCMN = [&](MachineInstr *DefMI, const AArch64CC::CondCode &CC) {
+    if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_SUB)
+      return false;
+
+    // Need to make sure NZCV is the same at the end of the transformation.
+    if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
+      return false;
+
+    // We want to match against SUBs.
+    if (DefMI->getOpcode() != TargetOpcode::G_SUB)
+      return false;
+
+    // Make sure that we're getting
+    // x = G_SUB 0, y
+    auto ValAndVReg =
+        getConstantVRegValWithLookThrough(DefMI->getOperand(1).getReg(), MRI);
+    if (!ValAndVReg || ValAndVReg->Value != 0)
+      return false;
+
+    // This can safely be represented as a CMN.
+    return true;
+  };
+
+  // Check if the RHS or LHS of the G_ICMP is defined by a SUB
+  MachineInstr *LHSDef = FindDef(I.getOperand(2).getReg());
+  MachineInstr *RHSDef = FindDef(I.getOperand(3).getReg());
+  const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
+      (CmpInst::Predicate)I.getOperand(1).getPredicate());
+  bool DidFold = false;
+  if (IsCMN(LHSDef, CC)) {
+    // We're doing this:
+    //
+    // Given:
+    //
+    // x = G_SUB 0, y
+    // G_ICMP x, z
+    //
+    // Update the G_ICMP:
+    //
+    // G_ICMP y, z
+    I.getOperand(2).setReg(LHSDef->getOperand(2).getReg());
+    DidFold = true;
+  } else if (IsCMN(RHSDef, CC)) {
+    // Same idea here, but with the RHS of the compare instead:
+    //
+    // Given:
+    //
+    // x = G_SUB 0, y
+    // G_ICMP z, x
+    //
+    // Update the G_ICMP:
+    //
+    // G_ICMP z, y
+    I.getOperand(3).setReg(RHSDef->getOperand(2).getReg());
+    DidFold = true;
+  }
+
+  if (DidFold) {
+    // We can fold. Emit a CMN.
+    static const unsigned OpcTable[2][2]{{AArch64::ADDSXrr, AArch64::ADDSXri},
+                                         {AArch64::ADDSWrr, AArch64::ADDSWri}};
+    bool Is32Bit =
+        (MRI.getType(I.getOperand(2).getReg()).getSizeInBits() == 32);
+    auto ImmFns = selectArithImmed(I.getOperand(3));
+    unsigned Opc = OpcTable[Is32Bit][ImmFns.hasValue()];
+    unsigned ZReg = Is32Bit ? AArch64::WZR : AArch64::XZR;
+
+    auto CmpMI = MIRBuilder.buildInstr(Opc, {ZReg}, {I.getOperand(2).getReg()});
+
+    // If we matched a valid constant immediate, add those operands.
+    if (ImmFns) {
+      for (auto &RenderFn : *ImmFns)
+        RenderFn(CmpMI);
+    } else {
+      CmpMI.addUse(I.getOperand(3).getReg());
+    }
+
+    constrainSelectedInstRegOperands(*CmpMI, TII, TRI, RBI);
+
+    // Add a CSet after the CMN.
+    emitCSetForICMP(I.getOperand(0).getReg(), I.getOperand(1).getPredicate(),
+             MIRBuilder);
+    I.eraseFromParent();
+  }
+
+  return DidFold;
+}
+
 bool AArch64InstructionSelector::tryOptVectorDup(MachineInstr &I) const {
   // Try to match a vector splat operation into a dup instruction.
   // We're looking for this pattern:
   //    %scalar:gpr(s64) = COPY $x0
   //    %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
   //    %cst0:gpr(s32) = G_CONSTANT i32 0
   //    %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
   //    %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)

llvm/trunk/test/CodeGen/AArch64/GlobalISel/opt-fold-cmn.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-unknown -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+#
+# Verify that we can fold G_SUB into G_ICMP when we have a pattern like this:
+#
+# x = G_SUB 0, y
+# G_ICMP intpred(something_safe) z, x
+#
+# Where "something_safe" is ne or eq.
+#
+# Tests whose names start with cmn_ should use ADDS for the G_ICMP. Tests whose
+# names start with no_cmn should use SUBS.
+#
+
+...
+---
+name:            cmn_s32_rhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: cmn_s32_rhs
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 0
+    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 1
+    ; CHECK: $wzr = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[MOVi32imm1]], [[MOVi32imm]], 1, implicit $nzcv
+    ; CHECK: $w0 = COPY [[CSELWr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = G_CONSTANT i32 0
+    %6:gpr(s32) = G_CONSTANT i32 1
+    %3:gpr(s32) = G_SUB %2, %1
+    %7:gpr(s32) = G_ICMP intpred(ne), %0(s32), %3
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s32) = G_SELECT %4(s1), %6, %2
+    $w0 = COPY %5(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            cmn_s32_lhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: cmn_s32_lhs
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 0
+    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 1
+    ; CHECK: $wzr = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[MOVi32imm1]], [[MOVi32imm]], 1, implicit $nzcv
+    ; CHECK: $w0 = COPY [[CSELWr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = G_CONSTANT i32 0
+    %6:gpr(s32) = G_CONSTANT i32 1
+    %3:gpr(s32) = G_SUB %2, %0
+    %7:gpr(s32) = G_ICMP intpred(ne), %3(s32), %1
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s32) = G_SELECT %4(s1), %6, %2
+    $w0 = COPY %5(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            no_cmn_s32_rhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: no_cmn_s32_rhs
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 0
+    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 1
+    ; CHECK: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MOVi32imm]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: $wzr = SUBSWrr [[COPY]], [[SUBSWrr]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[MOVi32imm1]], [[MOVi32imm]], 1, implicit $nzcv
+    ; CHECK: $w0 = COPY [[CSELWr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = G_CONSTANT i32 0
+    %6:gpr(s32) = G_CONSTANT i32 1
+    %3:gpr(s32) = G_SUB %2, %1
+    %7:gpr(s32) = G_ICMP intpred(slt), %0(s32), %3
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s32) = G_SELECT %4(s1), %6, %2
+    $w0 = COPY %5(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            no_cmn_s32_lhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: no_cmn_s32_lhs
+    ; CHECK: liveins: $w0, $w1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 0
+    ; CHECK: [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 1
+    ; CHECK: [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[MOVi32imm]], [[COPY]], implicit-def $nzcv
+    ; CHECK: $wzr = SUBSWrr [[SUBSWrr]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELWr:%[0-9]+]]:gpr32 = CSELWr [[MOVi32imm1]], [[MOVi32imm]], 1, implicit $nzcv
+    ; CHECK: $w0 = COPY [[CSELWr]]
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = G_CONSTANT i32 0
+    %6:gpr(s32) = G_CONSTANT i32 1
+    %3:gpr(s32) = G_SUB %2, %0
+    %7:gpr(s32) = G_ICMP intpred(slt), %3(s32), %1
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s32) = G_SELECT %4(s1), %6, %2
+    $w0 = COPY %5(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            cmn_s64_rhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: cmn_s64_rhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 0
+    ; CHECK: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+    ; CHECK: $xzr = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[MOVi64imm]], 1, implicit $nzcv
+    ; CHECK: $x0 = COPY [[CSELXr]]
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = G_CONSTANT i64 0
+    %6:gpr(s64) = G_CONSTANT i64 1
+    %3:gpr(s64) = G_SUB %2, %1
+    %7:gpr(s32) = G_ICMP intpred(ne), %0(s64), %3
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s64) = G_SELECT %4(s1), %6, %2
+    $x0 = COPY %5(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            cmn_s64_lhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: cmn_s64_lhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 0
+    ; CHECK: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+    ; CHECK: $xzr = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[MOVi64imm]], 1, implicit $nzcv
+    ; CHECK: $x0 = COPY [[CSELXr]]
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = G_CONSTANT i64 0
+    %6:gpr(s64) = G_CONSTANT i64 1
+    %3:gpr(s64) = G_SUB %2, %0
+    %7:gpr(s32) = G_ICMP intpred(ne), %3(s64), %1
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s64) = G_SELECT %4(s1), %6, %2
+    $x0 = COPY %5(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            no_cmn_s64_rhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: no_cmn_s64_rhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 0
+    ; CHECK: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+    ; CHECK: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr [[MOVi64imm]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: $xzr = SUBSXrr [[COPY]], [[SUBSXrr]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[MOVi64imm]], 1, implicit $nzcv
+    ; CHECK: $x0 = COPY [[CSELXr]]
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = G_CONSTANT i64 0
+    %6:gpr(s64) = G_CONSTANT i64 1
+    %3:gpr(s64) = G_SUB %2, %1
+    %7:gpr(s32) = G_ICMP intpred(slt), %0(s64), %3
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s64) = G_SELECT %4(s1), %6, %2
+    $x0 = COPY %5(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            no_cmn_s64_lhs
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: no_cmn_s64_lhs
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 0
+    ; CHECK: [[MOVi64imm1:%[0-9]+]]:gpr64 = MOVi64imm 1
+    ; CHECK: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr [[MOVi64imm]], [[COPY]], implicit-def $nzcv
+    ; CHECK: $xzr = SUBSXrr [[SUBSXrr]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
+    ; CHECK: $wzr = ANDSWri [[CSINCWr]], 0, implicit-def $nzcv
+    ; CHECK: [[CSELXr:%[0-9]+]]:gpr64 = CSELXr [[MOVi64imm1]], [[MOVi64imm]], 1, implicit $nzcv
+    ; CHECK: $x0 = COPY [[CSELXr]]
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s64) = G_CONSTANT i64 0
+    %6:gpr(s64) = G_CONSTANT i64 1
+    %3:gpr(s64) = G_SUB %2, %0
+    %7:gpr(s32) = G_ICMP intpred(slt), %3(s64), %1
+    %4:gpr(s1) = G_TRUNC %7(s32)
+    %5:gpr(s64) = G_SELECT %4(s1), %6, %2
+    $x0 = COPY %5(s64)
+    RET_ReallyLR implicit $x0
+
+...