ARM: Better codegen for 64-bit compares.

[pcc]

Summary

This introduces a custom lowering for ISD::SETCCE (introduced in r253572)
that allows us to emit a short code sequence for 64-bit compares.

Before:

push {r7, lr}
cmp r0, r2
mov.w r0, #0
mov.w r12, #0
it hs
movhs r0, #1
cmp r1, r3
it ge
movge.w r12, #1
it eq
moveq r12, r0
cmp.w r12, #0
bne .LBB1_2
@ BB#1: @ %bb1
bl f
pop {r7, pc}
.LBB1_2: @ %bb2
bl g
pop {r7, pc}

After:

push {r7, lr}
subs r0, r0, r2
sbcs.w r0, r1, r3
bge .LBB1_2
@ BB#1: @ %bb1
bl f
pop {r7, pc}
.LBB1_2: @ %bb2
bl g
pop {r7, pc}

Saves around 80KB in Chromium's libchrome.so.

Some notes on this patch:

• I don't much like the ARMISD::BRCOND and ARMISD::CMOV combines I introduced (nothing else needs them). However, they are necessary in order to avoid poor codegen, and they seem similar to existing combines in other backends (e.g. X86 combines (brcond (cmp (setcc Compare))) to (brcond Compare)).

• No support for Thumb-1. This is in principle possible, but we'd need to implement ARMISD::SUBE for Thumb-1.

[hans]

Looks reasonable to me, but I don't know enough about ARM to be a good reviewer here.

lib/Target/ARM/ARMISelLowering.cpp

4917

I'm too unfamiliar with ARM to follow this.

Should I understand it that Cmp.getValue(1) is something suitable to be moved into the flags register, and that makes it possible to do the CMOV below? (Which the BRCOND combine below will eliminate if we really want to branch on this rather than get a value?)

[pcc]

lib/Target/ARM/ARMISelLowering.cpp

4917

Yes, exactly. Cmp.getValue(1) represents the flags register itself; ARMISD::CMOV does not take a flags register as input so we need a CopyToReg to get the scheduling correct.

[pcc]

Ping.

[hans]

+Tilmann for size optimizations, +James for ARM

[jmolloy]

Hi Peter,

Sorry for the round trip time on this, I've only just become aware of it.

Some comments but it generally looks fine. Thanks!

James

lib/Target/ARM/ARMISelLowering.cpp

4916

This seems fishy to me. Shouldn't we be chaining to DAG.getRoot() (maybe?), rather than getEntryNode()?

Surely we want CPSR's value just before the SETCCE, not at the start of the BB?

10685

This is very hard to read due to the many isa<> and casts<>. Can we please hoist these out as a dyn_cast to reduce code complexity?

10760

Same here

[pcc]

• Use dyn_cast to improve readability of DAG matchers

[pcc]

lib/Target/ARM/ARMISelLowering.cpp

4916

The correct ordering is created by the edge from the SUBE to the copy. I don't think the chain edge is significant here.

Using the graph root wouldn't be correct, as that would create cycles in the graph.

I'm also not sure if using the entry node is entirely correct, but in my defense, there seems to be similar code in other backends [1].

[1] http://llvm-cs.pcc.me.uk/?q=getCopyToReg.*getEntryNode

[jmolloy]

Thanks Peter, LGTM

[• tilmann]

Great code size reduction, thanks for working on this!

LGTM

lib/Target/ARM/ARMISelLowering.h

public:
     MachineBasicBlock *
       EmitInstrWithCustomInserter(MachineInstr *MI,
                                   MachineBasicBlock *MBB) const override;
 
     void AdjustInstrPostInstrSelection(MachineInstr *MI,
                                        SDNode *Node) const override;
 
     SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const;
+    SDValue PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const;
     SDValue PerformCMOVToBFICombine(SDNode *N, SelectionDAG &DAG) const;
     SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
     bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override;
 
     /// allowsMisalignedMemoryAccesses - Returns true if the target allows
     /// unaligned memory accesses of the specified type. Returns whether it
     /// is "fast" by reference in the second argument.

lib/Target/ARM/ARMISelLowering.cpp

ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::SETCC,     MVT::f64, Expand);
   setOperationAction(ISD::SELECT,    MVT::i32, Custom);
   setOperationAction(ISD::SELECT,    MVT::f32, Custom);
   setOperationAction(ISD::SELECT,    MVT::f64, Custom);
   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
   setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
 
+  // Thumb-1 cannot currently select ARMISD::SUBE.
+  if (!Subtarget->isThumb1Only())
+    setOperationAction(ISD::SETCCE, MVT::i32, Custom);
+
   setOperationAction(ISD::BRCOND,    MVT::Other, Expand);
   setOperationAction(ISD::BR_CC,     MVT::i32,   Custom);
   setOperationAction(ISD::BR_CC,     MVT::f32,   Custom);
   setOperationAction(ISD::BR_CC,     MVT::f64,   Custom);
   setOperationAction(ISD::BR_JT,     MVT::Other, Custom);
 
   // We don't support sin/cos/fmod/copysign/pow
   setOperationAction(ISD::FSIN,      MVT::f64, Expand);
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
   Result = DAG.getSExtOrTrunc(Result, dl, VT);
 
   if (Invert)
     Result = DAG.getNOT(dl, Result, VT);
 
   return Result;
 }
 
+static SDValue LowerSETCCE(SDValue Op, SelectionDAG &DAG) {
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+  SDValue Carry = Op.getOperand(2);
+  SDValue Cond = Op.getOperand(3);
+  SDLoc DL(Op);
+
+  assert(LHS.getSimpleValueType().isInteger() && "SETCCE is integer only.");
+
+  assert(Carry.getOpcode() != ISD::CARRY_FALSE);
+  SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
+  SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
+
+  SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
+  SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
+  SDValue ARMcc = DAG.getConstant(
+      IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
+  SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
+  SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
+                                   Cmp.getValue(1), SDValue());
+  return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
+                     CCR, Chain.getValue(1));
+}
+
 /// isNEONModifiedImm - Check if the specified splat value corresponds to a
 /// valid vector constant for a NEON instruction with a "modified immediate"
 /// operand (e.g., VMOV).  If so, return the encoded value.
 static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
                                  unsigned SplatBitSize, SelectionDAG &DAG,
                                  SDLoc dl, EVT &VT, bool is128Bits,
                                  NEONModImmType type) {
   unsigned OpCmode, Imm;
SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::UREM:          return LowerREM(Op.getNode(), DAG);
   case ISD::SHL_PARTS:     return LowerShiftLeftParts(Op, DAG);
   case ISD::SRL_PARTS:
   case ISD::SRA_PARTS:     return LowerShiftRightParts(Op, DAG);
   case ISD::CTTZ:
   case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
   case ISD::CTPOP:         return LowerCTPOP(Op.getNode(), DAG, Subtarget);
   case ISD::SETCC:         return LowerVSETCC(Op, DAG);
+  case ISD::SETCCE:        return LowerSETCCE(Op, DAG);
   case ISD::ConstantFP:    return LowerConstantFP(Op, DAG, Subtarget);
   case ISD::BUILD_VECTOR:  return LowerBUILD_VECTOR(Op, DAG, Subtarget);
   case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
   case ISD::FLT_ROUNDS_:   return LowerFLT_ROUNDS_(Op, DAG);
   case ISD::MUL:           return LowerMUL(Op, DAG);
for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
     V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
                     // Confusingly, the operand is an *inverted* mask.
                     DAG.getConstant(~Mask, dl, VT));
   }
 
   return V;
 }
 
+/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
+SDValue
+ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
+  SDValue Cmp = N->getOperand(4);
+  if (Cmp.getOpcode() != ARMISD::CMPZ)
+    // Only looking at NE cases.
+    return SDValue();
+
+  EVT VT = N->getValueType(0);
+  SDLoc dl(N);
+  SDValue LHS = Cmp.getOperand(0);
+  SDValue RHS = Cmp.getOperand(1);
+  SDValue Chain = N->getOperand(0);
+  SDValue BB = N->getOperand(1);
+  SDValue ARMcc = N->getOperand(2);
+  ARMCC::CondCodes CC =
+    (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
+
+  // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0))
+  // -> (brcond Chain BB CC CPSR Cmp)
+  if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
+      LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
+      LHS->getOperand(0)->hasOneUse()) {
+    auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0));
+    auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1));
+    auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
+    auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
+    if ((LHS00C && LHS00C->getZExtValue() == 0) &&
+        (LHS01C && LHS01C->getZExtValue() == 1) &&
+        (LHS1C && LHS1C->getZExtValue() == 1) &&
+        (RHSC && RHSC->getZExtValue() == 0)) {
+      return DAG.getNode(
+          ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2),
+          LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4));
+    }
+  }
+
+  return SDValue();
+}
+
 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
 SDValue
 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   SDValue Cmp = N->getOperand(4);
   if (Cmp.getOpcode() != ARMISD::CMPZ)
     // Only looking at EQ and NE cases.
     return SDValue();
 
Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
                       N->getOperand(3), Cmp);
   } else if (CC == ARMCC::EQ && TrueVal == RHS) {
     SDValue ARMcc;
     SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
     Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
                       N->getOperand(3), NewCmp);
   }
 
+  // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0))
+  // -> (cmov F T CC CPSR Cmp)
+  if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) {
+    auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0));
+    auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
+    auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
+    if ((LHS0C && LHS0C->getZExtValue() == 0) &&
+        (LHS1C && LHS1C->getZExtValue() == 1) &&
+        (RHSC && RHSC->getZExtValue() == 0)) {
+      return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
+                         LHS->getOperand(2), LHS->getOperand(3),
+                         LHS->getOperand(4));
+    }
+  }
+
   if (Res.getNode()) {
     APInt KnownZero, KnownOne;
     DAG.computeKnownBits(SDValue(N,0), KnownZero, KnownOne);
     // Capture demanded bits information that would be otherwise lost.
     if (KnownZero == 0xfffffffe)
       Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
                         DAG.getValueType(MVT::i1));
     else if (KnownZero == 0xffffff00)
SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG);
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:        return PerformShiftCombine(N, DCI.DAG, Subtarget);
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget);
   case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG);
+  case ARMISD::BRCOND: return PerformBRCONDCombine(N, DCI.DAG);
   case ISD::LOAD:       return PerformLOADCombine(N, DCI);
   case ARMISD::VLD2DUP:
   case ARMISD::VLD3DUP:
   case ARMISD::VLD4DUP:
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
   case ISD::INTRINSIC_VOID:

test/CodeGen/ARM/atomic-64bit.ll

; CHECK-THUMB: dmb {{ish$}}
   store atomic i64 %val, i64* %ptr seq_cst, align 8
   ret void
 }
 
 define i64 @test10(i64* %ptr, i64 %val) {
 ; CHECK-LABEL: test10:
 ; CHECK: dmb {{ish$}}
 ; CHECK: ldrexd [[REG1:(r[0-9]?[02468])]], [[REG2:(r[0-9]?[13579])]]
-; CHECK: mov     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK: mov     [[CARRY_HI:[a-z0-9]+]], #0
 ; CHECK: mov     [[OUT_HI:[a-z0-9]+]], r2
-; CHECK-LE: cmp     [[REG1]], r1
-; CHECK-BE: cmp     [[REG2]], r2
-; CHECK: movwls  [[CARRY_LO]], #1
-; CHECK-LE: cmp     [[REG2]], r2
-; CHECK-BE: cmp     [[REG1]], r1
-; CHECK: movwle  [[CARRY_HI]], #1
-; CHECK: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK: cmp     [[CARRY_HI]], #0
+; CHECK-LE: subs {{[^,]+}}, r1, [[REG1]]
+; CHECK-BE: subs {{[^,]+}}, r2, [[REG2]]
+; CHECK-LE: sbcs {{[^,]+}}, r2, [[REG2]]
+; CHECK-BE: sbcs {{[^,]+}}, r1, [[REG1]]
+; CHECK: mov     [[CMP:[a-z0-9]+]], #0
+; CHECK: movwge  [[CMP]], #1
+; CHECK: cmp     [[CMP]], #0
 ; CHECK: movne   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
 ; CHECK: movne   [[OUT_LO]], [[REG1]]
-; CHECK: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
 ; CHECK: dmb {{ish$}}
 
 ; CHECK-THUMB-LABEL: test10:
 ; CHECK-THUMB: dmb {{ish$}}
 ; CHECK-THUMB: ldrexd [[REG1:[a-z0-9]+]], [[REG2:[a-z0-9]+]]
-; CHECK-THUMB: mov.w     [[CARRY_LO:[a-z0-9]+|lr]], #0
-; CHECK-THUMB: movs     [[CARRY_HI:[a-z0-9]+|lr]], #0
-; CHECK-THUMB-LE: cmp     [[REG1]], r2
-; CHECK-THUMB-BE: cmp     [[REG2]], r3
-; CHECK-THUMB: movls.w  [[CARRY_LO]], #1
-; CHECK-THUMB-LE: cmp     [[REG2]], r3
-; CHECK-THUMB-BE: cmp     [[REG1]], r2
-; CHECK-THUMB: movle  [[CARRY_HI]], #1
-; CHECK-THUMB: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK-THUMB: mov     [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: cmp     [[CARRY_HI]], #0
-; CHECK-THUMB: mov     [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB: mov      [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB-LE: subs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
+; CHECK-THUMB: movge.w   [[CMP]], #1
+; CHECK-THUMB: cmp.w     [[CMP]], #0
+; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
 ; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
 ; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
-; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
 ; CHECK-THUMB: dmb {{ish$}}
 
   %r = atomicrmw min i64* %ptr, i64 %val seq_cst
   ret i64 %r
 }
 
 define i64 @test11(i64* %ptr, i64 %val) {
 ; CHECK-LABEL: test11:
 ; CHECK: dmb {{ish$}}
 ; CHECK: ldrexd [[REG1:(r[0-9]?[02468])]], [[REG2:(r[0-9]?[13579])]]
-; CHECK: mov     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK: mov     [[CARRY_HI:[a-z0-9]+]], #0
 ; CHECK: mov     [[OUT_HI:[a-z0-9]+]], r2
-; CHECK-LE: cmp     [[REG1]], r1
-; CHECK-BE: cmp     [[REG2]], r2
-; CHECK: movwls  [[CARRY_LO]], #1
-; CHECK-LE: cmp     [[REG2]], r2
-; CHECK-BE: cmp     [[REG1]], r1
-; CHECK: movwls  [[CARRY_HI]], #1
-; CHECK: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK: cmp     [[CARRY_HI]], #0
+; CHECK-LE: subs    {{[^,]+}}, r1, [[REG1]]
+; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
+; CHECK: mov     [[CMP:[a-z0-9]+]], #0
+; CHECK: movwhs  [[CMP]], #1
+; CHECK: cmp     [[CMP]], #0
 ; CHECK: movne   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
 ; CHECK: movne   [[OUT_LO]], [[REG1]]
-; CHECK: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
 ; CHECK: dmb {{ish$}}
 
 ; CHECK-THUMB-LABEL: test11:
 ; CHECK-THUMB: dmb {{ish$}}
 ; CHECK-THUMB: ldrexd [[REG1:[a-z0-9]+]], [[REG2:[a-z0-9]+]]
-; CHECK-THUMB: mov.w     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK-THUMB: movs     [[CARRY_HI:[a-z0-9]+]], #0
-; CHECK-THUMB-LE: cmp     [[REG1]], r2
-; CHECK-THUMB-BE: cmp     [[REG2]], r3
-; CHECK-THUMB: movls.w  [[CARRY_LO]], #1
-; CHECK-THUMB-LE: cmp     [[REG2]], r3
-; CHECK-THUMB-BE: cmp     [[REG1]], r2
-; CHECK-THUMB: movls  [[CARRY_HI]], #1
-; CHECK-THUMB: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK-THUMB: mov     [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: cmp     [[CARRY_HI]], #0
-; CHECK-THUMB: mov     [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB: mov      [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB-LE: subs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
+; CHECK-THUMB: movhs.w   [[CMP]], #1
+; CHECK-THUMB: cmp.w     [[CMP]], #0
+; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
 ; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
 ; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
-; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
 ; CHECK-THUMB: dmb {{ish$}}
 
   %r = atomicrmw umin i64* %ptr, i64 %val seq_cst
   ret i64 %r
 }
 
 define i64 @test12(i64* %ptr, i64 %val) {
 ; CHECK-LABEL: test12:
 ; CHECK: dmb {{ish$}}
 ; CHECK: ldrexd [[REG1:(r[0-9]?[02468])]], [[REG2:(r[0-9]?[13579])]]
-; CHECK: mov     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK: mov     [[CARRY_HI:[a-z0-9]+]], #0
 ; CHECK: mov     [[OUT_HI:[a-z0-9]+]], r2
-; CHECK-LE: cmp     [[REG1]], r1
-; CHECK-BE: cmp     [[REG2]], r2
-; CHECK: movwhi  [[CARRY_LO]], #1
-; CHECK-LE: cmp     [[REG2]], r2
-; CHECK-BE: cmp     [[REG1]], r1
-; CHECK: movwgt  [[CARRY_HI]], #1
-; CHECK: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK: cmp     [[CARRY_HI]], #0
+; CHECK-LE: subs    {{[^,]+}}, r1, [[REG1]]
+; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
+; CHECK: mov     [[CMP:[a-z0-9]+]], #0
+; CHECK: movwlt  [[CMP]], #1
+; CHECK: cmp     [[CMP]], #0
 ; CHECK: movne   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
 ; CHECK: movne   [[OUT_LO]], [[REG1]]
-; CHECK: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
 ; CHECK: dmb {{ish$}}
 
 ; CHECK-THUMB-LABEL: test12:
 ; CHECK-THUMB: dmb {{ish$}}
 ; CHECK-THUMB: ldrexd [[REG1:[a-z0-9]+]], [[REG2:[a-z0-9]+]]
-; CHECK-THUMB: mov.w     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK-THUMB: movs     [[CARRY_HI:[a-z0-9]+]], #0
-; CHECK-THUMB-LE: cmp     [[REG1]], r2
-; CHECK-THUMB-BE: cmp     [[REG2]], r3
-; CHECK-THUMB: movhi.w  [[CARRY_LO]], #1
-; CHECK-THUMB-LE: cmp     [[REG2]], r3
-; CHECK-THUMB-BE: cmp     [[REG1]], r2
-; CHECK-THUMB: movgt  [[CARRY_HI]], #1
-; CHECK-THUMB: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK-THUMB: mov     [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: cmp     [[CARRY_HI]], #0
-; CHECK-THUMB: mov     [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB: mov      [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB-LE: subs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
+; CHECK-THUMB: movlt.w   [[CMP]], #1
+; CHECK-THUMB: cmp.w     [[CMP]], #0
+; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
 ; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
 ; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
-; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
 ; CHECK-THUMB: dmb {{ish$}}
 
   %r = atomicrmw max i64* %ptr, i64 %val seq_cst
   ret i64 %r
 }
 
 define i64 @test13(i64* %ptr, i64 %val) {
 ; CHECK-LABEL: test13:
 ; CHECK: dmb {{ish$}}
 ; CHECK: ldrexd [[REG1:(r[0-9]?[02468])]], [[REG2:(r[0-9]?[13579])]]
-; CHECK: mov     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK: mov     [[CARRY_HI:[a-z0-9]+]], #0
 ; CHECK: mov     [[OUT_HI:[a-z0-9]+]], r2
-; CHECK-LE: cmp     [[REG1]], r1
-; CHECK-BE: cmp     [[REG2]], r2
-; CHECK: movwhi  [[CARRY_LO]], #1
-; CHECK-LE: cmp     [[REG2]], r2
-; CHECK-BE: cmp     [[REG1]], r1
-; CHECK: movwhi  [[CARRY_HI]], #1
-; CHECK: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK: cmp     [[CARRY_HI]], #0
+; CHECK-LE: subs    {{[^,]+}}, r1, [[REG1]]
+; CHECK-BE: subs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-LE: sbcs    {{[^,]+}}, r2, [[REG2]]
+; CHECK-BE: sbcs    {{[^,]+}}, r1, [[REG1]]
+; CHECK: mov     [[CMP:[a-z0-9]+]], #0
+; CHECK: movwlo  [[CMP]], #1
+; CHECK: cmp     [[CMP]], #0
 ; CHECK: movne   [[OUT_HI]], [[REG2]]
 ; CHECK: mov     [[OUT_LO:[a-z0-9]+]], r1
 ; CHECK: movne   [[OUT_LO]], [[REG1]]
-; CHECK: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK: cmp
 ; CHECK: bne
 ; CHECK: dmb {{ish$}}
 
 ; CHECK-THUMB-LABEL: test13:
 ; CHECK-THUMB: dmb {{ish$}}
 ; CHECK-THUMB: ldrexd [[REG1:[a-z0-9]+]], [[REG2:[a-z0-9]+]]
-; CHECK-THUMB: mov.w     [[CARRY_LO:[a-z0-9]+]], #0
-; CHECK-THUMB: movs     [[CARRY_HI:[a-z0-9]+]], #0
-; CHECK-THUMB-LE: cmp     [[REG1]], r2
-; CHECK-THUMB-BE: cmp     [[REG2]], r3
-; CHECK-THUMB: movhi.w  [[CARRY_LO]], #1
-; CHECK-THUMB-LE: cmp     [[REG2]], r3
-; CHECK-THUMB-BE: cmp     [[REG1]], r2
-; CHECK-THUMB: movhi  [[CARRY_HI]], #1
-; CHECK-THUMB: moveq   [[CARRY_HI]], [[CARRY_LO]]
-; CHECK-THUMB: mov     [[OUT_HI:[a-z0-9]+]], r3
-; CHECK-THUMB: cmp     [[CARRY_HI]], #0
-; CHECK-THUMB: mov     [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB: mov      [[OUT_LO:[a-z0-9]+]], r2
+; CHECK-THUMB-LE: subs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB-BE: subs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-LE: sbcs.w {{[^,]+}}, r3, [[REG2]]
+; CHECK-THUMB-BE: sbcs.w {{[^,]+}}, r2, [[REG1]]
+; CHECK-THUMB: mov.w     [[CMP:[a-z0-9]+]], #0
+; CHECK-THUMB: movlo.w   [[CMP]], #1
+; CHECK-THUMB: cmp.w     [[CMP]], #0
+; CHECK-THUMB: mov       [[OUT_HI:[a-z0-9]+]], r3
 ; CHECK-THUMB: movne   [[OUT_HI]], [[REG2]]
 ; CHECK-THUMB: movne   [[OUT_LO]], [[REG1]]
-; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[REG3]], [[REG4]]
+; CHECK-THUMB: strexd {{[a-z0-9]+}}, [[OUT_LO]], [[OUT_HI]]
 ; CHECK-THUMB: cmp
 ; CHECK-THUMB: bne
 ; CHECK-THUMB: dmb {{ish$}}
   %r = atomicrmw umax i64* %ptr, i64 %val seq_cst
   ret i64 %r
 }
 

test/CodeGen/ARM/atomic-ops-v8.ll

 ; CHECK-NOT: mcr
 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
 ; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
-; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
-; CHECK-ARM: mov [[HICARRY:r[0-9]+|lr]], #0
-; CHECK-ARM-LE: cmp [[OLD1]], r0
-; CHECK-ARM-LE: movwls [[LOCARRY]], #1
-; CHECK-ARM-LE: cmp [[OLD2]], r1
-; CHECK-ARM-LE: movwle [[HICARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD2]], r1
-; CHECK-ARM-BE: movwls [[LOCARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD1]], r0
-; CHECK-ARM-BE: movwle [[HICARRY]], #1
-; CHECK-ARM: moveq [[HICARRY]], [[LOCARRY]]
-; CHECK-ARM: cmp [[HICARRY]], #0
 ; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
+; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
+; CHECK-ARM: movwge [[CMP:r[0-9]+|lr]], #1
+; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
 ; CHECK-ARM: movne [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
 ; CHECK-ARM: movne [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
 ; CHECK-NOT: dmb

 ; CHECK-NOT: mcr
 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
 ; CHECK: ldrexd [[OLD1:r[0-9]+]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
-; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
-; CHECK-ARM: mov [[HICARRY:r[0-9]+|lr]], #0
-; CHECK-ARM-LE: cmp [[OLD1]], r0
-; CHECK-ARM-LE: movwhi [[LOCARRY]], #1
-; CHECK-ARM-LE: cmp [[OLD2]], r1
-; CHECK-ARM-LE: movwgt [[HICARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD2]], r1
-; CHECK-ARM-BE: movwhi [[LOCARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD1]], r0
-; CHECK-ARM-BE: movwgt [[HICARRY]], #1
-; CHECK-ARM: moveq [[HICARRY]], [[LOCARRY]]
-; CHECK-ARM: cmp [[HICARRY]], #0
 ; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
+; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
+; CHECK-ARM: movwlt [[CMP:r[0-9]+|lr]], #1
+; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
 ; CHECK-ARM: movne [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
 ; CHECK-ARM: movne [[MINLO]], [[OLD1]]
 ; CHECK-ARM: strexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: strexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
 ; CHECK-NOT: dmb

 ; CHECK-NOT: mcr
 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
 ; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
-; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
-; CHECK-ARM: mov [[HICARRY:r[0-9]+|lr]], #0
-; CHECK-ARM-LE: cmp [[OLD1]], r0
-; CHECK-ARM-LE: movwls [[LOCARRY]], #1
-; CHECK-ARM-LE: cmp [[OLD2]], r1
-; CHECK-ARM-LE: movwls [[HICARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD2]], r1
-; CHECK-ARM-BE: movwls [[LOCARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD1]], r0
-; CHECK-ARM-BE: movwls [[HICARRY]], #1
-; CHECK-ARM: moveq [[HICARRY]], [[LOCARRY]]
-; CHECK-ARM: cmp [[HICARRY]], #0
 ; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
+; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
+; CHECK-ARM: movwhs [[CMP:r[0-9]+|lr]], #1
+; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
 ; CHECK-ARM: movne [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
 ; CHECK-ARM: movne [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
 ; CHECK-NOT: dmb

 ; CHECK-NOT: mcr
 ; CHECK: movw r[[ADDR:[0-9]+]], :lower16:var64
 ; CHECK: movt r[[ADDR]], :upper16:var64
 
 ; CHECK: .LBB{{[0-9]+}}_1:
 ; CHECK: ldaexd [[OLD1:r[0-9]+|lr]], [[OLD2:r[0-9]+|lr]], [r[[ADDR]]]
   ; r0, r1 below is a reasonable guess but could change: it certainly comes into the
   ; function there.
-; CHECK-ARM: mov [[LOCARRY:r[0-9]+|lr]], #0
-; CHECK-ARM: mov [[HICARRY:r[0-9]+|lr]], #0
-; CHECK-ARM-LE: cmp [[OLD1]], r0
-; CHECK-ARM-LE: movwhi [[LOCARRY]], #1
-; CHECK-ARM-LE: cmp [[OLD2]], r1
-; CHECK-ARM-LE: movwhi [[HICARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD2]], r1
-; CHECK-ARM-BE: movwhi [[LOCARRY]], #1
-; CHECK-ARM-BE: cmp [[OLD1]], r0
-; CHECK-ARM-BE: movwhi [[HICARRY]], #1
-; CHECK-ARM: moveq [[HICARRY]], [[LOCARRY]]
-; CHECK-ARM: cmp [[HICARRY]], #0
 ; CHECK-ARM: mov [[MINHI:r[0-9]+]], r1
+; CHECK-ARM-LE: subs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM-LE: sbcs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: subs {{[^,]+}}, r1, [[OLD2]]
+; CHECK-ARM-BE: sbcs {{[^,]+}}, r0, [[OLD1]]
+; CHECK-ARM: mov [[CMP:r[0-9]+|lr]], #0
+; CHECK-ARM: movwlo [[CMP:r[0-9]+|lr]], #1
+; CHECK-ARM: cmp [[CMP:r[0-9]+|lr]], #0
 ; CHECK-ARM: movne [[MINHI]], [[OLD2]]
 ; CHECK-ARM: mov [[MINLO:r[0-9]+]], r0
 ; CHECK-ARM: movne [[MINLO]], [[OLD1]]
 ; CHECK-ARM: stlexd [[STATUS:r[0-9]+]], [[MINLO]], [[MINHI]], [r[[ADDR]]]
 ; CHECK-THUMB: stlexd [[STATUS:r[0-9]+]], {{r[0-9]+}}, {{r[0-9]+}}, [r[[ADDR]]]
 ; CHECK-NEXT: cmp [[STATUS]], #0
 ; CHECK-NEXT: bne .LBB{{[0-9]+}}_1
 ; CHECK-NOT: dmb

test/CodeGen/ARM/wide-compares.ll

+; RUN: llc -mtriple=armv7-unknown-linux < %s | FileCheck --check-prefix=CHECK --check-prefix=CHECK-ARM %s
+; RUN: llc -mtriple=thumbv6-unknown-linux < %s | FileCheck --check-prefix=CHECK-THUMB1 %s
+; RUN: llc -mtriple=thumbv7-unknown-linux < %s | FileCheck --check-prefix=CHECK --check-prefix=CHECK-THUMB2 %s
+
+; CHECK-THUMB1-NOT: sbc
+
+; CHECK-LABEL: test_slt1:
+define i32 @test_slt1(i64 %a, i64 %b) {
+entry:
+  ; CHECK-ARM: subs {{[^,]+}}, r0, r2
+  ; CHECK-ARM: mov [[TMP:[0-9a-z]+]], #2
+  ; CHECK-ARM: sbcs {{[^,]+}}, r1, r3
+  ; CHECK-ARM: movwlt [[TMP]], #1
+  ; CHECK-ARM: mov r0, [[TMP]]
+  ; CHECK-ARM: bx lr
+  ; CHECK-THUMB2: subs {{[^,]+}}, r0, r2
+  ; CHECK-THUMB2: mov.w [[TMP:[0-9a-z]+]], #2
+  ; CHECK-THUMB2: sbcs.w {{[^,]+}}, r1, r3
+  ; CHECK-THUMB2: it lt
+  ; CHECK-THUMB2: movlt.w [[TMP]], #1
+  ; CHECK-THUMB2: mov r0, [[TMP]]
+  ; CHECK-THUMB2: bx lr
+  %cmp = icmp slt i64 %a, %b
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  ret i32 1
+bb2:
+  ret i32 2
+}
+
+; CHECK-LABEL: test_slt2:
+define void @test_slt2(i64 %a, i64 %b) {
+entry:
+  %cmp = icmp slt i64 %a, %b
+  ; CHECK-ARM: subs {{[^,]+}}, r0, r2
+  ; CHECK-ARM: sbcs {{[^,]+}}, r1, r3
+  ; CHECK-THUMB2: subs {{[^,]+}}, r0, r2
+  ; CHECK-THUMB2: sbcs.w {{[^,]+}}, r1, r3
+  ; CHECK: bge [[BB2:\.[0-9A-Za-z_]+]]
+  br i1 %cmp, label %bb1, label %bb2
+bb1:
+  call void @f()
+  ret void
+bb2:
+  ; CHECK: [[BB2]]:
+  ; CHECK-NEXT: bl g
+  call void @g()
+  ret void
+}
+
+declare void @f()
+declare void @g()