[AArch64] Add lowerings for {ADD,SUB}CARRY and S{ADD,SUB}O_CARRY

[paulwalker-arm]

One nit plus a suggestion but otherwise the patch looks good to me. There are several regressions though, presumably caused by my request to split the original patch up. I'm not qualified to say how important these test cases are from a performance point of view but given the number of regressions I think it's worth creating a patch series (via Depends On) so the patches can be reviewed in isolation but pushed together. Sorry if this is messing you around.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

3365

Unnecessary {} for single line body.

3369–3390

Up to you but I think these are better passed in a parameters so you end up with:

case ISD::ADDCARRY:
  return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, false /*unsigned*/);
case ISD::SUBCARRY:
  return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, false /*unsigned*/);
case ISD::SADDO_CARRY:
  return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, true /*signed*/);
case ISD::SSUBO_CARRY:
  return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, true /*signed*/);

[dmgreen]

Yeah, we would need to make sure that the regressions are fixed either soon after, or preferably before this. 128 bit arithmetic is fairly rare, but can come up from certain disciplines. (Sometimes it's possible to test the optimizations independently with other code patterns. Sometimes that's not possible though).

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

3365

Is this just checking for i32 or i64? It might be better to be explicit about that.
Also, I see this code is elsewhere too, but is it possible for types to be illegal here?

3392

OpLhs -> OpLHS

[Kmeakin]

Update revision according to feedback

• Remove unecessary {} for single-line body.
• Pass in Opcode and IsSigned as parameters
• Explicitly check for MVT::i32 and MVT::i64
• Capitalisation of OpLHS and OpRHS

[Kmeakin]

Fix indentation

[efriedma]

According to https://github.com/llvm/llvm-project/issues/55253, this causes a miscompile.

[kazu]

In D123322#3492096, @efriedma wrote:

According to https://github.com/llvm/llvm-project/issues/55253, this causes a miscompile.

I posted a fix: https://reviews.llvm.org/D124976

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::SSUBO, MVT::i64, Custom);
   setOperationAction(ISD::USUBO, MVT::i32, Custom);
   setOperationAction(ISD::USUBO, MVT::i64, Custom);
   setOperationAction(ISD::SMULO, MVT::i32, Custom);
   setOperationAction(ISD::SMULO, MVT::i64, Custom);
   setOperationAction(ISD::UMULO, MVT::i32, Custom);
   setOperationAction(ISD::UMULO, MVT::i64, Custom);
 
+  setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
+  setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
+  setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
+  setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
+  setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
+  setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom);
+  setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
+  setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom);
+
   setOperationAction(ISD::FSIN, MVT::f32, Expand);
   setOperationAction(ISD::FSIN, MVT::f64, Expand);
   setOperationAction(ISD::FCOS, MVT::f32, Expand);
   setOperationAction(ISD::FCOS, MVT::f64, Expand);
   setOperationAction(ISD::FPOW, MVT::f32, Expand);
   setOperationAction(ISD::FPOW, MVT::f64, Expand);
   setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
   setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
   }
 
   if (!ExtraOp)
     return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
   return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
                      Op.getOperand(2));
 }
 
+// Sets 'C' bit of NZCV to 0 if value is 0, else sets 'C' bit to 1
+static SDValue valueToCarryFlag(SDValue Value, SelectionDAG &DAG) {
+  SDLoc DL(Value);
+  SDValue One = DAG.getConstant(1, DL, Value.getValueType());
+  SDValue Cmp =
+      DAG.getNode(AArch64ISD::SUBS, DL,
+                  DAG.getVTList(Value.getValueType(), MVT::Glue), Value, One);
+  return Cmp.getValue(1);
+}
+
+// Value is 1 if 'C' bit of NZCV is 1, else 0
+static SDValue carryFlagToValue(SDValue Flag, EVT VT, SelectionDAG &DAG) {
+  assert(Flag.getResNo() == 1);
+  SDLoc DL(Flag);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue One = DAG.getConstant(1, DL, VT);
+  SDValue CC = DAG.getConstant(AArch64CC::HS, DL, MVT::i32);
+  return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag);
+}
+
+// Value is 1 if 'V' bit of NZCV is 1, else 0
+static SDValue overflowFlagToValue(SDValue Flag, EVT VT, SelectionDAG &DAG) {
+  assert(Flag.getResNo() == 1);
+  SDLoc DL(Flag);
+  SDValue Zero = DAG.getConstant(0, DL, VT);
+  SDValue One = DAG.getConstant(1, DL, VT);
+  SDValue CC = DAG.getConstant(AArch64CC::VS, DL, MVT::i32);
+  return DAG.getNode(AArch64ISD::CSEL, DL, VT, One, Zero, CC, Flag);
+}
+
+// This lowering is inefficient, but it will get cleaned up by
+// `performAddSubCombine`
+static SDValue lowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG, unsigned Opcode,
+                                bool IsSigned) {
+  EVT VT0 = Op.getValue(0).getValueType();
+  EVT VT1 = Op.getValue(1).getValueType();
+
+  if (VT0 != MVT::i32 && VT0 != MVT::i64)
+    return SDValue();
+
+  SDValue OpLHS = Op.getOperand(0);
+  SDValue OpRHS = Op.getOperand(1);
+  SDValue OpCarryIn = valueToCarryFlag(Op.getOperand(2), DAG);
+
+  SDLoc DL(Op);
+  SDVTList VTs = DAG.getVTList(VT0, VT1);
+
+  SDValue Sum = DAG.getNode(Opcode, DL, DAG.getVTList(VT0, MVT::Glue), OpLHS,
+                            OpRHS, OpCarryIn);
+
+  SDValue OutFlag = IsSigned ? overflowFlagToValue(Sum.getValue(1), VT1, DAG)
+                             : carryFlagToValue(Sum.getValue(1), VT1, DAG);
+
+  return DAG.getNode(ISD::MERGE_VALUES, DL, VTs, Sum, OutFlag);
+}
+
 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
   // Let legalize expand this if it isn't a legal type yet.
   if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
     return SDValue();
 
   SDLoc dl(Op);
   AArch64CC::CondCode CC;
   // The actual operation that sets the overflow or carry flag.
   SDValue Value, Overflow;
   std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
 
   // We use 0 and 1 as false and true values.
   SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
   SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
 
   // We use an inverted condition, because the conditional select is inverted
   // too. This will allow it to be selected to a single instruction:
case ISD::VACOPY:
     return LowerVACOPY(Op, DAG);
   case ISD::VAARG:
     return LowerVAARG(Op, DAG);
   case ISD::ADDC:
   case ISD::ADDE:
   case ISD::SUBC:
   case ISD::SUBE:
     return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
+  case ISD::ADDCARRY:
+    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, false /*unsigned*/);
+  case ISD::SUBCARRY:
+    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, false /*unsigned*/);
+  case ISD::SADDO_CARRY:
+    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::ADCS, true /*signed*/);
+  case ISD::SSUBO_CARRY:
+    return lowerADDSUBCARRY(Op, DAG, AArch64ISD::SBCS, true /*signed*/);
   case ISD::SADDO:
   case ISD::UADDO:
   case ISD::SSUBO:
   case ISD::USUBO:
   case ISD::SMULO:
   case ISD::UMULO:
     return LowerXALUO(Op, DAG);
   case ISD::FADD:

llvm/test/CodeGen/AArch64/adc.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=arm64-apple-ios7.0 | FileCheck --check-prefix=CHECK-LE %s
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64_be-none-linux-gnu | FileCheck --check-prefix=CHECK-BE %s
 
 define i128 @test_simple(i128 %a, i128 %b, i128 %c) {
 ; CHECK-LE-LABEL: test_simple:
 ; CHECK-LE:       ; %bb.0:
 ; CHECK-LE-NEXT:    adds x8, x0, x2
+; CHECK-LE-NEXT:    cset w9, hs
+; CHECK-LE-NEXT:    cmp w9, #1
 ; CHECK-LE-NEXT:    adcs x9, x1, x3
 ; CHECK-LE-NEXT:    subs x0, x8, x4
+; CHECK-LE-NEXT:    cset w8, lo
+; CHECK-LE-NEXT:    cmp w8, #1
 ; CHECK-LE-NEXT:    sbcs x1, x9, x5
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_simple:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    adds x8, x1, x3
+; CHECK-BE-NEXT:    cset w9, hs
+; CHECK-BE-NEXT:    cmp w9, #1
 ; CHECK-BE-NEXT:    adcs x9, x0, x2
 ; CHECK-BE-NEXT:    subs x1, x8, x5
+; CHECK-BE-NEXT:    cset w8, lo
+; CHECK-BE-NEXT:    cmp w8, #1
 ; CHECK-BE-NEXT:    sbcs x0, x9, x4
 ; CHECK-BE-NEXT:    ret
 
   %valadd = add i128 %a, %b
 
   %valsub = sub i128 %valadd, %c
 
   ret i128 %valsub
 }
 
 define i128 @test_imm(i128 %a) {
 ; CHECK-LE-LABEL: test_imm:
 ; CHECK-LE:       ; %bb.0:
 ; CHECK-LE-NEXT:    adds x0, x0, #12
+; CHECK-LE-NEXT:    cset w8, hs
+; CHECK-LE-NEXT:    cmp w8, #1
 ; CHECK-LE-NEXT:    adcs x1, x1, xzr
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_imm:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    adds x1, x1, #12
+; CHECK-BE-NEXT:    cset w8, hs
+; CHECK-BE-NEXT:    cmp w8, #1
 ; CHECK-BE-NEXT:    adcs x0, x0, xzr
 ; CHECK-BE-NEXT:    ret
 
   %val = add i128 %a, 12
 
   ret i128 %val
 }
 
 define i128 @test_shifted(i128 %a, i128 %b) {
 ; CHECK-LE-LABEL: test_shifted:
 ; CHECK-LE:       ; %bb.0:
-; CHECK-LE-NEXT:    extr x8, x3, x2, #19
 ; CHECK-LE-NEXT:    adds x0, x0, x2, lsl #45
+; CHECK-LE-NEXT:    extr x8, x3, x2, #19
+; CHECK-LE-NEXT:    cset w9, hs
+; CHECK-LE-NEXT:    cmp w9, #1
 ; CHECK-LE-NEXT:    adcs x1, x1, x8
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_shifted:
 ; CHECK-BE:       // %bb.0:
-; CHECK-BE-NEXT:    extr x8, x2, x3, #19
 ; CHECK-BE-NEXT:    adds x1, x1, x3, lsl #45
+; CHECK-BE-NEXT:    extr x8, x2, x3, #19
+; CHECK-BE-NEXT:    cset w9, hs
+; CHECK-BE-NEXT:    cmp w9, #1
 ; CHECK-BE-NEXT:    adcs x0, x0, x8
 ; CHECK-BE-NEXT:    ret
 
   %rhs = shl i128 %b, 45
 
   %val = add i128 %a, %rhs
 
   ret i128 %val
 }
 
 define i128 @test_extended(i128 %a, i16 %b) {
 ; CHECK-LE-LABEL: test_extended:
 ; CHECK-LE:       ; %bb.0:
 ; CHECK-LE-NEXT:    ; kill: def $w2 killed $w2 def $x2
 ; CHECK-LE-NEXT:    sxth x8, w2
 ; CHECK-LE-NEXT:    adds x0, x0, w2, sxth #3
 ; CHECK-LE-NEXT:    asr x9, x8, #63
 ; CHECK-LE-NEXT:    extr x8, x9, x8, #61
+; CHECK-LE-NEXT:    cset w9, hs
+; CHECK-LE-NEXT:    cmp w9, #1
 ; CHECK-LE-NEXT:    adcs x1, x1, x8
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: test_extended:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    // kill: def $w2 killed $w2 def $x2
 ; CHECK-BE-NEXT:    sxth x8, w2
 ; CHECK-BE-NEXT:    adds x1, x1, w2, sxth #3
 ; CHECK-BE-NEXT:    asr x9, x8, #63
 ; CHECK-BE-NEXT:    extr x8, x9, x8, #61
+; CHECK-BE-NEXT:    cset w9, hs
+; CHECK-BE-NEXT:    cmp w9, #1
 ; CHECK-BE-NEXT:    adcs x0, x0, x8
 ; CHECK-BE-NEXT:    ret
 
   %ext = sext i16 %b to i128
   %rhs = shl i128 %ext, 3
 
   %val = add i128 %a, %rhs
 
   ret i128 %val
 }

llvm/test/CodeGen/AArch64/addcarry-crash.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s | FileCheck %s
 target triple = "arm64-apple-ios7.0"
 
 define i64 @foo(i64* nocapture readonly %ptr, i64 %a, i64 %b, i64 %c) local_unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    ldr w8, [x0, #4]
 ; CHECK-NEXT:    lsr x9, x1, #32
 ; CHECK-NEXT:    cmn x3, x2
 ; CHECK-NEXT:    mul x8, x8, x9
-; CHECK-NEXT:    cinc x0, x8, hs
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    adcs x0, x8, xzr
 ; CHECK-NEXT:    ret
 entry:
   %0 = lshr i64 %a, 32
   %1 = load i64, i64* %ptr, align 8
   %2 = lshr i64 %1, 32
   %3 = mul nuw i64 %2, %0
   %4 = add i64 %c, %b
   %5 = icmp ult i64 %4, %c
   %6 = zext i1 %5 to i64
   %7 = add i64 %3, %6
   ret i64 %7
 }
 
 attributes #0 = { norecurse nounwind readonly }

llvm/test/CodeGen/AArch64/arm64-atomic-128.ll

 
 define void @fetch_and_add(i128* %p, i128 %bits) {
 ; CHECK-LABEL: fetch_and_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:  .LBB6_1: // %atomicrmw.start
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldaxp x9, x8, [x0]
 ; CHECK-NEXT:    adds x10, x9, x2
+; CHECK-NEXT:    cset w11, hs
+; CHECK-NEXT:    cmp w11, #1
 ; CHECK-NEXT:    adcs x11, x8, x3
 ; CHECK-NEXT:    stlxp w12, x10, x11, [x0]
 ; CHECK-NEXT:    cbnz w12, .LBB6_1
 ; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
 ; CHECK-NEXT:    adrp x10, :got:var
 ; CHECK-NEXT:    ldr x10, [x10, :got_lo12:var]
 ; CHECK-NEXT:    stp x9, x8, [x10]
 ; CHECK-NEXT:    ret
 
   %val = atomicrmw add i128* %p, i128 %bits seq_cst
   store i128 %val, i128* @var, align 16
   ret void
 }
 
 define void @fetch_and_sub(i128* %p, i128 %bits) {
 ; CHECK-LABEL: fetch_and_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:  .LBB7_1: // %atomicrmw.start
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldaxp x9, x8, [x0]
 ; CHECK-NEXT:    subs x10, x9, x2
+; CHECK-NEXT:    cset w11, lo
+; CHECK-NEXT:    cmp w11, #1
 ; CHECK-NEXT:    sbcs x11, x8, x3
 ; CHECK-NEXT:    stlxp w12, x10, x11, [x0]
 ; CHECK-NEXT:    cbnz w12, .LBB7_1
 ; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
 ; CHECK-NEXT:    adrp x10, :got:var
 ; CHECK-NEXT:    ldr x10, [x10, :got_lo12:var]
 ; CHECK-NEXT:    stp x9, x8, [x10]
 ; CHECK-NEXT:    ret

llvm/test/CodeGen/AArch64/arm64-vabs.ll

; GISEL-NEXT:    ret
   ret <2 x i64> %absel
 }
 
 
 define <2 x i128> @uabd_i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: uabd_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fmov x9, d0
-; CHECK-NEXT:    fmov x11, d1
+; CHECK-NEXT:    fmov x10, d1
 ; CHECK-NEXT:    mov.d x8, v0[1]
-; CHECK-NEXT:    mov.d x10, v1[1]
-; CHECK-NEXT:    asr x12, x9, #63
-; CHECK-NEXT:    asr x13, x11, #63
-; CHECK-NEXT:    subs x9, x9, x11
-; CHECK-NEXT:    sbcs x11, x12, x13
-; CHECK-NEXT:    asr x12, x8, #63
-; CHECK-NEXT:    asr x13, x10, #63
-; CHECK-NEXT:    subs x8, x8, x10
-; CHECK-NEXT:    sbcs x10, x12, x13
-; CHECK-NEXT:    negs x12, x8
-; CHECK-NEXT:    ngcs x13, x10
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    csel x2, x12, x8, lt
-; CHECK-NEXT:    csel x3, x13, x10, lt
-; CHECK-NEXT:    negs x8, x9
-; CHECK-NEXT:    ngcs x10, x11
-; CHECK-NEXT:    cmp x11, #0
-; CHECK-NEXT:    csel x8, x8, x9, lt
-; CHECK-NEXT:    csel x1, x10, x11, lt
+; CHECK-NEXT:    mov.d x11, v1[1]
+; CHECK-NEXT:    subs x12, x9, x10
+; CHECK-NEXT:    asr x9, x9, #63
+; CHECK-NEXT:    cset w13, lo
+; CHECK-NEXT:    asr x10, x10, #63
+; CHECK-NEXT:    cmp w13, #1
+; CHECK-NEXT:    sbcs x9, x9, x10
+; CHECK-NEXT:    subs x10, x8, x11
+; CHECK-NEXT:    asr x8, x8, #63
+; CHECK-NEXT:    cset w13, lo
+; CHECK-NEXT:    asr x11, x11, #63
+; CHECK-NEXT:    cmp w13, #1
+; CHECK-NEXT:    sbcs x8, x8, x11
+; CHECK-NEXT:    asr x11, x8, #63
+; CHECK-NEXT:    eor x10, x10, x11
+; CHECK-NEXT:    eor x8, x8, x11
+; CHECK-NEXT:    subs x2, x10, x11
+; CHECK-NEXT:    asr x10, x9, #63
+; CHECK-NEXT:    cset w13, lo
+; CHECK-NEXT:    eor x12, x12, x10
+; CHECK-NEXT:    cmp w13, #1
+; CHECK-NEXT:    eor x9, x9, x10
+; CHECK-NEXT:    sbcs x3, x8, x11
+; CHECK-NEXT:    subs x8, x12, x10
+; CHECK-NEXT:    cset w11, lo
+; CHECK-NEXT:    cmp w11, #1
+; CHECK-NEXT:    sbcs x1, x9, x10
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov.d v0[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %aext = sext <2 x i64> %a to <2 x i128>
   %bext = sext <2 x i64> %b to <2 x i128>
   %abdiff = sub nsw <2 x i128> %aext, %bext
   %abcmp = icmp slt <2 x i128> %abdiff, zeroinitializer
   %ababs = sub nsw <2 x i128> zeroinitializer, %abdiff
   %absel = select <2 x i1> %abcmp, <2 x i128> %ababs, <2 x i128> %abdiff
   ret <2 x i128> %absel
 }

llvm/test/CodeGen/AArch64/atomicrmw-O0.ll

 ; NOLSE-NEXT:    b .LBB4_1
 ; NOLSE-NEXT:  .LBB4_1: // %atomicrmw.start
 ; NOLSE-NEXT:    // =>This Loop Header: Depth=1
 ; NOLSE-NEXT:    // Child Loop BB4_2 Depth 2
 ; NOLSE-NEXT:    ldr x11, [sp, #40] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x8, [sp, #32] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x13, [sp, #24] // 8-byte Folded Reload
 ; NOLSE-NEXT:    adds x14, x8, #1
+; NOLSE-NEXT:    cset w9, hs
+; NOLSE-NEXT:    subs w9, w9, #1
 ; NOLSE-NEXT:    mov x9, xzr
 ; NOLSE-NEXT:    adcs x15, x11, x9
 ; NOLSE-NEXT:  .LBB4_2: // %atomicrmw.start
 ; NOLSE-NEXT:    // Parent Loop BB4_1 Depth=1
 ; NOLSE-NEXT:    // => This Inner Loop Header: Depth=2
 ; NOLSE-NEXT:    ldaxp x10, x9, [x13]
 ; NOLSE-NEXT:    cmp x10, x8
 ; NOLSE-NEXT:    cset w12, ne

 ; LSE-NEXT:    str x8, [sp, #72] // 8-byte Folded Spill
 ; LSE-NEXT:    b .LBB4_1
 ; LSE-NEXT:  .LBB4_1: // %atomicrmw.start
 ; LSE-NEXT:    // =>This Inner Loop Header: Depth=1
 ; LSE-NEXT:    ldr x10, [sp, #72] // 8-byte Folded Reload
 ; LSE-NEXT:    ldr x8, [sp, #64] // 8-byte Folded Reload
 ; LSE-NEXT:    ldr x9, [sp, #56] // 8-byte Folded Reload
 ; LSE-NEXT:    adds x2, x8, #1
+; LSE-NEXT:    cset w11, hs
+; LSE-NEXT:    subs w11, w11, #1
 ; LSE-NEXT:    mov x11, xzr
 ; LSE-NEXT:    adcs x11, x10, x11
 ; LSE-NEXT:    // kill: def $x2 killed $x2 def $x2_x3
 ; LSE-NEXT:    mov x3, x11
 ; LSE-NEXT:    mov x0, x8
 ; LSE-NEXT:    mov x1, x10
 ; LSE-NEXT:    stp x0, x1, [sp, #8] // 16-byte Folded Spill
 ; LSE-NEXT:    caspal x0, x1, x2, x3, [x9]

llvm/test/CodeGen/AArch64/i128-math.ll

 
 declare { i128, i1 } @llvm.smul.with.overflow.i128(i128, i128)
 declare   i128       @llvm.smul.sat.i128(i128, i128)
 
 define i128 @u128_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x0, x0, x2
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    adcs x1, x1, x3
 ; CHECK-NEXT:    ret
   %1 = add i128 %x, %y
   ret i128 %1
 }
 
 define { i128, i8 } @u128_checked_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_checked_add:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adds x8, x0, x2
-; CHECK-NEXT:    adcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    mov x1, x9
-; CHECK-NEXT:    csel w10, w10, w11, eq
-; CHECK-NEXT:    eor w2, w10, #0x1
+; CHECK-NEXT:    adds x0, x0, x2
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    adcs x1, x1, x3
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    eor w2, w8, #0x1
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.uadd.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = xor i1 %3, true
   %5 = zext i1 %4 to i8
   %6 = insertvalue { i128, i8 } undef, i128 %2, 0
   %7 = insertvalue { i128, i8 } %6, i8 %5, 1
   ret { i128, i8 } %7
 }
 
 define { i128, i8 } @u128_overflowing_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_overflowing_add:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adds x8, x0, x2
-; CHECK-NEXT:    adcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    mov x1, x9
-; CHECK-NEXT:    csel w2, w10, w11, eq
+; CHECK-NEXT:    adds x0, x0, x2
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    adcs x1, x1, x3
+; CHECK-NEXT:    cset w2, hs
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.uadd.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = zext i1 %3 to i8
   %5 = insertvalue { i128, i8 } undef, i128 %2, 0
   %6 = insertvalue { i128, i8 } %5, i8 %4, 1
   ret { i128, i8 } %6
 }
 
 define i128 @u128_saturating_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x2
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
 ; CHECK-NEXT:    csinv x0, x8, xzr, eq
 ; CHECK-NEXT:    csinv x1, x9, xzr, eq
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.uadd.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @u128_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x0, x0, x2
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    sbcs x1, x1, x3
 ; CHECK-NEXT:    ret
   %1 = sub i128 %x, %y
   ret i128 %1
 }
 
 define { i128, i8 } @u128_checked_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_checked_sub:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    subs x8, x0, x2
-; CHECK-NEXT:    sbcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    cset w10, hi
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, hi
-; CHECK-NEXT:    mov x1, x9
-; CHECK-NEXT:    csel w10, w10, w11, eq
-; CHECK-NEXT:    eor w2, w10, #0x1
+; CHECK-NEXT:    subs x0, x0, x2
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    sbcs x1, x1, x3
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    eor w2, w8, #0x1
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.usub.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = xor i1 %3, true
   %5 = zext i1 %4 to i8
   %6 = insertvalue { i128, i8 } undef, i128 %2, 0
   %7 = insertvalue { i128, i8 } %6, i8 %5, 1
   ret { i128, i8 } %7
 }
 
 define { i128, i8 } @u128_overflowing_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_overflowing_sub:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    subs x8, x0, x2
-; CHECK-NEXT:    sbcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    cset w10, hi
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, hi
-; CHECK-NEXT:    mov x1, x9
-; CHECK-NEXT:    csel w2, w10, w11, eq
+; CHECK-NEXT:    subs x0, x0, x2
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    sbcs x1, x1, x3
+; CHECK-NEXT:    cset w2, hs
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.usub.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = zext i1 %3 to i8
   %5 = insertvalue { i128, i8 } undef, i128 %2, 0
   %6 = insertvalue { i128, i8 } %5, i8 %4, 1
   ret { i128, i8 } %6
 }
 
 define i128 @u128_saturating_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x2
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x1, x3
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    cset w10, hi
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, hi
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
 ; CHECK-NEXT:    csel x0, xzr, x8, ne
 ; CHECK-NEXT:    csel x1, xzr, x9, ne
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.usub.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @i128_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x0, x0, x2
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    adcs x1, x1, x3
 ; CHECK-NEXT:    ret
   %1 = add i128 %x, %y
   ret i128 %1
 }
 
 define { i128, i8 } @i128_checked_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_checked_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x0, x0, x2
-; CHECK-NEXT:    eor x9, x1, x3
-; CHECK-NEXT:    adcs x8, x1, x3
-; CHECK-NEXT:    eor x10, x1, x8
-; CHECK-NEXT:    mov x1, x8
-; CHECK-NEXT:    bics xzr, x10, x9
-; CHECK-NEXT:    cset w2, ge
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    adcs x1, x1, x3
+; CHECK-NEXT:    cset w8, vs
+; CHECK-NEXT:    eor w2, w8, #0x1
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.sadd.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = xor i1 %3, true
   %5 = zext i1 %4 to i8
   %6 = insertvalue { i128, i8 } undef, i128 %2, 0
   %7 = insertvalue { i128, i8 } %6, i8 %5, 1
   ret { i128, i8 } %7
 }
 
 define { i128, i8 } @i128_overflowing_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_overflowing_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x0, x0, x2
-; CHECK-NEXT:    eor x9, x1, x3
-; CHECK-NEXT:    adcs x8, x1, x3
-; CHECK-NEXT:    eor x10, x1, x8
-; CHECK-NEXT:    mov x1, x8
-; CHECK-NEXT:    bics xzr, x10, x9
-; CHECK-NEXT:    cset w2, lt
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    adcs x1, x1, x3
+; CHECK-NEXT:    cset w2, vs
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.sadd.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = zext i1 %3 to i8
   %5 = insertvalue { i128, i8 } undef, i128 %2, 0
   %6 = insertvalue { i128, i8 } %5, i8 %4, 1
   ret { i128, i8 } %6
 }
 
 define i128 @i128_saturating_add(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_saturating_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x0, x2
-; CHECK-NEXT:    eor x11, x1, x3
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x1, x3
 ; CHECK-NEXT:    asr x10, x9, #63
-; CHECK-NEXT:    eor x12, x1, x9
-; CHECK-NEXT:    bics xzr, x12, x11
+; CHECK-NEXT:    cset w11, vs
+; CHECK-NEXT:    cmp w11, #0
 ; CHECK-NEXT:    eor x11, x10, #0x8000000000000000
-; CHECK-NEXT:    csel x0, x10, x8, lt
-; CHECK-NEXT:    csel x1, x11, x9, lt
+; CHECK-NEXT:    csel x0, x10, x8, ne
+; CHECK-NEXT:    csel x1, x11, x9, ne
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.sadd.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @i128_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x0, x0, x2
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    sbcs x1, x1, x3
 ; CHECK-NEXT:    ret
   %1 = sub i128 %x, %y
   ret i128 %1
 }
 
 define { i128, i8 } @i128_checked_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_checked_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x0, x0, x2
-; CHECK-NEXT:    eor x9, x1, x3
-; CHECK-NEXT:    sbcs x8, x1, x3
-; CHECK-NEXT:    eor x10, x1, x8
-; CHECK-NEXT:    mov x1, x8
-; CHECK-NEXT:    tst x9, x10
-; CHECK-NEXT:    cset w2, ge
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    sbcs x1, x1, x3
+; CHECK-NEXT:    cset w8, vs
+; CHECK-NEXT:    eor w2, w8, #0x1
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.ssub.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = xor i1 %3, true
   %5 = zext i1 %4 to i8
   %6 = insertvalue { i128, i8 } undef, i128 %2, 0
   %7 = insertvalue { i128, i8 } %6, i8 %5, 1
   ret { i128, i8 } %7
 }
 
 define { i128, i8 } @i128_overflowing_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_overflowing_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x0, x0, x2
-; CHECK-NEXT:    eor x9, x1, x3
-; CHECK-NEXT:    sbcs x8, x1, x3
-; CHECK-NEXT:    eor x10, x1, x8
-; CHECK-NEXT:    mov x1, x8
-; CHECK-NEXT:    tst x9, x10
-; CHECK-NEXT:    cset w2, lt
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    sbcs x1, x1, x3
+; CHECK-NEXT:    cset w2, vs
 ; CHECK-NEXT:    ret
   %1 = tail call { i128, i1 } @llvm.ssub.with.overflow.i128(i128 %x, i128 %y)
   %2 = extractvalue { i128, i1 } %1, 0
   %3 = extractvalue { i128, i1 } %1, 1
   %4 = zext i1 %3 to i8
   %5 = insertvalue { i128, i8 } undef, i128 %2, 0
   %6 = insertvalue { i128, i8 } %5, i8 %4, 1
   ret { i128, i8 } %6
 }
 
 define i128 @i128_saturating_sub(i128 %x, i128 %y) {
 ; CHECK-LABEL: i128_saturating_sub:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x0, x2
-; CHECK-NEXT:    eor x11, x1, x3
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x1, x3
 ; CHECK-NEXT:    asr x10, x9, #63
-; CHECK-NEXT:    eor x12, x1, x9
-; CHECK-NEXT:    tst x11, x12
+; CHECK-NEXT:    cset w11, vs
+; CHECK-NEXT:    cmp w11, #0
 ; CHECK-NEXT:    eor x11, x10, #0x8000000000000000
-; CHECK-NEXT:    csel x0, x10, x8, lt
-; CHECK-NEXT:    csel x1, x11, x9, lt
+; CHECK-NEXT:    csel x0, x10, x8, ne
+; CHECK-NEXT:    csel x1, x11, x9, ne
 ; CHECK-NEXT:    ret
   %1 = tail call i128 @llvm.ssub.sat.i128(i128 %x, i128 %y)
   ret i128 %1
 }
 
 define i128 @u128_mul(i128 %x, i128 %y) {
 ; CHECK-LABEL: u128_mul:
 ; CHECK:       // %bb.0:

llvm/test/CodeGen/AArch64/icmp-shift-opt.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck %s
 
 ; Optimize expanded SRL/SHL used as an input of
 ; SETCC comparing it with zero by removing rotation.
 ;
 ; See https://bugs.llvm.org/show_bug.cgi?id=50197
 define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind {
 ; CHECK-LABEL: opt_setcc_lt_power_of_2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:  .LBB0_1: // %loop
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds x0, x0, #1
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    adcs x1, x1, xzr
 ; CHECK-NEXT:    orr x8, x1, x0, lsr #60
 ; CHECK-NEXT:    cbnz x8, .LBB0_1
 ; CHECK-NEXT:  // %bb.2: // %exit
 ; CHECK-NEXT:    ret
   br label %loop
 
 loop:

llvm/test/CodeGen/AArch64/neg-abs.ll

 
 define i128 @neg_abs128(i128 %x) {
 ; CHECK-LABEL: neg_abs128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr x8, x1, #63
 ; CHECK-NEXT:    eor x9, x0, x8
 ; CHECK-NEXT:    eor x10, x1, x8
 ; CHECK-NEXT:    subs x0, x8, x9
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x1, x8, x10
 ; CHECK-NEXT:    ret
   %abs = tail call i128 @llvm.abs.i128(i128 %x, i1 true)
   %neg = sub nsw i128 0, %abs
   ret i128 %neg
 }
 
 

 ; CHECK-NEXT:    ret
   %abs = tail call i16 @llvm.abs.i16(i16 %x, i1 true)
   ret i16 %abs
 }
 
 define i128 @abs128(i128 %x) {
 ; CHECK-LABEL: abs128:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    negs x8, x0
-; CHECK-NEXT:    ngcs x9, x1
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    csel x0, x8, x0, lt
-; CHECK-NEXT:    csel x1, x9, x1, lt
+; CHECK-NEXT:    asr x8, x1, #63
+; CHECK-NEXT:    eor x9, x0, x8
+; CHECK-NEXT:    eor x10, x1, x8
+; CHECK-NEXT:    subs x0, x9, x8
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    sbcs x1, x10, x8
 ; CHECK-NEXT:    ret
   %abs = tail call i128 @llvm.abs.i128(i128 %x, i1 true)
   ret i128 %abs
 }
 

llvm/test/CodeGen/AArch64/neon-abd.ll

 }
 
 define <2 x i64> @sabd_2d(<2 x i64> %a, <2 x i64> %b) #0 {
 ; CHECK-LABEL: sabd_2d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, v0.d[1]
 ; CHECK-NEXT:    fmov x10, d0
 ; CHECK-NEXT:    mov x9, v1.d[1]
-; CHECK-NEXT:    asr x11, x10, #63
-; CHECK-NEXT:    asr x12, x8, #63
-; CHECK-NEXT:    asr x13, x9, #63
-; CHECK-NEXT:    subs x8, x8, x9
-; CHECK-NEXT:    fmov x9, d1
-; CHECK-NEXT:    sbcs x12, x12, x13
-; CHECK-NEXT:    asr x13, x9, #63
-; CHECK-NEXT:    subs x9, x10, x9
-; CHECK-NEXT:    sbcs x10, x11, x13
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    cneg x9, x9, lt
-; CHECK-NEXT:    cmp x12, #0
-; CHECK-NEXT:    cneg x8, x8, lt
-; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    subs x11, x8, x9
+; CHECK-NEXT:    asr x8, x8, #63
+; CHECK-NEXT:    cset w12, lo
+; CHECK-NEXT:    asr x9, x9, #63
+; CHECK-NEXT:    cmp w12, #1
+; CHECK-NEXT:    fmov x12, d1
+; CHECK-NEXT:    sbcs x8, x8, x9
+; CHECK-NEXT:    asr x8, x8, #63
+; CHECK-NEXT:    subs x9, x10, x12
+; CHECK-NEXT:    asr x10, x10, #63
+; CHECK-NEXT:    cset w13, lo
+; CHECK-NEXT:    asr x12, x12, #63
+; CHECK-NEXT:    cmp w13, #1
+; CHECK-NEXT:    eor x11, x11, x8
+; CHECK-NEXT:    sbcs x10, x10, x12
+; CHECK-NEXT:    sub x8, x11, x8
+; CHECK-NEXT:    asr x10, x10, #63
+; CHECK-NEXT:    eor x9, x9, x10
+; CHECK-NEXT:    sub x9, x9, x10
 ; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %a.sext = sext <2 x i64> %a to <2 x i128>
   %b.sext = sext <2 x i64> %b to <2 x i128>
   %sub = sub <2 x i128> %a.sext, %b.sext
   %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
   %trunc = trunc <2 x i128> %abs to <2 x i64>
   ret <2 x i64> %trunc

 
 define <2 x i64> @uabd_2d(<2 x i64> %a, <2 x i64> %b) #0 {
 ; CHECK-LABEL: uabd_2d:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov x8, v0.d[1]
 ; CHECK-NEXT:    fmov x10, d0
 ; CHECK-NEXT:    mov x9, v1.d[1]
 ; CHECK-NEXT:    subs x8, x8, x9
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    fmov x9, d1
 ; CHECK-NEXT:    ngcs x11, xzr
+; CHECK-NEXT:    asr x11, x11, #63
 ; CHECK-NEXT:    subs x9, x10, x9
+; CHECK-NEXT:    eor x8, x8, x11
+; CHECK-NEXT:    cset w10, lo
+; CHECK-NEXT:    sub x8, x8, x11
+; CHECK-NEXT:    cmp w10, #1
 ; CHECK-NEXT:    ngcs x10, xzr
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    cneg x9, x9, lt
-; CHECK-NEXT:    cmp x11, #0
-; CHECK-NEXT:    cneg x8, x8, lt
-; CHECK-NEXT:    fmov d0, x9
+; CHECK-NEXT:    asr x10, x10, #63
 ; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    eor x9, x9, x10
+; CHECK-NEXT:    sub x9, x9, x10
+; CHECK-NEXT:    fmov d0, x9
 ; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %a.zext = zext <2 x i64> %a to <2 x i128>
   %b.zext = zext <2 x i64> %b to <2 x i128>
   %sub = sub <2 x i128> %a.zext, %b.zext
   %abs = call <2 x i128> @llvm.abs.v2i128(<2 x i128> %sub, i1 true)
   %trunc = trunc <2 x i128> %abs to <2 x i64>
   ret <2 x i64> %trunc

llvm/test/CodeGen/AArch64/nzcv-save.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-eabi | FileCheck %s
 
 ; DAG ends up with two uses for the flags from an ADCS node, which means they
 ; must be saved for later.
 define void @f(i256* nocapture %a, i256* nocapture %b, i256* nocapture %cc, i256* nocapture %dd) nounwind uwtable noinline ssp {
 ; CHECK-LABEL: f:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ldp x9, x8, [x2]
-; CHECK-NEXT:    ldp x11, x10, [x3]
-; CHECK-NEXT:    adds x9, x9, x11
-; CHECK-NEXT:    ldp x12, x11, [x2, #16]
-; CHECK-NEXT:    adcs x8, x8, x10
-; CHECK-NEXT:    ldp x13, x10, [x3, #16]
-; CHECK-NEXT:    adcs x12, x12, x13
-; CHECK-NEXT:    mrs x13, NZCV
-; CHECK-NEXT:    adcs x14, x11, x10
-; CHECK-NEXT:    orr x11, x11, #0x100
-; CHECK-NEXT:    msr NZCV, x13
-; CHECK-NEXT:    stp x9, x8, [x0]
-; CHECK-NEXT:    adcs x10, x11, x10
-; CHECK-NEXT:    stp x12, x14, [x0, #16]
-; CHECK-NEXT:    stp x9, x8, [x1]
-; CHECK-NEXT:    stp x12, x10, [x1, #16]
+; CHECK-NEXT:    ldp x8, x10, [x2]
+; CHECK-NEXT:    ldp x9, x11, [x3]
+; CHECK-NEXT:    ldp x12, x13, [x2, #16]
+; CHECK-NEXT:    adds x8, x8, x9
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
+; CHECK-NEXT:    adcs x10, x10, x11
+; CHECK-NEXT:    orr x14, x13, #0x100
+; CHECK-NEXT:    cset w11, hs
+; CHECK-NEXT:    cmp w11, #1
+; CHECK-NEXT:    ldp x9, x11, [x3, #16]
+; CHECK-NEXT:    stp x8, x10, [x0]
+; CHECK-NEXT:    adcs x9, x12, x9
+; CHECK-NEXT:    cset w12, hs
+; CHECK-NEXT:    cmp w12, #1
+; CHECK-NEXT:    adcs x13, x13, x11
+; CHECK-NEXT:    cmp w12, #1
+; CHECK-NEXT:    adcs x11, x14, x11
+; CHECK-NEXT:    stp x9, x13, [x0, #16]
+; CHECK-NEXT:    stp x8, x10, [x1]
+; CHECK-NEXT:    stp x9, x11, [x1, #16]
 ; CHECK-NEXT:    ret
 entry:
   %c = load i256, i256* %cc
   %d = load i256, i256* %dd
   %add = add nsw i256 %c, %d
   store i256 %add, i256* %a, align 8
   %or = or i256 %c, 1606938044258990275541962092341162602522202993782792835301376
   %add6 = add nsw i256 %or, %d
   store i256 %add6, i256* %b, align 8
   ret void
 }

llvm/test/CodeGen/AArch64/sadd_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x2, x6
-; CHECK-NEXT:    eor x10, x3, x7
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x3, x7
-; CHECK-NEXT:    eor x11, x3, x9
-; CHECK-NEXT:    asr x12, x9, #63
-; CHECK-NEXT:    bics xzr, x11, x10
-; CHECK-NEXT:    eor x10, x1, x5
-; CHECK-NEXT:    csel x2, x12, x8, lt
-; CHECK-NEXT:    eor x8, x12, #0x8000000000000000
-; CHECK-NEXT:    csel x3, x8, x9, lt
+; CHECK-NEXT:    cset w10, vs
+; CHECK-NEXT:    asr x11, x9, #63
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    csel x2, x11, x8, ne
+; CHECK-NEXT:    eor x8, x11, #0x8000000000000000
+; CHECK-NEXT:    csel x3, x8, x9, ne
 ; CHECK-NEXT:    adds x8, x0, x4
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x1, x5
-; CHECK-NEXT:    eor x11, x1, x9
-; CHECK-NEXT:    asr x12, x9, #63
-; CHECK-NEXT:    bics xzr, x11, x10
-; CHECK-NEXT:    eor x10, x12, #0x8000000000000000
-; CHECK-NEXT:    csel x8, x12, x8, lt
-; CHECK-NEXT:    csel x1, x10, x9, lt
+; CHECK-NEXT:    cset w10, vs
+; CHECK-NEXT:    asr x11, x9, #63
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    eor x10, x11, #0x8000000000000000
+; CHECK-NEXT:    csel x8, x11, x8, ne
+; CHECK-NEXT:    csel x1, x10, x9, ne
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/ssub_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x2, x6
-; CHECK-NEXT:    eor x10, x3, x7
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x3, x7
-; CHECK-NEXT:    eor x11, x3, x9
-; CHECK-NEXT:    asr x12, x9, #63
-; CHECK-NEXT:    tst x10, x11
-; CHECK-NEXT:    eor x10, x1, x5
-; CHECK-NEXT:    csel x2, x12, x8, lt
-; CHECK-NEXT:    eor x8, x12, #0x8000000000000000
-; CHECK-NEXT:    csel x3, x8, x9, lt
+; CHECK-NEXT:    cset w10, vs
+; CHECK-NEXT:    asr x11, x9, #63
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    csel x2, x11, x8, ne
+; CHECK-NEXT:    eor x8, x11, #0x8000000000000000
+; CHECK-NEXT:    csel x3, x8, x9, ne
 ; CHECK-NEXT:    subs x8, x0, x4
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x1, x5
-; CHECK-NEXT:    eor x11, x1, x9
-; CHECK-NEXT:    asr x12, x9, #63
-; CHECK-NEXT:    tst x10, x11
-; CHECK-NEXT:    eor x10, x12, #0x8000000000000000
-; CHECK-NEXT:    csel x8, x12, x8, lt
-; CHECK-NEXT:    csel x1, x10, x9, lt
+; CHECK-NEXT:    cset w10, vs
+; CHECK-NEXT:    asr x11, x9, #63
+; CHECK-NEXT:    cmp w10, #0
+; CHECK-NEXT:    eor x10, x11, #0x8000000000000000
+; CHECK-NEXT:    csel x8, x11, x8, ne
+; CHECK-NEXT:    csel x1, x10, x9, ne
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/uadd_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x2, x6
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x3, x7
-; CHECK-NEXT:    cmp x8, x2
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x3
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csinv x3, x9, xzr, eq
 ; CHECK-NEXT:    csinv x2, x8, xzr, eq
+; CHECK-NEXT:    csinv x3, x9, xzr, eq
 ; CHECK-NEXT:    adds x8, x0, x4
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x1, x5
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
 ; CHECK-NEXT:    csinv x8, x8, xzr, eq
 ; CHECK-NEXT:    csinv x1, x9, xzr, eq
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/usub_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
 ; CHECK-LABEL: v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    subs x8, x2, x6
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x3, x7
-; CHECK-NEXT:    cmp x8, x2
-; CHECK-NEXT:    cset w10, hi
-; CHECK-NEXT:    cmp x9, x3
-; CHECK-NEXT:    cset w11, hi
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    csel x3, xzr, x9, ne
 ; CHECK-NEXT:    csel x2, xzr, x8, ne
+; CHECK-NEXT:    csel x3, xzr, x9, ne
 ; CHECK-NEXT:    subs x8, x0, x4
+; CHECK-NEXT:    cset w9, lo
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    sbcs x9, x1, x5
-; CHECK-NEXT:    cmp x8, x0
-; CHECK-NEXT:    cset w10, hi
-; CHECK-NEXT:    cmp x9, x1
-; CHECK-NEXT:    cset w11, hi
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    cmp w10, #0
 ; CHECK-NEXT:    csel x8, xzr, x8, ne
 ; CHECK-NEXT:    csel x1, xzr, x9, ne
 ; CHECK-NEXT:    fmov d0, x8
 ; CHECK-NEXT:    mov v0.d[1], x1
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
   %z = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }

llvm/test/CodeGen/AArch64/vec_uaddo.ll

; CHECK-NEXT:    ret
   store <4 x i1> %val, <4 x i1>* %p2
   ret <4 x i32> %res
 }
 
 define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
 ; CHECK-LABEL: uaddo_v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x8, x2, x6
+; CHECK-NEXT:    cset w9, hs
+; CHECK-NEXT:    cmp w9, #1
 ; CHECK-NEXT:    adcs x9, x3, x7
-; CHECK-NEXT:    cmp x8, x2
-; CHECK-NEXT:    cset w10, lo
-; CHECK-NEXT:    cmp x9, x3
-; CHECK-NEXT:    cset w11, lo
-; CHECK-NEXT:    csel w10, w10, w11, eq
+; CHECK-NEXT:    cset w10, hs
 ; CHECK-NEXT:    adds x11, x0, x4
+; CHECK-NEXT:    cset w12, hs
+; CHECK-NEXT:    cmp w12, #1
 ; CHECK-NEXT:    adcs x12, x1, x5
-; CHECK-NEXT:    cmp x11, x0
-; CHECK-NEXT:    cset w13, lo
-; CHECK-NEXT:    cmp x12, x1
-; CHECK-NEXT:    cset w14, lo
-; CHECK-NEXT:    csel w13, w13, w14, eq
+; CHECK-NEXT:    cset w13, hs
 ; CHECK-NEXT:    fmov s0, w13
 ; CHECK-NEXT:    mov v0.s[1], w10
 ; CHECK-NEXT:    ldr x10, [sp]
 ; CHECK-NEXT:    stp x8, x9, [x10, #16]
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #31
 ; CHECK-NEXT:    stp x11, x12, [x10]
 ; CHECK-NEXT:    cmlt v0.2s, v0.2s, #0
 ; CHECK-NEXT:    ret
   %t = call {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
   %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
   %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
   %res = sext <2 x i1> %obit to <2 x i32>
   store <2 x i128> %val, <2 x i128>* %p2
   ret <2 x i32> %res
 }

llvm/test/CodeGen/AArch64/vecreduce-add-legalization.ll

; CHECK-NEXT:    ret
   %b = call i24 @llvm.vector.reduce.add.v4i24(<4 x i24> %a)
   ret i24 %b
 }
 
 define i128 @test_v2i128(<2 x i128> %a) nounwind {
 ; CHECK-LABEL: test_v2i128:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adds x0, x0, x2
+; CHECK-NEXT:    cset w8, hs
+; CHECK-NEXT:    cmp w8, #1
 ; CHECK-NEXT:    adcs x1, x1, x3
 ; CHECK-NEXT:    ret
   %b = call i128 @llvm.vector.reduce.add.v2i128(<2 x i128> %a)
   ret i128 %b
 }
 
 define i32 @test_v16i32(<16 x i32> %a) nounwind {
 ; CHECK-LABEL: test_v16i32: