Index: llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
===================================================================
--- llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -269,6 +269,12 @@
                          MachineIRBuilder &MIRBuilder) const;
   MachineInstr *emitSUBS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
                          MachineIRBuilder &MIRBuilder) const;
+  MachineInstr *emitADCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
+                         MachineIRBuilder &MIRBuilder) const;
+  MachineInstr *emitSetCarryFlag(MachineOperand &CarryIn,
+                                 MachineIRBuilder &MIRBuilder) const;
+  MachineInstr *emitSBCS(Register Dst, MachineOperand &LHS, MachineOperand &RHS,
+                         MachineIRBuilder &MIRBuilder) const;
   MachineInstr *emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                         MachineIRBuilder &MIRBuilder) const;
   MachineInstr *emitTST(MachineOperand &LHS, MachineOperand &RHS,
@@ -3081,6 +3087,15 @@
     I.eraseFromParent();
     return true;
   }
+  case TargetOpcode::G_SADDE:
+  case TargetOpcode::G_UADDE:
+  case TargetOpcode::G_SSUBE:
+  case TargetOpcode::G_USUBE: {
+    // Add 0xFFFFFFFF and carryIn to update PSTATE.C flag
+    // FIXME : Try some other optimal operation 
+    emitSetCarryFlag(I.getOperand(4), MIB);
+  }
+    [[fallthrough]];
   case TargetOpcode::G_SADDO:
   case TargetOpcode::G_UADDO:
   case TargetOpcode::G_SSUBO:
@@ -4556,6 +4571,54 @@
   return emitAddSub(OpcTable, Dst, LHS, RHS, MIRBuilder);
 }
 
+MachineInstr *
+AArch64InstructionSelector::emitADCS(Register Dst, MachineOperand &LHS,
+                                     MachineOperand &RHS,
+                                     MachineIRBuilder &MIRBuilder) const {
+  MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
+  assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
+  auto Ty = MRI.getType(LHS.getReg());
+  assert(!Ty.isVector() && "Expected a scalar or pointer?");
+  unsigned Size = Ty.getSizeInBits();
+  assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
+  bool Is32Bit = Size == 32;
+  const std::array<unsigned, 2> OpcTable{AArch64::ADCSXr, AArch64::ADCSWr};
+  return emitInstr(OpcTable[Is32Bit], {Dst}, {LHS, RHS}, MIRBuilder);
+}
+
+MachineInstr *AArch64InstructionSelector::emitSetCarryFlag(
+    MachineOperand &CarryIn, MachineIRBuilder &MIRBuilder) const {
+  MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
+  assert(CarryIn.isReg() && "Expected register operands?");
+  auto Ty = MRI.getType(CarryIn.getReg());
+  unsigned Size = Ty.getSizeInBits();
+  assert(Size == 32 && "Expected 32-bit type only");
+  auto DestReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
+  auto CopyReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
+  MIRBuilder.buildInstr(AArch64::MOVi32imm, {CopyReg}, {}).addImm(0xFFFFFFFF);
+  AArch64GenRegisterBankInfo::constrainGenericRegister(
+      CopyReg, AArch64::GPR32RegClass, MRI);
+  auto MI =
+      MIRBuilder.buildInstr(AArch64::ADDSWrr, {DestReg}, {CarryIn, CopyReg});
+  constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
+  return &*MI;
+}
+
+MachineInstr *
+AArch64InstructionSelector::emitSBCS(Register Dst, MachineOperand &LHS,
+                                     MachineOperand &RHS,
+                                     MachineIRBuilder &MIRBuilder) const {
+  MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
+  assert(LHS.isReg() && RHS.isReg() && "Expected register operands?");
+  auto Ty = MRI.getType(LHS.getReg());
+  assert(!Ty.isVector() && "Expected a scalar or pointer?");
+  unsigned Size = Ty.getSizeInBits();
+  assert((Size == 32 || Size == 64) && "Expected a 32-bit or 64-bit type only");
+  bool Is32Bit = Size == 32;
+  const std::array<unsigned, 2> OpcTable{AArch64::SBCSXr, AArch64::SBCSWr};
+  return emitInstr(OpcTable[Is32Bit], {Dst}, {LHS, RHS}, MIRBuilder);
+}
+
 MachineInstr *
 AArch64InstructionSelector::emitSUBS(Register Dst, MachineOperand &LHS,
                                      MachineOperand &RHS,
@@ -4791,6 +4854,14 @@
     return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
   case TargetOpcode::G_USUBO:
     return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
+  case TargetOpcode::G_SADDE:
+    return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
+  case TargetOpcode::G_UADDE:
+    return std::make_pair(emitADCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
+  case TargetOpcode::G_SSUBE:
+    return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
+  case TargetOpcode::G_USUBE:
+    return std::make_pair(emitSBCS(Dst, LHS, RHS, MIRBuilder), AArch64CC::LO);
   }
 }
 
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2.ll
@@ -925,14 +925,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -944,14 +948,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -963,14 +971,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -982,14 +994,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1001,14 +1017,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1285,7 +1305,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -1298,7 +1320,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -1311,7 +1335,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -1324,7 +1350,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -1337,7 +1365,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1675,14 +1705,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1694,14 +1728,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1713,14 +1751,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1732,14 +1774,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1751,14 +1797,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -2035,7 +2085,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -2048,7 +2100,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -2061,7 +2115,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -2074,7 +2130,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -2087,7 +2145,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2_lse128.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2_lse128.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-lse2_lse128.ll
@@ -510,10 +510,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    casp x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldp x4, x5, [x0]
@@ -527,10 +531,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspa x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldp x4, x5, [x0]
@@ -544,10 +552,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspl x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldp x4, x5, [x0]
@@ -561,10 +573,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldp x4, x5, [x0]
@@ -578,10 +594,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldp x4, x5, [x0]
@@ -810,7 +830,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -823,7 +845,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -836,7 +860,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -849,7 +875,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -862,7 +890,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1015,10 +1045,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    casp x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldp x4, x5, [x0]
@@ -1032,10 +1066,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspa x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldp x4, x5, [x0]
@@ -1049,10 +1087,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspl x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldp x4, x5, [x0]
@@ -1066,10 +1108,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldp x4, x5, [x0]
@@ -1083,10 +1129,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldp x4, x5, [x0]
@@ -1315,7 +1365,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -1328,7 +1380,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -1341,7 +1395,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -1354,7 +1410,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -1367,7 +1425,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-outline_atomics.ll
@@ -555,10 +555,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x2, x0, x9
-; -O0:    bl __aarch64_cas16_relax
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -570,10 +578,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -585,10 +601,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x2, x0, x9
-; -O0:    bl __aarch64_cas16_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -600,10 +624,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -615,10 +647,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -845,7 +885,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -858,7 +900,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -871,7 +915,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -884,7 +930,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -897,7 +945,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1130,10 +1180,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x2, x0, x9
-; -O0:    bl __aarch64_cas16_relax
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1145,10 +1203,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1160,10 +1226,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x2, x0, x9
-; -O0:    bl __aarch64_cas16_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1175,10 +1249,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1190,10 +1272,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x2, x0, x9
-; -O0:    bl __aarch64_cas16_acq_rel
-; -O0:    subs x10, x10, x11
-; -O0:    ccmp x8, x9, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1440,7 +1530,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -1453,7 +1545,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -1466,7 +1560,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -1479,7 +1575,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -1492,7 +1590,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc.ll
@@ -925,14 +925,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -944,14 +948,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -963,14 +971,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -982,14 +994,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1001,14 +1017,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1285,7 +1305,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -1298,7 +1320,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -1311,7 +1335,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -1324,7 +1350,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -1337,7 +1365,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1675,14 +1705,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1694,14 +1728,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1713,14 +1751,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1732,14 +1774,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1751,14 +1797,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -2035,7 +2085,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -2048,7 +2100,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -2061,7 +2115,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -2074,7 +2130,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -2087,7 +2145,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-rcpc3.ll
@@ -925,14 +925,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -944,14 +948,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -963,14 +971,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -982,14 +994,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1001,14 +1017,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1285,7 +1305,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -1298,7 +1320,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -1311,7 +1335,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -1324,7 +1350,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -1337,7 +1365,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1675,14 +1705,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1694,14 +1728,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1713,14 +1751,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1732,14 +1774,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1751,14 +1797,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -2035,7 +2085,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -2048,7 +2100,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -2061,7 +2115,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -2074,7 +2130,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -2087,7 +2145,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8_1a.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8_1a.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8_1a.ll
@@ -545,10 +545,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    casp x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldp x4, x5, [x0]
@@ -562,10 +566,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspa x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldp x4, x5, [x0]
@@ -579,10 +587,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspl x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldp x4, x5, [x0]
@@ -596,10 +608,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldp x4, x5, [x0]
@@ -613,10 +629,14 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x2, x10, x12
+; -O0:    adds x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldp x4, x5, [x0]
@@ -845,7 +865,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -858,7 +880,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -871,7 +895,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -884,7 +910,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -897,7 +925,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1050,10 +1080,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    casp x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldp x4, x5, [x0]
@@ -1067,10 +1101,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspa x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldp x4, x5, [x0]
@@ -1084,10 +1122,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspl x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldp x4, x5, [x0]
@@ -1101,10 +1143,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldp x4, x5, [x0]
@@ -1118,10 +1164,14 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x2, x10, x12
+; -O0:    subs x2, x9, x11
+; -O0:    and w11, w9, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    caspal x0, x1, x2, x3, [x8]
-; -O0:    subs x11, x8, x11
-; -O0:    ccmp x9, x10, #0, eq
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldp x4, x5, [x0]
@@ -1350,7 +1400,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -1363,7 +1415,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -1376,7 +1430,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -1389,7 +1445,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -1402,7 +1460,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
===================================================================
--- llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
+++ llvm/test/CodeGen/AArch64/Atomics/aarch64-atomicrmw-v8a.ll
@@ -925,14 +925,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_monotonic:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -944,14 +948,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acquire:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -963,14 +971,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_release:
-; -O0:    adds x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -982,14 +994,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_acq_rel:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1001,14 +1017,18 @@
 
 define dso_local i128 @atomicrmw_add_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_aligned_seq_cst:
-; -O0:    adds x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    adds x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_add_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1285,7 +1305,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_monotonic:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_monotonic:
@@ -1298,7 +1320,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acquire:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acquire:
@@ -1311,7 +1335,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_release:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_release:
@@ -1324,7 +1350,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_acq_rel:
@@ -1337,7 +1365,9 @@
 
 define dso_local i128 @atomicrmw_add_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
-; -O0:    adds x8, x11, x8
+; -O0:    adds x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_add_i128_unaligned_seq_cst:
@@ -1675,14 +1705,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_monotonic:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_monotonic:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1694,14 +1728,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acquire:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stxp w8, x14, x15, [x9]
-; -O0:    stxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stxp w8, x14, x15, [x11]
+; -O0:    stxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acquire:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1713,14 +1751,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_release:
-; -O0:    subs x14, x11, x10
-; -O0:    ldxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_release:
 ; -O1:    ldxp x0, x1, [x8]
@@ -1732,14 +1774,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_acq_rel:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -1751,14 +1797,18 @@
 
 define dso_local i128 @atomicrmw_sub_i128_aligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
-; -O0:    subs x14, x11, x10
-; -O0:    ldaxp x10, x12, [x9]
-; -O0:    cmp x10, x11
-; -O0:    cmp x12, x13
-; -O0:    stlxp w8, x14, x15, [x9]
-; -O0:    stlxp w8, x10, x12, [x9]
-; -O0:    subs x12, x12, x13
-; -O0:    ccmp x10, x11, #0, eq
+; -O0:    subs x14, x8, x10
+; -O0:    and w10, w8, #0x1
+; -O0:    adds w10, w10, w12
+; -O0:    ldaxp x10, x9, [x11]
+; -O0:    cmp x10, x12
+; -O0:    cmp x9, x13
+; -O0:    stlxp w8, x14, x15, [x11]
+; -O0:    stlxp w8, x10, x9, [x11]
+; -O0:    eor x8, x10, x8
+; -O0:    eor x11, x9, x11
+; -O0:    orr x8, x8, x11
+; -O0:    subs x8, x8, #0
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_aligned_seq_cst:
 ; -O1:    ldaxp x0, x1, [x8]
@@ -2035,7 +2085,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_monotonic(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_monotonic:
@@ -2048,7 +2100,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acquire(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acquire:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acquire:
@@ -2061,7 +2115,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_release(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_release:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_release:
@@ -2074,7 +2130,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_acq_rel(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_acq_rel:
@@ -2087,7 +2145,9 @@
 
 define dso_local i128 @atomicrmw_sub_i128_unaligned_seq_cst(ptr %ptr, i128 %value) {
 ; -O0-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
-; -O0:    subs x8, x11, x8
+; -O0:    subs x9, x8, x9
+; -O0:    and w11, w8, #0x1
+; -O0:    adds w11, w11, w12
 ; -O0:    bl __atomic_compare_exchange
 ;
 ; -O1-LABEL: atomicrmw_sub_i128_unaligned_seq_cst:
Index: llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/AArch64/GlobalISel/select-sadde.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-uknown -verify-machineinstrs -global-isel -run-pass=instruction-select -global-isel-abort=1 %s -o - | FileCheck %s
+
+...
+---
+name:            sadde_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $w2, $x2
+
+    ; CHECK-LABEL: name: sadde_s32
+    ; CHECK: liveins: $w0, $w1, $w2, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSWr:%[0-9]+]]:gpr32 = ADCSWr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[ADCSWr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %6:gpr(s32) = COPY $w2
+    %3:gpr(s32), %4:gpr(s32) = G_SADDE %0, %1, %6
+    $w0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $w0, implicit $w1
+
+...
+---
+name:            sadde_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: sadde_s64
+    ; CHECK: liveins: $w0, $x0, $x1, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSXr:%[0-9]+]]:gpr64 = ADCSXr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $x0 = COPY [[ADCSXr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0, implicit $w1
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %6:gpr(s32) = COPY $w0
+    %3:gpr(s64), %4:gpr(s32) = G_SADDE %0, %1, %6
+    $x0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $x0, implicit $w1
+
+...
Index: llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/AArch64/GlobalISel/select-ssube.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-uknown -verify-machineinstrs -global-isel-abort=1 -run-pass=instruction-select %s -o - | FileCheck %s
+
+...
+---
+name:            ssube_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $w2, $x2
+
+    ; CHECK-LABEL: name: ssube_s32
+    ; CHECK: liveins: $w0, $w1, $w2, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSWr:%[0-9]+]]:gpr32 = SBCSWr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[SBCSWr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = COPY $w2
+    %3:gpr(s32), %4:gpr(s32) = G_SSUBE %0, %1, %2
+    $w0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $w0, implicit $w1
+
+...
+---
+name:            ssube_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: ssube_s64
+    ; CHECK: liveins: $w0, $x0, $x1, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSXr:%[0-9]+]]:gpr64 = SBCSXr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK-NEXT: $x0 = COPY [[SBCSXr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0, implicit $w1
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s32) = COPY $w0
+    %3:gpr(s64), %4:gpr(s32) = G_SSUBE %0, %1, %2
+    $x0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $x0, implicit $w1
+
+...
Index: llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/AArch64/GlobalISel/select-uadde.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-uknown -verify-machineinstrs -global-isel-abort=1 -run-pass=instruction-select %s -o - | FileCheck %s
+
+...
+---
+name:            uadde_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $w2, $x2
+
+    ; CHECK-LABEL: name: uadde_s32
+    ; CHECK: liveins: $w0, $w1, $w2, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSWr:%[0-9]+]]:gpr32 = ADCSWr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[ADCSWr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0, implicit $w1
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = COPY $w2
+    %3:gpr(s32), %4:gpr(s32) = G_UADDE %0, %1, %2
+    $w0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $w0, implicit $w1
+
+...
+---
+name:            uadde_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: uadde_s64
+    ; CHECK: liveins: $w0, $x0, $x1, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[ADCSXr:%[0-9]+]]:gpr64 = ADCSXr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv
+    ; CHECK-NEXT: $x0 = COPY [[ADCSXr]]
+    ; CHECK-NEXT: $w1 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0, implicit $w1
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s32) = COPY $w2
+    %3:gpr(s64), %4:gpr(s32) = G_UADDE %0, %1, %2
+    $x0 = COPY %3
+    $w1 = COPY %4
+    RET_ReallyLR implicit $x0, implicit $w1
+
+...
Index: llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/AArch64/GlobalISel/select-usube.mir
@@ -0,0 +1,64 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-uknown -verify-machineinstrs -global-isel-abort=1 -run-pass=instruction-select %s -o - | FileCheck %s
+
+...
+---
+name:            usube_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $w2, $x2
+
+    ; CHECK-LABEL: name: usube_s32
+    ; CHECK: liveins: $w0, $w1, $w2, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w2
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSWr:%[0-9]+]]:gpr32 = SBCSWr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 2, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = COPY $w1
+    %2:gpr(s32) = COPY $w2
+    %3:gpr(s32), %4:gpr(s32) = G_USUBE %0, %1, %2
+    $w0 = COPY %4(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            usube_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: usube_s64
+    ; CHECK: liveins: $w0, $x0, $x1, $x2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 4294967295
+    ; CHECK-NEXT: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY2]], [[MOVi32imm]], implicit-def $nzcv
+    ; CHECK-NEXT: [[SBCSXr:%[0-9]+]]:gpr64 = SBCSXr [[COPY]], [[COPY1]], implicit-def $nzcv, implicit $nzcv
+    ; CHECK-NEXT: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 2, implicit $nzcv
+    ; CHECK-NEXT: $w0 = COPY [[CSINCWr]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:gpr(s64) = COPY $x0
+    %1:gpr(s64) = COPY $x1
+    %2:gpr(s32) = COPY $w0
+    %3:gpr(s64), %4:gpr(s32) = G_USUBE %0, %1, %2
+    $w0 = COPY %4
+    RET_ReallyLR implicit $w0
+
+...