[AArch64] Fix i128 cmpxchg using ldxp/stxp.

[efriedma]

Summary

Basically two parts to this fix:

1. Stop using AtomicExpand to expand cmpxchg i128
2. Fix AArch64ExpandPseudoInsts to use a correct expansion. In particular, we can't exit the loop until an stxp succeeds; otherwise, the load can tear.

Fixes https://bugs.llvm.org/show_bug.cgi?id=51102

[SjoerdMeijer]

This is something @samparker looked at recently, can you perhaps have a look this?

[samparker]

This made my brain melt a little... anyway, from Section B 2.2.1 of Arm ARM:

When the Store-Exclusive of a Load-Exclusive/Store-Exclusive pair instruction
using two 64-bit quantities succeeds, it causes a single-copy atomic update
of the entire memory location being updated.

To atomically load two 64-bit quantities, perform a Load-Exclusive
pair/Store-Exclusive pair sequence of reading and writing the same value for
which the Store-Exclusive pair succeeds, and use the read values from the
Load-Exclusive pair.

And it looks like this is what we are doing.

Ordering changes also LGTM.

[MaskRay]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17058

This was unused. I deleted it in 2b79a4eeb101bc9662427353187e35b9bc9da116 to pacify -Wunused-but-set-variable

[efriedma]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17058

Fixed it to do what I meant to do in 664a1fd9f04de65f0a37d612845dc9e095abe067

@samparker Let me know if you want me to add more test coverage.

[samparker]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17058

Ah! Yes please, hard enough to get this stuff right so probably worth adding cases for the other orderings...

[efriedma]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17058

301b21a941ebd94cef2f1912a547960223983887

[samparker]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

17058

cheers!

llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp

bool AArch64ExpandPseudo::expandCMP_SWAP_128(
   // value on both; However undef should be replaced by xzr anyway.
   assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
   Register AddrReg = MI.getOperand(3).getReg();
   Register DesiredLoReg = MI.getOperand(4).getReg();
   Register DesiredHiReg = MI.getOperand(5).getReg();
   Register NewLoReg = MI.getOperand(6).getReg();
   Register NewHiReg = MI.getOperand(7).getReg();
 
+  unsigned LdxpOp, StxpOp;
+
+  switch (MI.getOpcode()) {
+  case AArch64::CMP_SWAP_128_MONOTONIC:
+    LdxpOp = AArch64::LDXPX;
+    StxpOp = AArch64::STXPX;
+    break;
+  case AArch64::CMP_SWAP_128_RELEASE:
+    LdxpOp = AArch64::LDXPX;
+    StxpOp = AArch64::STLXPX;
+    break;
+  case AArch64::CMP_SWAP_128_ACQUIRE:
+    LdxpOp = AArch64::LDAXPX;
+    StxpOp = AArch64::STXPX;
+    break;
+  case AArch64::CMP_SWAP_128:
+    LdxpOp = AArch64::LDAXPX;
+    StxpOp = AArch64::STLXPX;
+    break;
+  default:
+    llvm_unreachable("Unexpected opcode");
+  }
+
   MachineFunction *MF = MBB.getParent();
   auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto FailBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
 
   MF->insert(++MBB.getIterator(), LoadCmpBB);
   MF->insert(++LoadCmpBB->getIterator(), StoreBB);
-  MF->insert(++StoreBB->getIterator(), DoneBB);
+  MF->insert(++StoreBB->getIterator(), FailBB);
+  MF->insert(++FailBB->getIterator(), DoneBB);
 
   // .Lloadcmp:
   //     ldaxp xDestLo, xDestHi, [xAddr]
   //     cmp xDestLo, xDesiredLo
   //     sbcs xDestHi, xDesiredHi
   //     b.ne .Ldone
-  BuildMI(LoadCmpBB, DL, TII->get(AArch64::LDAXPX))
+  BuildMI(LoadCmpBB, DL, TII->get(LdxpOp))
       .addReg(DestLo.getReg(), RegState::Define)
       .addReg(DestHi.getReg(), RegState::Define)
       .addReg(AddrReg);
   BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
       .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
       .addReg(DesiredLoReg)
       .addImm(0);
   BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
     .addUse(AArch64::WZR)
     .addUse(AArch64::WZR)
     .addImm(AArch64CC::EQ);
   BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
       .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
       .addReg(DesiredHiReg)
       .addImm(0);
   BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
       .addUse(StatusReg, RegState::Kill)
       .addUse(StatusReg, RegState::Kill)
       .addImm(AArch64CC::EQ);
   BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
       .addUse(StatusReg, getKillRegState(StatusDead))
-      .addMBB(DoneBB);
-  LoadCmpBB->addSuccessor(DoneBB);
+      .addMBB(FailBB);
+  LoadCmpBB->addSuccessor(FailBB);
   LoadCmpBB->addSuccessor(StoreBB);
 
   // .Lstore:
   //     stlxp wStatus, xNewLo, xNewHi, [xAddr]
   //     cbnz wStatus, .Lloadcmp
-  BuildMI(StoreBB, DL, TII->get(AArch64::STLXPX), StatusReg)
+  BuildMI(StoreBB, DL, TII->get(StxpOp), StatusReg)
       .addReg(NewLoReg)
       .addReg(NewHiReg)
       .addReg(AddrReg);
   BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
       .addReg(StatusReg, getKillRegState(StatusDead))
       .addMBB(LoadCmpBB);
+  BuildMI(StoreBB, DL, TII->get(AArch64::B)).addMBB(DoneBB);
   StoreBB->addSuccessor(LoadCmpBB);
   StoreBB->addSuccessor(DoneBB);
 
+  // .Lfail:
+  //     stlxp wStatus, xDestLo, xDestHi, [xAddr]
+  //     cbnz wStatus, .Lloadcmp
+  BuildMI(FailBB, DL, TII->get(StxpOp), StatusReg)
+      .addReg(DestLo.getReg())
+      .addReg(DestHi.getReg())
+      .addReg(AddrReg);
+  BuildMI(FailBB, DL, TII->get(AArch64::CBNZW))
+      .addReg(StatusReg, getKillRegState(StatusDead))
+      .addMBB(LoadCmpBB);
+  FailBB->addSuccessor(LoadCmpBB);
+  FailBB->addSuccessor(DoneBB);
+
   DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
   DoneBB->transferSuccessors(&MBB);
 
   MBB.addSuccessor(LoadCmpBB);
 
   NextMBBI = MBB.end();
   MI.eraseFromParent();
 
   // Recompute liveness bottom up.
   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *DoneBB);
+  computeAndAddLiveIns(LiveRegs, *FailBB);
   computeAndAddLiveIns(LiveRegs, *StoreBB);
   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
+
   // Do an extra pass in the loop to get the loop carried dependencies right.
+  FailBB->clearLiveIns();
+  computeAndAddLiveIns(LiveRegs, *FailBB);
   StoreBB->clearLiveIns();
   computeAndAddLiveIns(LiveRegs, *StoreBB);
   LoadCmpBB->clearLiveIns();
   computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
 
   return true;
 }
 
return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
                           AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
                           AArch64::WZR, NextMBBI);
   case AArch64::CMP_SWAP_64:
     return expandCMP_SWAP(MBB, MBBI,
                           AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
                           AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
                           AArch64::XZR, NextMBBI);
   case AArch64::CMP_SWAP_128:
+  case AArch64::CMP_SWAP_128_RELEASE:
+  case AArch64::CMP_SWAP_128_ACQUIRE:
+  case AArch64::CMP_SWAP_128_MONOTONIC:
     return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
 
   case AArch64::AESMCrrTied:
   case AArch64::AESIMCrrTied: {
     MachineInstrBuilder MIB =
     BuildMI(MBB, MBBI, MI.getDebugLoc(),
             TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
                                                       AArch64::AESIMCrr))

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 
 static void ReplaceCMP_SWAP_128Results(SDNode *N,
                                        SmallVectorImpl<SDValue> &Results,
                                        SelectionDAG &DAG,
                                        const AArch64Subtarget *Subtarget) {
   assert(N->getValueType(0) == MVT::i128 &&
          "AtomicCmpSwap on types less than 128 should be legal");
 
+  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
   if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) {
     // LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type,
     // so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG.
     SDValue Ops[] = {
         createGPRPairNode(DAG, N->getOperand(2)), // Compare value
         createGPRPairNode(DAG, N->getOperand(3)), // Store value
         N->getOperand(1), // Ptr
         N->getOperand(0), // Chain in
     };
 
-    MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
-
     unsigned Opcode;
     switch (MemOp->getMergedOrdering()) {
     case AtomicOrdering::Monotonic:
       Opcode = AArch64::CASPX;
       break;
     case AtomicOrdering::Acquire:
       Opcode = AArch64::CASPAX;
       break;
if (Subtarget->hasLSE() || Subtarget->outlineAtomics()) {
     SDValue Hi = DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
                                             SDValue(CmpSwap, 0));
     Results.push_back(
         DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128, Lo, Hi));
     Results.push_back(SDValue(CmpSwap, 1)); // Chain out
     return;
   }
 
+  unsigned Opcode;
+  switch (MemOp->getMergedOrdering()) {
+  case AtomicOrdering::Monotonic:
+    Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
+    break;
+  case AtomicOrdering::Acquire:
+    Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
+    break;
+  case AtomicOrdering::Release:
+    Opcode = AArch64::CMP_SWAP_128_RELEASE;
+    break;
+  case AtomicOrdering::AcquireRelease:
+  case AtomicOrdering::SequentiallyConsistent:
+    Opcode = AArch64::CMP_SWAP_128;
+    break;
+  default:
+    llvm_unreachable("Unexpected ordering!");
+  }
+
   auto Desired = splitInt128(N->getOperand(2), DAG);
   auto New = splitInt128(N->getOperand(3), DAG);
   SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
                    New.first,        New.second,    N->getOperand(0)};
   SDNode *CmpSwap = DAG.getMachineNode(
       AArch64::CMP_SWAP_128, SDLoc(N),
       DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops);
-
-  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
   DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
 
   Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
                                 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
   Results.push_back(SDValue(CmpSwap, 3));
 }
 
 void AArch64TargetLowering::ReplaceNodeResults(
if (Subtarget->hasLSE() || Subtarget->outlineAtomics())
     return AtomicExpansionKind::None;
   // At -O0, fast-regalloc cannot cope with the live vregs necessary to
   // implement cmpxchg without spilling. If the address being exchanged is also
   // on the stack and close enough to the spill slot, this can lead to a
   // situation where the monitor always gets cleared and the atomic operation
   // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
   if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
     return AtomicExpansionKind::None;
+
+  // 128-bit atomic cmpxchg is weird; AtomicExpand doesn't know how to expand
+  // it.
+  unsigned Size = AI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
+  if (Size > 64)
+    return AtomicExpansionKind::None;
+
   return AtomicExpansionKind::LLSC;
 }
 
 Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
                                              Type *ValueTy, Value *Addr,
                                              AtomicOrdering Ord) const {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
   bool IsAcquire = isAcquireOrStronger(Ord);

llvm/lib/Target/AArch64/AArch64InstrAtomics.td

def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
                   Sched<[WriteAtomic]>;
 
 def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
                          (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
                   Sched<[WriteAtomic]>;
 }
 
 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
-    mayLoad = 1, mayStore = 1 in
-def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
+    mayLoad = 1, mayStore = 1 in {
+class cmp_swap_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32common:$scratch),
                            (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
                                 GPR64:$newLo, GPR64:$newHi), []>,
                      Sched<[WriteAtomic]>;
+def CMP_SWAP_128 : cmp_swap_128;
+def CMP_SWAP_128_RELEASE : cmp_swap_128;
+def CMP_SWAP_128_ACQUIRE : cmp_swap_128;
+def CMP_SWAP_128_MONOTONIC : cmp_swap_128;
+}
 
 // v8.1 Atomic instructions:
 let Predicates = [HasLSE] in {
   defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;
   defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;
   defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;
   defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">;
   defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

if (ST->hasLSE()) {
     CAS = MIRBuilder.buildInstr(Opcode, {CASDst}, {CASDesired, CASNew, Addr});
 
     MIRBuilder.buildExtract({DstLo}, {CASDst}, 0);
     MIRBuilder.buildExtract({DstHi}, {CASDst}, 64);
   } else {
     // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
     // can take arbitrary registers so it just has the normal GPR64 operands the
     // rest of AArch64 is expecting.
+    auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
+    unsigned Opcode;
+    switch (Ordering) {
+    case AtomicOrdering::Acquire:
+      Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
+      break;
+    case AtomicOrdering::Release:
+      Opcode = AArch64::CMP_SWAP_128_RELEASE;
+      break;
+    case AtomicOrdering::AcquireRelease:
+    case AtomicOrdering::SequentiallyConsistent:
+      Opcode = AArch64::CMP_SWAP_128;
+      break;
+    default:
+      Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
+      break;
+    }
+
     auto Scratch = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
-    CAS = MIRBuilder.buildInstr(AArch64::CMP_SWAP_128, {DstLo, DstHi, Scratch},
+    CAS = MIRBuilder.buildInstr(Opcode, {DstLo, DstHi, Scratch},
                                 {Addr, DesiredI->getOperand(0),
                                  DesiredI->getOperand(1), NewI->getOperand(0),
                                  NewI->getOperand(1)});
   }
 
   CAS.cloneMemRefs(MI);
   constrainSelectedInstRegOperands(*CAS, *ST->getInstrInfo(),
                                    *MRI.getTargetRegisterInfo(),

llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll

 ; CHECK-LLSC-O0-LABEL: val_compare_and_swap:
 ; CHECK-LLSC-O0:  .LBB0_1:
 ; CHECK-LLSC-O0:    ldaxp [[OLD_LO:x[0-9]+]], [[OLD_HI:x[0-9]+]], [x0]
 ; CHECK-LLSC-O0:    cmp [[OLD_LO]], x2
 ; CHECK-LLSC-O0:    cset [[EQUAL_TMP:w[0-9]+]], ne
 ; CHECK-LLSC-O0:    cmp [[OLD_HI]], x3
 ; CHECK-LLSC-O0:    cinc [[EQUAL:w[0-9]+]], [[EQUAL_TMP]], ne
 ; CHECK-LLSC-O0:    cbnz [[EQUAL]], .LBB0_3
-; CHECK-LLSC-O0:    stlxp [[STATUS:w[0-9]+]], x4, x5, [x0]
+; CHECK-LLSC-O0:    stxp [[STATUS:w[0-9]+]], x4, x5, [x0]
 ; CHECK-LLSC-O0:    cbnz [[STATUS]], .LBB0_1
 ; CHECK-LLSC-O0:  .LBB0_3:
 ; CHECK-LLSC-O0:    mov v[[OLD:[0-9]+]].d[0], [[OLD_LO]]
 ; CHECK-LLSC-O0:    mov v[[OLD]].d[1], [[OLD_HI]]
 ; CHECK-LLSC-O0:    str q[[OLD]], [x0]
 
 
 ; CHECK-CAS-O0-LABEL: val_compare_and_swap:

llvm/test/CodeGen/AArch64/GlobalISel/legalize-cmpxchg-128.mir

liveins: $x0, $x1, $x2, $x3, $x4
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
     ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
     ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
     ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
     ; CHECK: [[COPY5:%[0-9]+]]:gpr64(s64) = COPY [[COPY1]](s64)
     ; CHECK: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64)
     ; CHECK: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64)
     ; CHECK: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64)
-    ; CHECK: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32 = CMP_SWAP_128 [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire 16)
+    ; CHECK: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire 16)
     ; CHECK: [[COPY9:%[0-9]+]]:gpr64 = COPY %16
     ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64)
     ; CHECK: G_STORE [[MV]](s128), [[COPY]](p0) :: (store 16)
     ; CHECK: RET_ReallyLR
     ; CHECK-NOLSE-LABEL: name: compare_swap_128
     ; CHECK-NOLSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4
     ; CHECK-NOLSE: [[COPY:%[0-9]+]]:gpr64(p0) = COPY $x0
     ; CHECK-NOLSE: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
     ; CHECK-NOLSE: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
     ; CHECK-NOLSE: [[COPY3:%[0-9]+]]:_(s64) = COPY $x3
     ; CHECK-NOLSE: [[COPY4:%[0-9]+]]:_(s64) = COPY $x4
     ; CHECK-NOLSE: [[COPY5:%[0-9]+]]:gpr64(s64) = COPY [[COPY1]](s64)
     ; CHECK-NOLSE: [[COPY6:%[0-9]+]]:gpr64(s64) = COPY [[COPY2]](s64)
     ; CHECK-NOLSE: [[COPY7:%[0-9]+]]:gpr64(s64) = COPY [[COPY3]](s64)
     ; CHECK-NOLSE: [[COPY8:%[0-9]+]]:gpr64(s64) = COPY [[COPY4]](s64)
-    ; CHECK-NOLSE: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32 = CMP_SWAP_128 [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128))
+    ; CHECK-NOLSE: early-clobber %13:gpr64(s64), early-clobber %14:gpr64(s64), early-clobber %16:gpr32common = CMP_SWAP_128_ACQUIRE [[COPY]](p0), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64) :: (load store acquire acquire (s128))
     ; CHECK-NOLSE: [[COPY9:%[0-9]+]]:gpr64 = COPY %16
     ; CHECK-NOLSE: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES %13(s64), %14(s64)
     ; CHECK-NOLSE: G_STORE [[MV]](s128), [[COPY]](p0) :: (store (s128))
     ; CHECK-NOLSE: RET_ReallyLR
     ; CHECK-LSE-LABEL: name: compare_swap_128
     ; CHECK-LSE: liveins: $x0_x1, $x1, $x0, $x1, $x2, $x3, $x4
     ; CHECK-LSE: [[COPY:%[0-9]+]]:gpr64sp(p0) = COPY $x0
     ; CHECK-LSE: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1

llvm/test/CodeGen/AArch64/arm64-atomic-128.ll

 ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-linux-gnu -verify-machineinstrs -mcpu=cyclone -mattr=+outline-atomics | FileCheck %s -check-prefix=OUTLINE-ATOMICS
 
 @var = global i128 0
 
 define i128 @val_compare_and_swap(i128* %p, i128 %oldval, i128 %newval) {
 ; OUTLINE-ATOMICS: bl __aarch64_cas16_acq
 ; CHECK-LABEL: val_compare_and_swap:
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: ldaxp   [[RESULTLO:x[0-9]+]], [[RESULTHI:x[0-9]+]], [x[[ADDR:[0-9]+]]]
-; CHECK-DAG: eor     [[MISMATCH_LO:x[0-9]+]], [[RESULTLO]], x2
-; CHECK-DAG: eor     [[MISMATCH_HI:x[0-9]+]], [[RESULTHI]], x3
-; CHECK: orr [[MISMATCH:x[0-9]+]], [[MISMATCH_LO]], [[MISMATCH_HI]]
-; CHECK: cbnz    [[MISMATCH]], [[DONE:.LBB[0-9]+_[0-9]+]]
-; CHECK: stxp   [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]]
+; CHECK: cmp
+; CHECK: cset
+; CHECK: cmp
+; CHECK: cinc   [[MISMATCH:w[0-9]+]]
+; CHECK: cbz    [[MISMATCH]], [[EQUAL:.LBB[0-9]+_[0-9]+]]
+; CHECK: stlxp  [[SCRATCH_RES:w[0-9]+]], [[RESULTLO]], [[RESULTHI]], [x[[ADDR]]]
+; CHECK: cbnz   [[SCRATCH_RES]], [[LABEL]]
+; CHECK: b      [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK: [[EQUAL]]:
+; CHECK: stlxp   [[SCRATCH_RES:w[0-9]+]], x4, x5, [x[[ADDR]]]
 ; CHECK: cbnz   [[SCRATCH_RES]], [[LABEL]]
 ; CHECK: [[DONE]]:
   %pair = cmpxchg i128* %p, i128 %oldval, i128 %newval acquire acquire
   %val = extractvalue { i128, i1 } %pair, 0
   ret i128 %val
 }
 
 define void @fetch_and_nand(i128* %p, i128 %bits) {

 define i128 @atomic_load_relaxed(i64, i64, i128* %p) {
 ; CHECK-LABEL: atomic_load_relaxed:
 ; CHECK-NOT: dmb
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: ldxp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x2]
 ; CHECK-NEXT: stxp [[SUCCESS:w[0-9]+]], [[LO]], [[HI]], [x2]
 ; CHECK: cbnz [[SUCCESS]], [[LABEL]]
 ; CHECK-NOT: dmb
     %r = load atomic i128, i128* %p monotonic, align 16
     ret i128 %r
 }
 
 
 define void @atomic_store_seq_cst(i128 %in, i128* %p) {
 ; CHECK-LABEL: atomic_store_seq_cst:
 ; CHECK-NOT: dmb
 ; CHECK: [[LABEL:.?LBB[0-9]+_[0-9]+]]:
 ; CHECK: ldaxp xzr, [[IGNORED:x[0-9]+]], [x2]

llvm/test/CodeGen/AArch64/atomicrmw-O0.ll

 ; NOLSE-NEXT:    cset w12, ne
 ; NOLSE-NEXT:    cmp x9, x11
 ; NOLSE-NEXT:    cinc w12, w12, ne
 ; NOLSE-NEXT:    cbnz w12, .LBB4_4
 ; NOLSE-NEXT:  // %bb.3: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=2
 ; NOLSE-NEXT:    stlxp w12, x14, x15, [x13]
 ; NOLSE-NEXT:    cbnz w12, .LBB4_2
+; NOLSE-NEXT:    b .LBB4_5
 ; NOLSE-NEXT:  .LBB4_4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB4_2 Depth=2
+; NOLSE-NEXT:    stlxp w12, x10, x9, [x13]
+; NOLSE-NEXT:    cbnz w12, .LBB4_2
+; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB4_1 Depth=1
 ; NOLSE-NEXT:    eor x11, x9, x11
 ; NOLSE-NEXT:    eor x8, x10, x8
 ; NOLSE-NEXT:    orr x8, x8, x11
 ; NOLSE-NEXT:    str x9, [sp, #8] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x10, [sp, #16] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x10, [sp, #32] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x9, [sp, #40] // 8-byte Folded Spill
 ; NOLSE-NEXT:    cbnz x8, .LBB4_1
-; NOLSE-NEXT:    b .LBB4_5
-; NOLSE-NEXT:  .LBB4_5: // %atomicrmw.end
+; NOLSE-NEXT:    b .LBB4_6
+; NOLSE-NEXT:  .LBB4_6: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x1, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x0, [sp, #16] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #48 // =48
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_add_128:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #80 // =80

 ; NOLSE-NEXT:    cset w12, ne
 ; NOLSE-NEXT:    cmp x9, x11
 ; NOLSE-NEXT:    cinc w12, w12, ne
 ; NOLSE-NEXT:    cbnz w12, .LBB9_4
 ; NOLSE-NEXT:  // %bb.3: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=2
 ; NOLSE-NEXT:    stlxp w12, x14, x15, [x13]
 ; NOLSE-NEXT:    cbnz w12, .LBB9_2
+; NOLSE-NEXT:    b .LBB9_5
 ; NOLSE-NEXT:  .LBB9_4: // %atomicrmw.start
+; NOLSE-NEXT:    // in Loop: Header=BB9_2 Depth=2
+; NOLSE-NEXT:    stlxp w12, x10, x9, [x13]
+; NOLSE-NEXT:    cbnz w12, .LBB9_2
+; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.start
 ; NOLSE-NEXT:    // in Loop: Header=BB9_1 Depth=1
 ; NOLSE-NEXT:    eor x11, x9, x11
 ; NOLSE-NEXT:    eor x8, x10, x8
 ; NOLSE-NEXT:    orr x8, x8, x11
 ; NOLSE-NEXT:    str x9, [sp, #8] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x10, [sp, #16] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x10, [sp, #32] // 8-byte Folded Spill
 ; NOLSE-NEXT:    str x9, [sp, #40] // 8-byte Folded Spill
 ; NOLSE-NEXT:    cbnz x8, .LBB9_1
-; NOLSE-NEXT:    b .LBB9_5
-; NOLSE-NEXT:  .LBB9_5: // %atomicrmw.end
+; NOLSE-NEXT:    b .LBB9_6
+; NOLSE-NEXT:  .LBB9_6: // %atomicrmw.end
 ; NOLSE-NEXT:    ldr x1, [sp, #8] // 8-byte Folded Reload
 ; NOLSE-NEXT:    ldr x0, [sp, #16] // 8-byte Folded Reload
 ; NOLSE-NEXT:    add sp, sp, #48 // =48
 ; NOLSE-NEXT:    ret
 ;
 ; LSE-LABEL: test_rmw_nand_128:
 ; LSE:       // %bb.0: // %entry
 ; LSE-NEXT:    sub sp, sp, #80 // =80