[RISCV] Initial ISel support for the experimental zacas extension

[dtcxzyw]

Summary

This patch implements the v1.0-rc3 draft extension based on AArch64's version.

It introduces two register classes (GPRPI64/GPRPI128) and some pseudo instructions for correct register allocation.
These pseudo instructions will be expanded in the RISCVExpandAtomicPseudoInsts pass.

BTW, I am not sure that the implementation of MC layer is correct.

[wangpc]

llvm/lib/Target/RISCV/RISCVInstrInfoA.td

107

Adding Scheds should be in separate patch.

[dtcxzyw]

• Rebase
• Remove Sched

[dtcxzyw]

Ping.

llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp

bool expandAtomicBinOp(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI,
                             AtomicRMWInst::BinOp, bool IsMasked, int Width,
                             MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI, bool IsMasked,
                            int Width, MachineBasicBlock::iterator &NextMBBI);
+  bool expandAMOCAS(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                    bool IsPaired, int Width,
+                    MachineBasicBlock::iterator &NextMBBI);
 #ifndef NDEBUG
   unsigned getInstSizeInBytes(const MachineFunction &MF) const {
     unsigned Size = 0;
     for (auto &MBB : MF)
       for (auto &MI : MBB)
         Size += TII->getInstSizeInBytes(MI);
     return Size;
   }
case RISCV::PseudoMaskedAtomicLoadUMin32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
                                 NextMBBI);
   case RISCV::PseudoCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
   case RISCV::PseudoCmpXchg64:
     return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
   case RISCV::PseudoMaskedCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
+  case RISCV::PseudoAMOCAS_W:
+    return expandAMOCAS(MBB, MBBI, false, 32, NextMBBI);
+  case RISCV::PseudoAMOCAS_D_64:
+    return expandAMOCAS(MBB, MBBI, false, 64, NextMBBI);
+  case RISCV::PseudoAMOCAS_D_32:
+    return expandAMOCAS(MBB, MBBI, true, 64, NextMBBI);
+  case RISCV::PseudoAMOCAS_Q:
+    return expandAMOCAS(MBB, MBBI, true, 128, NextMBBI);
   }
 
   return false;
 }
 
 static unsigned getLRForRMW32(AtomicOrdering Ordering,
                               const RISCVSubtarget *Subtarget) {
   switch (Ordering) {
static unsigned getSCForRMW(AtomicOrdering Ordering, int Width,
                             const RISCVSubtarget *Subtarget) {
   if (Width == 32)
     return getSCForRMW32(Ordering, Subtarget);
   if (Width == 64)
     return getSCForRMW64(Ordering, Subtarget);
   llvm_unreachable("Unexpected SC width\n");
 }
 
+static unsigned getAMOCASForRMW32(AtomicOrdering Ordering,
+                                  const RISCVSubtarget *Subtarget) {
+  if (Subtarget->hasStdExtZtso())
+    return RISCV::AMOCAS_W;
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::AMOCAS_W;
+  case AtomicOrdering::Acquire:
+    return RISCV::AMOCAS_W_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::AMOCAS_W_RL;
+  case AtomicOrdering::AcquireRelease:
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::AMOCAS_W_AQ_RL;
+  }
+}
+
+static unsigned getAMOCASForRMW64(AtomicOrdering Ordering,
+                                  const RISCVSubtarget *Subtarget) {
+  if (Subtarget->hasStdExtZtso())
+    return RISCV::AMOCAS_D;
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::AMOCAS_D;
+  case AtomicOrdering::Acquire:
+    return RISCV::AMOCAS_D_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::AMOCAS_D_RL;
+  case AtomicOrdering::AcquireRelease:
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::AMOCAS_D_AQ_RL;
+  }
+}
+
+static unsigned getAMOCASForRMW128(AtomicOrdering Ordering,
+                                   const RISCVSubtarget *Subtarget) {
+  if (Subtarget->hasStdExtZtso())
+    return RISCV::AMOCAS_Q;
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::AMOCAS_Q;
+  case AtomicOrdering::Acquire:
+    return RISCV::AMOCAS_Q_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::AMOCAS_Q_RL;
+  case AtomicOrdering::AcquireRelease:
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::AMOCAS_Q_AQ_RL;
+  }
+}
+
+static unsigned getAMOCASForRMW(AtomicOrdering Ordering, int Width,
+                                const RISCVSubtarget *Subtarget) {
+  if (Width == 32)
+    return getAMOCASForRMW32(Ordering, Subtarget);
+  if (Width == 64)
+    return getAMOCASForRMW64(Ordering, Subtarget);
+  if (Width == 128)
+    return getAMOCASForRMW128(Ordering, Subtarget);
+  llvm_unreachable("Unexpected AMOCAS width\n");
+}
+
 static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
                                    DebugLoc DL, MachineBasicBlock *ThisMBB,
                                    MachineBasicBlock *LoopMBB,
                                    MachineBasicBlock *DoneMBB,
                                    AtomicRMWInst::BinOp BinOp, int Width,
                                    const RISCVSubtarget *STI) {
   Register DestReg = MI.getOperand(0).getReg();
   Register ScratchReg = MI.getOperand(1).getReg();
bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
   computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);
 
   return true;
 }
 
+static Register getGPRPairEvenReg(Register PairedReg) {
+  switch (PairedReg) {
+  case RISCV::X0_PD:
+    return RISCV::X0;
+  case RISCV::X2_PD:
+    return RISCV::X2;
+  case RISCV::X4_PD:
+    return RISCV::X4;
+  case RISCV::X6_PD:
+    return RISCV::X6;
+  case RISCV::X8_PD:
+    return RISCV::X8;
+  case RISCV::X10_PD:
+    return RISCV::X10;
+  case RISCV::X12_PD:
+    return RISCV::X12;
+  case RISCV::X14_PD:
+    return RISCV::X14;
+  case RISCV::X16_PD:
+    return RISCV::X16;
+  case RISCV::X18_PD:
+    return RISCV::X18;
+  case RISCV::X20_PD:
+    return RISCV::X20;
+  case RISCV::X22_PD:
+    return RISCV::X22;
+  case RISCV::X24_PD:
+    return RISCV::X24;
+  case RISCV::X26_PD:
+    return RISCV::X26;
+  case RISCV::X28_PD:
+    return RISCV::X28;
+  case RISCV::X30_PD:
+    return RISCV::X30;
+  default:
+    llvm_unreachable("Unexpected GPR pair");
+  }
+}
+
+bool RISCVExpandAtomicPseudo::expandAMOCAS(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsPaired,
+    int Width, MachineBasicBlock::iterator &NextMBBI) {
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+
+  Register DestReg = MI.getOperand(0).getReg();
+  if (IsPaired)
+    DestReg = getGPRPairEvenReg(DestReg);
+  Register AddrReg = MI.getOperand(1).getReg();
+  Register NewValReg = MI.getOperand(3).getReg();
+  if (IsPaired)
+    NewValReg = getGPRPairEvenReg(NewValReg);
+  AtomicOrdering Ordering =
+      static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
+
+  MachineInstr *NewMI =
+      BuildMI(MBB, MBBI, DL, TII->get(getAMOCASForRMW(Ordering, Width, STI)))
+          .addReg(DestReg)
+          .addReg(AddrReg)
+          .addReg(NewValReg);
+  NewMI->getOperand(0).setIsDef(true);
+
+  MI.eraseFromParent();
+  return true;
+}
+
 } // end of anonymous namespace
 
 INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo",
                 RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
 
 namespace llvm {
 
 FunctionPass *createRISCVExpandAtomicPseudoPass() {
   return new RISCVExpandAtomicPseudo();
 }
 
 } // end of namespace llvm

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.is64Bit())
     setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
 
   if (Subtarget.hasStdExtZicbop()) {
     setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
   }
 
   if (Subtarget.hasStdExtA()) {
+    if (Subtarget.hasStdExtZacas())
+      setMaxAtomicSizeInBitsSupported(Subtarget.getXLen() * 2);
+    else
       setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
     setMinCmpXchgSizeInBits(32);
   } else if (Subtarget.hasForcedAtomics()) {
     setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
   } else {
     setMaxAtomicSizeInBitsSupported(0);
   }
 
   setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
if (Subtarget.hasForcedAtomics()) {
     setOperationAction(
         {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
          ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
          ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
          ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
         XLenVT, Expand);
   }
 
+  // Set atomic_cmp_swap operations to expand to AMOCAS.D (RV32) and AMOCAS.Q
+  // (RV64).
+  if (Subtarget.hasStdExtZacas())
+    setOperationAction(ISD::ATOMIC_CMP_SWAP,
+                       Subtarget.is64Bit() ? MVT::i128 : MVT::i64, Custom);
+
   if (Subtarget.hasVendorXTHeadMemIdx()) {
     for (unsigned im = (unsigned)ISD::PRE_INC; im != (unsigned)ISD::POST_DEC;
          ++im) {
       setIndexedLoadAction(im, MVT::i8, Legal);
       setIndexedStoreAction(im, MVT::i8, Legal);
       setIndexedLoadAction(im, MVT::i16, Legal);
       setIndexedStoreAction(im, MVT::i16, Legal);
       setIndexedLoadAction(im, MVT::i32, Legal);
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
   SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
   SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
   SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
   SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
                                DAG.getValueType(MVT::i32));
   return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
 }
 
+// Create an even/odd pair of X registers holding integer value V.
+static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V, MVT VT,
+                                 MVT SubRegVT) {
+  SDLoc DL(V.getNode());
+  SDValue VLo = DAG.getAnyExtOrTrunc(V, DL, SubRegVT);
+  SDValue VHi = DAG.getAnyExtOrTrunc(
+      DAG.getNode(
+          ISD::SRL, DL, VT, V,
+          DAG.getConstant(SubRegVT == MVT::i64 ? 64 : 32, DL, SubRegVT)),
+      DL, SubRegVT);
+  SDValue RegClass = DAG.getTargetConstant(
+      VT == MVT::i128 ? RISCV::GPRPI128RegClassID : RISCV::GPRPI64RegClassID,
+      DL, MVT::i32);
+  SDValue SubReg0 = DAG.getTargetConstant(RISCV::sub_32, DL, MVT::i32);
+  SDValue SubReg1 = DAG.getTargetConstant(RISCV::sub_32_hi, DL, MVT::i32);
+  const SDValue Ops[] = {RegClass, VLo, SubReg0, VHi, SubReg1};
+  return SDValue(
+      DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops), 0);
+}
+
+static void ReplaceCMP_SWAP_2XLenResults(SDNode *N,
+                                         SmallVectorImpl<SDValue> &Results,
+                                         SelectionDAG &DAG,
+                                         const RISCVSubtarget &Subtarget) {
+  MVT VT = N->getSimpleValueType(0);
+  assert(N->getValueType(0) == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) &&
+         "AtomicCmpSwap on types less than 2*XLen should be legal");
+  assert(Subtarget.hasStdExtZacas());
+  MVT SubRegVT = (VT == MVT::i64 ? MVT::i32 : MVT::i64);
+
+  SDLoc DL(N);
+  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
+  AtomicOrdering Ordering = MemOp->getMergedOrdering();
+  SDValue Ops[] = {
+      N->getOperand(1),                                       // Ptr
+      createGPRPairNode(DAG, N->getOperand(2), VT, SubRegVT), // Compare value
+      createGPRPairNode(DAG, N->getOperand(3), VT, SubRegVT), // Store value
+      DAG.getTargetConstant(static_cast<unsigned>(Ordering), DL,
+                            MVT::i32), // Ordering
+      N->getOperand(0),                // Chain in
+  };
+
+  unsigned Opcode =
+      (VT == MVT::i64 ? RISCV::PseudoAMOCAS_D_32 : RISCV::PseudoAMOCAS_Q);
+  MachineSDNode *CmpSwap = DAG.getMachineNode(
+      Opcode, DL, DAG.getVTList(MVT::Untyped, MVT::Other), Ops);
+  DAG.setNodeMemRefs(CmpSwap, {MemOp});
+
+  unsigned SubReg1 = RISCV::sub_32, SubReg2 = RISCV::sub_32_hi;
+  SDValue Lo =
+      DAG.getTargetExtractSubreg(SubReg1, DL, SubRegVT, SDValue(CmpSwap, 0));
+  SDValue Hi =
+      DAG.getTargetExtractSubreg(SubReg2, DL, SubRegVT, SDValue(CmpSwap, 0));
+  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, VT, Lo, Hi));
+  Results.push_back(SDValue(CmpSwap, 1));
+}
+
 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
                                              SmallVectorImpl<SDValue> &Results,
                                              SelectionDAG &DAG) const {
   SDLoc DL(N);
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Don't know how to custom type legalize this operation!");
+  case ISD::ATOMIC_CMP_SWAP: {
+    ReplaceCMP_SWAP_2XLenResults(N, Results, DAG, Subtarget);
+    break;
+  }
   case ISD::STRICT_FP_TO_SINT:
   case ISD::STRICT_FP_TO_UINT:
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT: {
     assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
            "Unexpected custom legalisation");
     bool IsStrict = N->isStrictFPOpcode();
     bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||

llvm/lib/Target/RISCV/RISCVInstrInfoA.td

defm AMOMAX_D   : AMO_rr_aq_rl<0b10100, 0b011, "amomax.d">,
                   Sched<[WriteAtomicD, ReadAtomicDA, ReadAtomicDD]>;
 defm AMOMINU_D  : AMO_rr_aq_rl<0b11000, 0b011, "amominu.d">,
                   Sched<[WriteAtomicD, ReadAtomicDA, ReadAtomicDD]>;
 defm AMOMAXU_D  : AMO_rr_aq_rl<0b11100, 0b011, "amomaxu.d">,
                   Sched<[WriteAtomicD, ReadAtomicDA, ReadAtomicDD]>;
 } // Predicates = [HasStdExtA, IsRV64]
 
 let Predicates = [HasStdExtZacas] in {
-defm AMOCAS_W : AMO_rr_aq_rl<0b00101, 0b010, "amocas.w">;
-defm AMOCAS_D : AMO_rr_aq_rl<0b00101, 0b011, "amocas.d">;
+defm AMOCAS_W : AMO_rr_aq_rl<0b00101, 0b010, "amocas.w">,
+                Sched<[WriteAtomicW, ReadAtomicWA, ReadAtomicWD]>;
+defm AMOCAS_D : AMO_rr_aq_rl<0b00101, 0b011, "amocas.d">,
+                Sched<[WriteAtomicD, ReadAtomicDA, ReadAtomicDD]>;
 } // Predicates = [HasStdExtZacas]
 
 let Predicates = [HasStdExtZacas, IsRV64] in {
-defm AMOCAS_Q : AMO_rr_aq_rl<0b00101, 0b100, "amocas.q">;
+defm AMOCAS_Q : AMO_rr_aq_rl<0b00101, 0b100, "amocas.q">,
+                Sched<[WriteAtomicD, ReadAtomicDA, ReadAtomicDD]>;
 } // Predicates = [HasStdExtZacas, IsRV64]
 
 //===----------------------------------------------------------------------===//
 // Pseudo-instructions and codegen patterns
 //===----------------------------------------------------------------------===//
 
 // Atomic load/store are available under both +a and +force-atomics.
 // Fences will be inserted for atomic load/stores according to the logic in
multiclass PseudoCmpXchgPat<string Op, Pseudo CmpXchgInst,
   def : Pat<(vt (!cast<PatFrag>(Op#"_release") GPR:$addr, GPR:$cmp, GPR:$new)),
             (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 5)>;
   def : Pat<(vt (!cast<PatFrag>(Op#"_acq_rel") GPR:$addr, GPR:$cmp, GPR:$new)),
             (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 6)>;
   def : Pat<(vt (!cast<PatFrag>(Op#"_seq_cst") GPR:$addr, GPR:$cmp, GPR:$new)),
             (CmpXchgInst GPR:$addr, GPR:$cmp, GPR:$new, 7)>;
 }
 
+let Predicates = [HasStdExtZacas] in {
+class PseudoAMOCAS<RegisterClass RC = GPR>
+    : Pseudo<(outs RC:$res),
+             (ins GPR:$addr, RC:$cmpval, RC:$newval, ixlenimm:$ordering), []> {
+  let Constraints = "@earlyclobber $res, $res = $cmpval";
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 0;
+}
+def PseudoAMOCAS_W: PseudoAMOCAS;
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoAMOCAS_W>;
+
+let Predicates = [HasStdExtZacas, IsRV32] in {
+  def PseudoAMOCAS_D_32: PseudoAMOCAS<GPRPI64>;
+}
+let Predicates = [HasStdExtZacas, IsRV64] in {
+  def PseudoAMOCAS_D_64: PseudoAMOCAS;
+  defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoAMOCAS_D_64>;
+  def PseudoAMOCAS_Q: PseudoAMOCAS<GPRPI128>;
+}
+}
+
 def PseudoCmpXchg32 : PseudoCmpXchg;
 defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32>;
 
 def PseudoMaskedCmpXchg32
     : Pseudo<(outs GPR:$res, GPR:$scratch),
              (ins GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask,
               ixlenimm:$ordering), []> {
   let Constraints = "@earlyclobber $res,@earlyclobber $scratch";

llvm/lib/Target/RISCV/RISCVRegisterInfo.td

def GPRPF64 : RegisterClass<"RISCV", [f64], 64, (add
     X10_PD, X12_PD, X14_PD, X16_PD,
     X6_PD,
     X28_PD, X30_PD,
     X8_PD,
     X18_PD, X20_PD, X22_PD, X24_PD, X26_PD,
     X0_PD, X2_PD, X4_PD
 )>;
 
+let RegInfos = RegInfoByHwMode<[RV32], [RegInfo<32, 32, 32>]> in
+def GPRPI64 : RegisterClass<"RISCV", [i64], 32, (add
+    X10_PD, X12_PD, X14_PD, X16_PD,
+    X6_PD,
+    X28_PD, X30_PD,
+    X8_PD,
+    X18_PD, X20_PD, X22_PD, X24_PD, X26_PD,
+    X0_PD, X2_PD, X4_PD
+)>;
+
+let RegInfos = RegInfoByHwMode<[RV64], [RegInfo<64, 64, 64>]> in
+def GPRPI128 : RegisterClass<"RISCV", [i128], 64, (add
+    X10_PD, X12_PD, X14_PD, X16_PD,
+    X6_PD,
+    X28_PD, X30_PD,
+    X8_PD,
+    X18_PD, X20_PD, X22_PD, X24_PD, X26_PD,
+    X0_PD, X2_PD, X4_PD
+)>;
+
 // The register class is added for inline assembly for vector mask types.
 def VM : VReg<VMaskVTs,
            (add (sequence "V%u", 8, 31),
                 (sequence "V%u", 0, 7)), 1>;
 
 foreach m = LMULList in {
   foreach nf = NFList<m>.L in {
     def "VRN" # nf # "M" # m # "NoV0": VReg<[untyped],

llvm/test/CodeGen/RISCV/atomic-cmpxchg-branch-on-result.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK,RV32IA %s
+; RUN:   | FileCheck -check-prefixes=RV32IA %s
+; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV32IA-ZACAS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK,RV64IA %s
+; RUN:   | FileCheck -check-prefixes=RV64IA %s
+; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV64IA-ZACAS %s
 
 ; Test cmpxchg followed by a branch on the cmpxchg success value to see if the
 ; branch is folded into the cmpxchg expansion.
 
 define void @cmpxchg_and_branch1(ptr %ptr, i32 signext %cmp, i32 signext %val) nounwind {
-; CHECK-LABEL: cmpxchg_and_branch1:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:  .LBB0_1: # %do_cmpxchg
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB0_3 Depth 2
-; CHECK-NEXT:  .LBB0_3: # %do_cmpxchg
-; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lr.w.aqrl a3, (a0)
-; CHECK-NEXT:    bne a3, a1, .LBB0_1
-; CHECK-NEXT:  # %bb.4: # %do_cmpxchg
-; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=2
-; CHECK-NEXT:    sc.w.rl a4, a2, (a0)
-; CHECK-NEXT:    bnez a4, .LBB0_3
-; CHECK-NEXT:  # %bb.5: # %do_cmpxchg
-; CHECK-NEXT:  # %bb.2: # %exit
-; CHECK-NEXT:    ret
+; RV32IA-LABEL: cmpxchg_and_branch1:
+; RV32IA:       # %bb.0: # %entry
+; RV32IA-NEXT:  .LBB0_1: # %do_cmpxchg
+; RV32IA-NEXT:    # =>This Loop Header: Depth=1
+; RV32IA-NEXT:    # Child Loop BB0_3 Depth 2
+; RV32IA-NEXT:  .LBB0_3: # %do_cmpxchg
+; RV32IA-NEXT:    # Parent Loop BB0_1 Depth=1
+; RV32IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV32IA-NEXT:    bne a3, a1, .LBB0_1
+; RV32IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV32IA-NEXT:    # in Loop: Header=BB0_3 Depth=2
+; RV32IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV32IA-NEXT:    bnez a4, .LBB0_3
+; RV32IA-NEXT:  # %bb.5: # %do_cmpxchg
+; RV32IA-NEXT:  # %bb.2: # %exit
+; RV32IA-NEXT:    ret
+;
+; RV32IA-ZACAS-LABEL: cmpxchg_and_branch1:
+; RV32IA-ZACAS:       # %bb.0: # %entry
+; RV32IA-ZACAS-NEXT:  .LBB0_1: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    mv a3, a1
+; RV32IA-ZACAS-NEXT:    amocas.w.aqrl a3, a2, (a0)
+; RV32IA-ZACAS-NEXT:    bne a3, a1, .LBB0_1
+; RV32IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV32IA-ZACAS-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_and_branch1:
+; RV64IA:       # %bb.0: # %entry
+; RV64IA-NEXT:  .LBB0_1: # %do_cmpxchg
+; RV64IA-NEXT:    # =>This Loop Header: Depth=1
+; RV64IA-NEXT:    # Child Loop BB0_3 Depth 2
+; RV64IA-NEXT:  .LBB0_3: # %do_cmpxchg
+; RV64IA-NEXT:    # Parent Loop BB0_1 Depth=1
+; RV64IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB0_1
+; RV64IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV64IA-NEXT:    # in Loop: Header=BB0_3 Depth=2
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB0_3
+; RV64IA-NEXT:  # %bb.5: # %do_cmpxchg
+; RV64IA-NEXT:  # %bb.2: # %exit
+; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_and_branch1:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:  .LBB0_1: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    mv a3, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aqrl a3, a2, (a0)
+; RV64IA-ZACAS-NEXT:    bne a3, a1, .LBB0_1
+; RV64IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV64IA-ZACAS-NEXT:    ret
 entry:
   br label %do_cmpxchg
 do_cmpxchg:
   %0 = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
   br i1 %1, label %exit, label %do_cmpxchg
 exit:
   ret void
 }
 
 define void @cmpxchg_and_branch2(ptr %ptr, i32 signext %cmp, i32 signext %val) nounwind {
-; CHECK-LABEL: cmpxchg_and_branch2:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:  .LBB1_1: # %do_cmpxchg
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB1_3 Depth 2
-; CHECK-NEXT:  .LBB1_3: # %do_cmpxchg
-; CHECK-NEXT:    # Parent Loop BB1_1 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lr.w.aqrl a3, (a0)
-; CHECK-NEXT:    bne a3, a1, .LBB1_5
-; CHECK-NEXT:  # %bb.4: # %do_cmpxchg
-; CHECK-NEXT:    # in Loop: Header=BB1_3 Depth=2
-; CHECK-NEXT:    sc.w.rl a4, a2, (a0)
-; CHECK-NEXT:    bnez a4, .LBB1_3
-; CHECK-NEXT:  .LBB1_5: # %do_cmpxchg
-; CHECK-NEXT:    # in Loop: Header=BB1_1 Depth=1
-; CHECK-NEXT:    beq a3, a1, .LBB1_1
-; CHECK-NEXT:  # %bb.2: # %exit
-; CHECK-NEXT:    ret
+; RV32IA-LABEL: cmpxchg_and_branch2:
+; RV32IA:       # %bb.0: # %entry
+; RV32IA-NEXT:  .LBB1_1: # %do_cmpxchg
+; RV32IA-NEXT:    # =>This Loop Header: Depth=1
+; RV32IA-NEXT:    # Child Loop BB1_3 Depth 2
+; RV32IA-NEXT:  .LBB1_3: # %do_cmpxchg
+; RV32IA-NEXT:    # Parent Loop BB1_1 Depth=1
+; RV32IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV32IA-NEXT:    bne a3, a1, .LBB1_5
+; RV32IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV32IA-NEXT:    # in Loop: Header=BB1_3 Depth=2
+; RV32IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV32IA-NEXT:    bnez a4, .LBB1_3
+; RV32IA-NEXT:  .LBB1_5: # %do_cmpxchg
+; RV32IA-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; RV32IA-NEXT:    beq a3, a1, .LBB1_1
+; RV32IA-NEXT:  # %bb.2: # %exit
+; RV32IA-NEXT:    ret
+;
+; RV32IA-ZACAS-LABEL: cmpxchg_and_branch2:
+; RV32IA-ZACAS:       # %bb.0: # %entry
+; RV32IA-ZACAS-NEXT:  .LBB1_1: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    mv a3, a1
+; RV32IA-ZACAS-NEXT:    amocas.w.aqrl a3, a2, (a0)
+; RV32IA-ZACAS-NEXT:    beq a3, a1, .LBB1_1
+; RV32IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV32IA-ZACAS-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_and_branch2:
+; RV64IA:       # %bb.0: # %entry
+; RV64IA-NEXT:  .LBB1_1: # %do_cmpxchg
+; RV64IA-NEXT:    # =>This Loop Header: Depth=1
+; RV64IA-NEXT:    # Child Loop BB1_3 Depth 2
+; RV64IA-NEXT:  .LBB1_3: # %do_cmpxchg
+; RV64IA-NEXT:    # Parent Loop BB1_1 Depth=1
+; RV64IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB1_5
+; RV64IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV64IA-NEXT:    # in Loop: Header=BB1_3 Depth=2
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB1_3
+; RV64IA-NEXT:  .LBB1_5: # %do_cmpxchg
+; RV64IA-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; RV64IA-NEXT:    beq a3, a1, .LBB1_1
+; RV64IA-NEXT:  # %bb.2: # %exit
+; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_and_branch2:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:  .LBB1_1: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    mv a3, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aqrl a3, a2, (a0)
+; RV64IA-ZACAS-NEXT:    beq a3, a1, .LBB1_1
+; RV64IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV64IA-ZACAS-NEXT:    ret
 entry:
   br label %do_cmpxchg
 do_cmpxchg:
   %0 = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
   br i1 %1, label %do_cmpxchg, label %exit
 exit:
   ret void

 ; RV32IA-NEXT:    and a5, a5, a0
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB2_3
 ; RV32IA-NEXT:  # %bb.5: # %do_cmpxchg
 ; RV32IA-NEXT:  # %bb.2: # %exit
 ; RV32IA-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_masked_and_branch1:
+; RV32IA-ZACAS:       # %bb.0: # %entry
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a4, a0, 3
+; RV32IA-ZACAS-NEXT:    li a0, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a0, a4
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a4
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a2, a2, a4
+; RV32IA-ZACAS-NEXT:  .LBB2_1: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # =>This Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    # Child Loop BB2_3 Depth 2
+; RV32IA-ZACAS-NEXT:  .LBB2_3: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # Parent Loop BB2_1 Depth=1
+; RV32IA-ZACAS-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32IA-ZACAS-NEXT:    lr.w.aqrl a4, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a4, a0
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB2_1
+; RV32IA-ZACAS-NEXT:  # %bb.4: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # in Loop: Header=BB2_3 Depth=2
+; RV32IA-ZACAS-NEXT:    xor a5, a4, a2
+; RV32IA-ZACAS-NEXT:    and a5, a5, a0
+; RV32IA-ZACAS-NEXT:    xor a5, a4, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB2_3
+; RV32IA-ZACAS-NEXT:  # %bb.5: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-LABEL: cmpxchg_masked_and_branch1:
 ; RV64IA:       # %bb.0: # %entry
 ; RV64IA-NEXT:    andi a3, a0, -4
 ; RV64IA-NEXT:    slli a4, a0, 3
 ; RV64IA-NEXT:    li a0, 255
 ; RV64IA-NEXT:    sllw a0, a0, a4
 ; RV64IA-NEXT:    andi a1, a1, 255
 ; RV64IA-NEXT:    sllw a1, a1, a4

 ; RV64IA-NEXT:    xor a5, a4, a2
 ; RV64IA-NEXT:    and a5, a5, a0
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB2_3
 ; RV64IA-NEXT:  # %bb.5: # %do_cmpxchg
 ; RV64IA-NEXT:  # %bb.2: # %exit
 ; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_masked_and_branch1:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a4, a0, 3
+; RV64IA-ZACAS-NEXT:    li a0, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a0, a4
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a4
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a2, a2, a4
+; RV64IA-ZACAS-NEXT:  .LBB2_1: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # =>This Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    # Child Loop BB2_3 Depth 2
+; RV64IA-ZACAS-NEXT:  .LBB2_3: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # Parent Loop BB2_1 Depth=1
+; RV64IA-ZACAS-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64IA-ZACAS-NEXT:    lr.w.aqrl a4, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a4, a0
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB2_1
+; RV64IA-ZACAS-NEXT:  # %bb.4: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # in Loop: Header=BB2_3 Depth=2
+; RV64IA-ZACAS-NEXT:    xor a5, a4, a2
+; RV64IA-ZACAS-NEXT:    and a5, a5, a0
+; RV64IA-ZACAS-NEXT:    xor a5, a4, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB2_3
+; RV64IA-ZACAS-NEXT:  # %bb.5: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV64IA-ZACAS-NEXT:    ret
 entry:
   br label %do_cmpxchg
 do_cmpxchg:
   %0 = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst seq_cst
   %1 = extractvalue { i8, i1 } %0, 1
   br i1 %1, label %exit, label %do_cmpxchg
 exit:
   ret void

 ; RV32IA-NEXT:    bnez a5, .LBB3_3
 ; RV32IA-NEXT:  .LBB3_5: # %do_cmpxchg
 ; RV32IA-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; RV32IA-NEXT:    and a4, a4, a0
 ; RV32IA-NEXT:    beq a1, a4, .LBB3_1
 ; RV32IA-NEXT:  # %bb.2: # %exit
 ; RV32IA-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_masked_and_branch2:
+; RV32IA-ZACAS:       # %bb.0: # %entry
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a4, a0, 3
+; RV32IA-ZACAS-NEXT:    li a0, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a0, a4
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a4
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a2, a2, a4
+; RV32IA-ZACAS-NEXT:  .LBB3_1: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # =>This Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    # Child Loop BB3_3 Depth 2
+; RV32IA-ZACAS-NEXT:  .LBB3_3: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # Parent Loop BB3_1 Depth=1
+; RV32IA-ZACAS-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32IA-ZACAS-NEXT:    lr.w.aqrl a4, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a4, a0
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB3_5
+; RV32IA-ZACAS-NEXT:  # %bb.4: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # in Loop: Header=BB3_3 Depth=2
+; RV32IA-ZACAS-NEXT:    xor a5, a4, a2
+; RV32IA-ZACAS-NEXT:    and a5, a5, a0
+; RV32IA-ZACAS-NEXT:    xor a5, a4, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB3_3
+; RV32IA-ZACAS-NEXT:  .LBB3_5: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; RV32IA-ZACAS-NEXT:    and a4, a4, a0
+; RV32IA-ZACAS-NEXT:    beq a1, a4, .LBB3_1
+; RV32IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-LABEL: cmpxchg_masked_and_branch2:
 ; RV64IA:       # %bb.0: # %entry
 ; RV64IA-NEXT:    andi a3, a0, -4
 ; RV64IA-NEXT:    slli a4, a0, 3
 ; RV64IA-NEXT:    li a0, 255
 ; RV64IA-NEXT:    sllw a0, a0, a4
 ; RV64IA-NEXT:    andi a1, a1, 255
 ; RV64IA-NEXT:    sllw a1, a1, a4

 ; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB3_3
 ; RV64IA-NEXT:  .LBB3_5: # %do_cmpxchg
 ; RV64IA-NEXT:    # in Loop: Header=BB3_1 Depth=1
 ; RV64IA-NEXT:    and a4, a4, a0
 ; RV64IA-NEXT:    beq a1, a4, .LBB3_1
 ; RV64IA-NEXT:  # %bb.2: # %exit
 ; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_masked_and_branch2:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a4, a0, 3
+; RV64IA-ZACAS-NEXT:    li a0, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a0, a4
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a4
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a2, a2, a4
+; RV64IA-ZACAS-NEXT:  .LBB3_1: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # =>This Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    # Child Loop BB3_3 Depth 2
+; RV64IA-ZACAS-NEXT:  .LBB3_3: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # Parent Loop BB3_1 Depth=1
+; RV64IA-ZACAS-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64IA-ZACAS-NEXT:    lr.w.aqrl a4, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a4, a0
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB3_5
+; RV64IA-ZACAS-NEXT:  # %bb.4: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # in Loop: Header=BB3_3 Depth=2
+; RV64IA-ZACAS-NEXT:    xor a5, a4, a2
+; RV64IA-ZACAS-NEXT:    and a5, a5, a0
+; RV64IA-ZACAS-NEXT:    xor a5, a4, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB3_3
+; RV64IA-ZACAS-NEXT:  .LBB3_5: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; RV64IA-ZACAS-NEXT:    and a4, a4, a0
+; RV64IA-ZACAS-NEXT:    beq a1, a4, .LBB3_1
+; RV64IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV64IA-ZACAS-NEXT:    ret
 entry:
   br label %do_cmpxchg
 do_cmpxchg:
   %0 = cmpxchg ptr %ptr, i8 %cmp, i8 %val seq_cst seq_cst
   %1 = extractvalue { i8, i1 } %0, 1
   br i1 %1, label %do_cmpxchg, label %exit
 exit:
   ret void
 }
 
 define void @cmpxchg_and_irrelevant_branch(ptr %ptr, i32 signext %cmp, i32 signext %val, i1 zeroext %bool) nounwind {
-; CHECK-LABEL: cmpxchg_and_irrelevant_branch:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:  .LBB4_1: # %do_cmpxchg
-; CHECK-NEXT:    # =>This Loop Header: Depth=1
-; CHECK-NEXT:    # Child Loop BB4_3 Depth 2
-; CHECK-NEXT:  .LBB4_3: # %do_cmpxchg
-; CHECK-NEXT:    # Parent Loop BB4_1 Depth=1
-; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    lr.w.aqrl a4, (a0)
-; CHECK-NEXT:    bne a4, a1, .LBB4_5
-; CHECK-NEXT:  # %bb.4: # %do_cmpxchg
-; CHECK-NEXT:    # in Loop: Header=BB4_3 Depth=2
-; CHECK-NEXT:    sc.w.rl a5, a2, (a0)
-; CHECK-NEXT:    bnez a5, .LBB4_3
-; CHECK-NEXT:  .LBB4_5: # %do_cmpxchg
-; CHECK-NEXT:    # in Loop: Header=BB4_1 Depth=1
-; CHECK-NEXT:    beqz a3, .LBB4_1
-; CHECK-NEXT:  # %bb.2: # %exit
-; CHECK-NEXT:    ret
+; RV32IA-LABEL: cmpxchg_and_irrelevant_branch:
+; RV32IA:       # %bb.0: # %entry
+; RV32IA-NEXT:  .LBB4_1: # %do_cmpxchg
+; RV32IA-NEXT:    # =>This Loop Header: Depth=1
+; RV32IA-NEXT:    # Child Loop BB4_3 Depth 2
+; RV32IA-NEXT:  .LBB4_3: # %do_cmpxchg
+; RV32IA-NEXT:    # Parent Loop BB4_1 Depth=1
+; RV32IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    bne a4, a1, .LBB4_5
+; RV32IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV32IA-NEXT:    # in Loop: Header=BB4_3 Depth=2
+; RV32IA-NEXT:    sc.w.rl a5, a2, (a0)
+; RV32IA-NEXT:    bnez a5, .LBB4_3
+; RV32IA-NEXT:  .LBB4_5: # %do_cmpxchg
+; RV32IA-NEXT:    # in Loop: Header=BB4_1 Depth=1
+; RV32IA-NEXT:    beqz a3, .LBB4_1
+; RV32IA-NEXT:  # %bb.2: # %exit
+; RV32IA-NEXT:    ret
+;
+; RV32IA-ZACAS-LABEL: cmpxchg_and_irrelevant_branch:
+; RV32IA-ZACAS:       # %bb.0: # %entry
+; RV32IA-ZACAS-NEXT:  .LBB4_1: # %do_cmpxchg
+; RV32IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    mv a4, a1
+; RV32IA-ZACAS-NEXT:    amocas.w.aqrl a4, a2, (a0)
+; RV32IA-ZACAS-NEXT:    beqz a3, .LBB4_1
+; RV32IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV32IA-ZACAS-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_and_irrelevant_branch:
+; RV64IA:       # %bb.0: # %entry
+; RV64IA-NEXT:  .LBB4_1: # %do_cmpxchg
+; RV64IA-NEXT:    # =>This Loop Header: Depth=1
+; RV64IA-NEXT:    # Child Loop BB4_3 Depth 2
+; RV64IA-NEXT:  .LBB4_3: # %do_cmpxchg
+; RV64IA-NEXT:    # Parent Loop BB4_1 Depth=1
+; RV64IA-NEXT:    # => This Inner Loop Header: Depth=2
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    bne a4, a1, .LBB4_5
+; RV64IA-NEXT:  # %bb.4: # %do_cmpxchg
+; RV64IA-NEXT:    # in Loop: Header=BB4_3 Depth=2
+; RV64IA-NEXT:    sc.w.rl a5, a2, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB4_3
+; RV64IA-NEXT:  .LBB4_5: # %do_cmpxchg
+; RV64IA-NEXT:    # in Loop: Header=BB4_1 Depth=1
+; RV64IA-NEXT:    beqz a3, .LBB4_1
+; RV64IA-NEXT:  # %bb.2: # %exit
+; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_and_irrelevant_branch:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:  .LBB4_1: # %do_cmpxchg
+; RV64IA-ZACAS-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    mv a4, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aqrl a4, a2, (a0)
+; RV64IA-ZACAS-NEXT:    beqz a3, .LBB4_1
+; RV64IA-ZACAS-NEXT:  # %bb.2: # %exit
+; RV64IA-ZACAS-NEXT:    ret
 entry:
   br label %do_cmpxchg
 do_cmpxchg:
   %0 = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
   br i1 %bool, label %exit, label %do_cmpxchg
 exit:
   ret void
 }

llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64IA %s
+; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IA-ZACAS %s
 
 ; This test ensures that the output of the 'lr.w' instruction is sign-extended.
 ; Previously, the default zero-extension was being used and 'cmp' parameter
 ; higher bits were masked to zero for the comparison.
 
 define i1 @cmpxchg_i32_seq_cst_seq_cst(ptr %ptr, i32 signext %cmp,
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0: # %entry
 ; RV64IA-NEXT:  .LBB0_1: # %entry
 ; RV64IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
 ; RV64IA-NEXT:    bne a3, a1, .LBB0_3
 ; RV64IA-NEXT:  # %bb.2: # %entry
 ; RV64IA-NEXT:    # in Loop: Header=BB0_1 Depth=1
 ; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
 ; RV64IA-NEXT:    bnez a4, .LBB0_1
 ; RV64IA-NEXT:  .LBB0_3: # %entry
 ; RV64IA-NEXT:    xor a1, a3, a1
 ; RV64IA-NEXT:    seqz a0, a1
 ; RV64IA-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_i32_seq_cst_seq_cst:
+; RV64IA-ZACAS:       # %bb.0: # %entry
+; RV64IA-ZACAS-NEXT:    mv a3, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aqrl a3, a2, (a0)
+; RV64IA-ZACAS-NEXT:    xor a1, a3, a1
+; RV64IA-ZACAS-NEXT:    seqz a0, a1
+; RV64IA-ZACAS-NEXT:    ret
         i32 signext %val) nounwind {
 entry:
   %0 = cmpxchg ptr %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
   ret i1 %1
 }

llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV32IA,RV32IA-WMO %s
+; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV32IA,RV32IA-ZACAS %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV32IA,RV32IA-TSO %s
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV64I %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64IA,RV64IA-WMO %s
+; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-zacas -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=RV64IA,RV64IA-ZACAS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a,+experimental-ztso -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefixes=RV64IA,RV64IA-TSO %s
 
 define void @cmpxchg_i8_monotonic_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-WMO-NEXT:  .LBB1_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB1_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB1_1
+; RV32IA-ZACAS-NEXT:  .LBB1_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-WMO-NEXT:  .LBB1_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB1_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB1_1
+; RV64IA-ZACAS-NEXT:  .LBB1_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB1_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-TSO-NEXT:  .LBB1_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB1_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB1_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB1_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB1_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB1_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB1_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire monotonic
   ret void
 }
 
 define void @cmpxchg_i8_acquire_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-WMO-NEXT:  .LBB2_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_acquire_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB2_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB2_1
+; RV32IA-ZACAS-NEXT:  .LBB2_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-WMO-NEXT:  .LBB2_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_acquire_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB2_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB2_1
+; RV64IA-ZACAS-NEXT:  .LBB2_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB2_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-TSO-NEXT:  .LBB2_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_acquire_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB2_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB2_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB2_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_acquire_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB2_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB2_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB2_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acquire acquire
   ret void
 }
 
 define void @cmpxchg_i8_release_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-WMO-NEXT:  .LBB3_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_release_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB3_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB3_1
+; RV32IA-ZACAS-NEXT:  .LBB3_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_release_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-WMO-NEXT:  .LBB3_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_release_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB3_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB3_1
+; RV64IA-ZACAS-NEXT:  .LBB3_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_release_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB3_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-TSO-NEXT:  .LBB3_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_release_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB3_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB3_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB3_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_release_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB3_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB3_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB3_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val release monotonic
   ret void
 }
 
 define void @cmpxchg_i8_release_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-WMO-NEXT:  .LBB4_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_release_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB4_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB4_1
+; RV32IA-ZACAS-NEXT:  .LBB4_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_release_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-WMO-NEXT:  .LBB4_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_release_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB4_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB4_1
+; RV64IA-ZACAS-NEXT:  .LBB4_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_release_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB4_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-TSO-NEXT:  .LBB4_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_release_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB4_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB4_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB4_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_release_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB4_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB4_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB4_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val release acquire
   ret void
 }
 
 define void @cmpxchg_i8_acq_rel_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-WMO-NEXT:  .LBB5_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB5_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB5_1
+; RV32IA-ZACAS-NEXT:  .LBB5_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-WMO-NEXT:  .LBB5_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB5_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB5_1
+; RV64IA-ZACAS-NEXT:  .LBB5_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB5_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-TSO-NEXT:  .LBB5_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB5_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB5_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB5_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB5_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB5_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB5_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acq_rel monotonic
   ret void
 }
 
 define void @cmpxchg_i8_acq_rel_acquire(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a5, a2, a0
 ; RV32IA-WMO-NEXT:    and a5, a5, a4
 ; RV32IA-WMO-NEXT:    xor a5, a2, a5
 ; RV32IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-WMO-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-WMO-NEXT:  .LBB6_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    li a4, 255
+; RV32IA-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-ZACAS-NEXT:    bne a5, a1, .LBB6_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a5, .LBB6_1
+; RV32IA-ZACAS-NEXT:  .LBB6_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    li a4, 255
 ; RV32IA-TSO-NEXT:    sll a4, a4, a0
 ; RV32IA-TSO-NEXT:    andi a1, a1, 255
 ; RV32IA-TSO-NEXT:    sll a1, a1, a0

 ; RV64IA-WMO-NEXT:    xor a5, a2, a0
 ; RV64IA-WMO-NEXT:    and a5, a5, a4
 ; RV64IA-WMO-NEXT:    xor a5, a2, a5
 ; RV64IA-WMO-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-WMO-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-WMO-NEXT:  .LBB6_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    li a4, 255
+; RV64IA-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-ZACAS-NEXT:    bne a5, a1, .LBB6_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a5, .LBB6_1
+; RV64IA-ZACAS-NEXT:  .LBB6_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    li a4, 255
 ; RV64IA-TSO-NEXT:    sllw a4, a4, a0
 ; RV64IA-TSO-NEXT:    andi a1, a1, 255
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    andi a2, a2, 255
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a5, a2, a4
 ; RV64IA-TSO-NEXT:    bne a5, a1, .LBB6_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a5, a2, a0
 ; RV64IA-TSO-NEXT:    and a5, a5, a4
 ; RV64IA-TSO-NEXT:    xor a5, a2, a5
 ; RV64IA-TSO-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-TSO-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-TSO-NEXT:  .LBB6_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a4, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB6_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV32IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a5, .LBB6_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB6_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    li a4, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a4, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a1, a1, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    andi a2, a2, 255
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    bne a5, a1, .LBB6_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a5, a5, a4
+; RV64IA-WMO-ZACAS-NEXT:    xor a5, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a5, a5, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a5, .LBB6_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB6_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val acq_rel acquire
   ret void
 }
 
 define void @cmpxchg_i8_seq_cst_monotonic(ptr %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB11_1
 ; RV32IA-WMO-NEXT:  .LBB11_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB11_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB11_1
+; RV32IA-ZACAS-NEXT:  .LBB11_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB11_1
 ; RV64IA-WMO-NEXT:  .LBB11_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB11_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB11_1
+; RV64IA-ZACAS-NEXT:  .LBB11_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB11_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB11_1
 ; RV64IA-TSO-NEXT:  .LBB11_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB11_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB11_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB11_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB11_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB11_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB11_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire monotonic
   ret void
 }
 
 define void @cmpxchg_i16_acquire_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB12_1
 ; RV32IA-WMO-NEXT:  .LBB12_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_acquire_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB12_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB12_1
+; RV32IA-ZACAS-NEXT:  .LBB12_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB12_1
 ; RV64IA-WMO-NEXT:  .LBB12_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_acquire_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB12_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB12_1
+; RV64IA-ZACAS-NEXT:  .LBB12_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB12_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB12_1
 ; RV64IA-TSO-NEXT:  .LBB12_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_acquire_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB12_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB12_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB12_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_acquire_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB12_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB12_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB12_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acquire acquire
   ret void
 }
 
 define void @cmpxchg_i16_release_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB13_1
 ; RV32IA-WMO-NEXT:  .LBB13_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_release_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB13_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB13_1
+; RV32IA-ZACAS-NEXT:  .LBB13_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_release_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB13_1
 ; RV64IA-WMO-NEXT:  .LBB13_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_release_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB13_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB13_1
+; RV64IA-ZACAS-NEXT:  .LBB13_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_release_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB13_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB13_1
 ; RV64IA-TSO-NEXT:  .LBB13_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_release_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB13_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB13_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB13_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_release_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB13_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB13_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB13_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val release monotonic
   ret void
 }
 
 define void @cmpxchg_i16_release_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB14_1
 ; RV32IA-WMO-NEXT:  .LBB14_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_release_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB14_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB14_1
+; RV32IA-ZACAS-NEXT:  .LBB14_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_release_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB14_1
 ; RV64IA-WMO-NEXT:  .LBB14_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_release_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB14_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB14_1
+; RV64IA-ZACAS-NEXT:  .LBB14_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_release_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB14_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB14_1
 ; RV64IA-TSO-NEXT:  .LBB14_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_release_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB14_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB14_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB14_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_release_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB14_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB14_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB14_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val release acquire
   ret void
 }
 
 define void @cmpxchg_i16_acq_rel_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB15_1
 ; RV32IA-WMO-NEXT:  .LBB15_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB15_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB15_1
+; RV32IA-ZACAS-NEXT:  .LBB15_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB15_1
 ; RV64IA-WMO-NEXT:  .LBB15_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB15_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB15_1
+; RV64IA-ZACAS-NEXT:  .LBB15_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB15_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB15_1
 ; RV64IA-TSO-NEXT:  .LBB15_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB15_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB15_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB15_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB15_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB15_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB15_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acq_rel monotonic
   ret void
 }
 
 define void @cmpxchg_i16_acq_rel_acquire(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    xor a4, a2, a0
 ; RV32IA-WMO-NEXT:    and a4, a4, a5
 ; RV32IA-WMO-NEXT:    xor a4, a2, a4
 ; RV32IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB16_1
 ; RV32IA-WMO-NEXT:  .LBB16_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-ZACAS-NEXT:    lui a4, 16
+; RV32IA-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-ZACAS-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-ZACAS-NEXT:    bne a4, a1, .LBB16_3
+; RV32IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-ZACAS-NEXT:    bnez a4, .LBB16_1
+; RV32IA-ZACAS-NEXT:  .LBB16_3:
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:    andi a3, a0, -4
 ; RV32IA-TSO-NEXT:    slli a0, a0, 3
 ; RV32IA-TSO-NEXT:    lui a4, 16
 ; RV32IA-TSO-NEXT:    addi a4, a4, -1
 ; RV32IA-TSO-NEXT:    sll a5, a4, a0
 ; RV32IA-TSO-NEXT:    and a1, a1, a4

 ; RV64IA-WMO-NEXT:    xor a4, a2, a0
 ; RV64IA-WMO-NEXT:    and a4, a4, a5
 ; RV64IA-WMO-NEXT:    xor a4, a2, a4
 ; RV64IA-WMO-NEXT:    sc.w.rl a4, a4, (a3)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB16_1
 ; RV64IA-WMO-NEXT:  .LBB16_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-ZACAS-NEXT:    lui a4, 16
+; RV64IA-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-ZACAS-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-ZACAS-NEXT:    bne a4, a1, .LBB16_3
+; RV64IA-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-ZACAS-NEXT:    bnez a4, .LBB16_1
+; RV64IA-ZACAS-NEXT:  .LBB16_3:
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    andi a3, a0, -4
 ; RV64IA-TSO-NEXT:    slli a0, a0, 3
 ; RV64IA-TSO-NEXT:    lui a4, 16
 ; RV64IA-TSO-NEXT:    addiw a4, a4, -1
 ; RV64IA-TSO-NEXT:    sllw a5, a4, a0
 ; RV64IA-TSO-NEXT:    and a1, a1, a4
 ; RV64IA-TSO-NEXT:    sllw a1, a1, a0
 ; RV64IA-TSO-NEXT:    and a2, a2, a4
 ; RV64IA-TSO-NEXT:    sllw a0, a2, a0
 ; RV64IA-TSO-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a2, (a3)
 ; RV64IA-TSO-NEXT:    and a4, a2, a5
 ; RV64IA-TSO-NEXT:    bne a4, a1, .LBB16_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
 ; RV64IA-TSO-NEXT:    xor a4, a2, a0
 ; RV64IA-TSO-NEXT:    and a4, a4, a5
 ; RV64IA-TSO-NEXT:    xor a4, a2, a4
 ; RV64IA-TSO-NEXT:    sc.w a4, a4, (a3)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB16_1
 ; RV64IA-TSO-NEXT:  .LBB16_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV32IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV32IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV32IA-WMO-ZACAS-NEXT:    addi a4, a4, -1
+; RV32IA-WMO-ZACAS-NEXT:    sll a5, a4, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a1, a1, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sll a0, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV32IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB16_3
+; RV32IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV32IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV32IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV32IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-WMO-ZACAS-NEXT:    bnez a4, .LBB16_1
+; RV32IA-WMO-ZACAS-NEXT:  .LBB16_3:
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    andi a3, a0, -4
+; RV64IA-WMO-ZACAS-NEXT:    slli a0, a0, 3
+; RV64IA-WMO-ZACAS-NEXT:    lui a4, 16
+; RV64IA-WMO-ZACAS-NEXT:    addiw a4, a4, -1
+; RV64IA-WMO-ZACAS-NEXT:    sllw a5, a4, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a1, a1, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a1, a1, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a2, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sllw a0, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a2, a5
+; RV64IA-WMO-ZACAS-NEXT:    bne a4, a1, .LBB16_3
+; RV64IA-WMO-ZACAS-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a0
+; RV64IA-WMO-ZACAS-NEXT:    and a4, a4, a5
+; RV64IA-WMO-ZACAS-NEXT:    xor a4, a2, a4
+; RV64IA-WMO-ZACAS-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-WMO-ZACAS-NEXT:    bnez a4, .LBB16_1
+; RV64IA-WMO-ZACAS-NEXT:  .LBB16_3:
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i16 %cmp, i16 %val acq_rel acquire
   ret void
 }
 
 define void @cmpxchg_i16_seq_cst_monotonic(ptr %ptr, i16 %cmp, i16 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    li a4, 0
 ; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
-; RV32IA-LABEL: cmpxchg_i32_monotonic_monotonic:
-; RV32IA:       # %bb.0:
-; RV32IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    bne a3, a1, .LBB20_3
-; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
-; RV32IA-NEXT:    sc.w a4, a2, (a0)
-; RV32IA-NEXT:    bnez a4, .LBB20_1
-; RV32IA-NEXT:  .LBB20_3:
-; RV32IA-NEXT:    ret
+; RV32IA-WMO-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV32IA-WMO:       # %bb.0:
+; RV32IA-WMO-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-WMO-NEXT:    lr.w a3, (a0)
+; RV32IA-WMO-NEXT:    bne a3, a1, .LBB20_3
+; RV32IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; RV32IA-WMO-NEXT:    sc.w a4, a2, (a0)
+; RV32IA-WMO-NEXT:    bnez a4, .LBB20_1
+; RV32IA-WMO-NEXT:  .LBB20_3:
+; RV32IA-WMO-NEXT:    ret
+;
+; RV32IA-ZACAS-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    amocas.w a1, a2, (a0)
+; RV32IA-ZACAS-NEXT:    ret
+;
+; RV32IA-TSO-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV32IA-TSO:       # %bb.0:
+; RV32IA-TSO-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV32IA-TSO-NEXT:    lr.w a3, (a0)
+; RV32IA-TSO-NEXT:    bne a3, a1, .LBB20_3
+; RV32IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; RV32IA-TSO-NEXT:    sc.w a4, a2, (a0)
+; RV32IA-TSO-NEXT:    bnez a4, .LBB20_1
+; RV32IA-TSO-NEXT:  .LBB20_3:
+; RV32IA-TSO-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i32_monotonic_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    li a3, 0
 ; RV64I-NEXT:    li a4, 0
 ; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
-; RV64IA-LABEL: cmpxchg_i32_monotonic_monotonic:
-; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    sext.w a1, a1
-; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    bne a3, a1, .LBB20_3
-; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
-; RV64IA-NEXT:    sc.w a4, a2, (a0)
-; RV64IA-NEXT:    bnez a4, .LBB20_1
-; RV64IA-NEXT:  .LBB20_3:
-; RV64IA-NEXT:    ret
+; RV64IA-WMO-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64IA-WMO:       # %bb.0:
+; RV64IA-WMO-NEXT:    sext.w a1, a1
+; RV64IA-WMO-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-WMO-NEXT:    lr.w a3, (a0)
+; RV64IA-WMO-NEXT:    bne a3, a1, .LBB20_3
+; RV64IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; RV64IA-WMO-NEXT:    sc.w a4, a2, (a0)
+; RV64IA-WMO-NEXT:    bnez a4, .LBB20_1
+; RV64IA-WMO-NEXT:  .LBB20_3:
+; RV64IA-WMO-NEXT:    ret
+;
+; RV64IA-ZACAS-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-ZACAS-NEXT:    amocas.w a1, a2, (a0)
+; RV64IA-ZACAS-NEXT:    ret
+;
+; RV64IA-TSO-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64IA-TSO:       # %bb.0:
+; RV64IA-TSO-NEXT:    sext.w a1, a1
+; RV64IA-TSO-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-TSO-NEXT:    lr.w a3, (a0)
+; RV64IA-TSO-NEXT:    bne a3, a1, .LBB20_3
+; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; RV64IA-TSO-NEXT:    sc.w a4, a2, (a0)
+; RV64IA-TSO-NEXT:    bnez a4, .LBB20_1
+; RV64IA-TSO-NEXT:  .LBB20_3:
+; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    amocas.w a1, a2, (a0)
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-WMO-ZACAS-NEXT:    amocas.w a1, a2, (a0)
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val monotonic monotonic
   ret void
 }
 
 define void @cmpxchg_i32_acquire_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    lr.w.aq a3, (a0)
 ; RV32IA-WMO-NEXT:    bne a3, a1, .LBB21_3
 ; RV32IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
 ; RV32IA-WMO-NEXT:    sc.w a4, a2, (a0)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB21_1
 ; RV32IA-WMO-NEXT:  .LBB21_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-TSO-NEXT:    lr.w a3, (a0)
 ; RV32IA-TSO-NEXT:    bne a3, a1, .LBB21_3
 ; RV32IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
 ; RV32IA-TSO-NEXT:    sc.w a4, a2, (a0)
 ; RV32IA-TSO-NEXT:    bnez a4, .LBB21_1

 ; RV64IA-WMO-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-WMO-NEXT:    bne a3, a1, .LBB21_3
 ; RV64IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
 ; RV64IA-WMO-NEXT:    sc.w a4, a2, (a0)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB21_1
 ; RV64IA-WMO-NEXT:  .LBB21_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    sext.w a1, a1
 ; RV64IA-TSO-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a3, (a0)
 ; RV64IA-TSO-NEXT:    bne a3, a1, .LBB21_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
 ; RV64IA-TSO-NEXT:    sc.w a4, a2, (a0)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB21_1
 ; RV64IA-TSO-NEXT:  .LBB21_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-WMO-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire monotonic
   ret void
 }
 
 define void @cmpxchg_i32_acquire_acquire(ptr %ptr, i32 %cmp, i32 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    lr.w.aq a3, (a0)
 ; RV32IA-WMO-NEXT:    bne a3, a1, .LBB22_3
 ; RV32IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
 ; RV32IA-WMO-NEXT:    sc.w a4, a2, (a0)
 ; RV32IA-WMO-NEXT:    bnez a4, .LBB22_1
 ; RV32IA-WMO-NEXT:  .LBB22_3:
 ; RV32IA-WMO-NEXT:    ret
 ;
+; RV32IA-ZACAS-LABEL: cmpxchg_i32_acquire_acquire:
+; RV32IA-ZACAS:       # %bb.0:
+; RV32IA-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV32IA-ZACAS-NEXT:    ret
+;
 ; RV32IA-TSO-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV32IA-TSO:       # %bb.0:
 ; RV32IA-TSO-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-TSO-NEXT:    lr.w a3, (a0)
 ; RV32IA-TSO-NEXT:    bne a3, a1, .LBB22_3
 ; RV32IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
 ; RV32IA-TSO-NEXT:    sc.w a4, a2, (a0)
 ; RV32IA-TSO-NEXT:    bnez a4, .LBB22_1

 ; RV64IA-WMO-NEXT:    lr.w.aq a3, (a0)
 ; RV64IA-WMO-NEXT:    bne a3, a1, .LBB22_3
 ; RV64IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
 ; RV64IA-WMO-NEXT:    sc.w a4, a2, (a0)
 ; RV64IA-WMO-NEXT:    bnez a4, .LBB22_1
 ; RV64IA-WMO-NEXT:  .LBB22_3:
 ; RV64IA-WMO-NEXT:    ret
 ;
+; RV64IA-ZACAS-LABEL: cmpxchg_i32_acquire_acquire:
+; RV64IA-ZACAS:       # %bb.0:
+; RV64IA-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV64IA-ZACAS-NEXT:    ret
+;
 ; RV64IA-TSO-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV64IA-TSO:       # %bb.0:
 ; RV64IA-TSO-NEXT:    sext.w a1, a1
 ; RV64IA-TSO-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-TSO-NEXT:    lr.w a3, (a0)
 ; RV64IA-TSO-NEXT:    bne a3, a1, .LBB22_3
 ; RV64IA-TSO-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
 ; RV64IA-TSO-NEXT:    sc.w a4, a2, (a0)
 ; RV64IA-TSO-NEXT:    bnez a4, .LBB22_1
 ; RV64IA-TSO-NEXT:  .LBB22_3:
 ; RV64IA-TSO-NEXT:    ret
+; RV32IA-WMO-ZACAS-LABEL: cmpxchg_i32_acquire_acquire:
+; RV32IA-WMO-ZACAS:       # %bb.0:
+; RV32IA-WMO-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV32IA-WMO-ZACAS-NEXT:    ret
+; RV64IA-WMO-ZACAS-LABEL: cmpxchg_i32_acquire_acquire:
+; RV64IA-WMO-ZACAS:       # %bb.0:
+; RV64IA-WMO-ZACAS-NEXT:    sext.w a1, a1
+; RV64IA-WMO-ZACAS-NEXT:    amocas.w.aq a1, a2, (a0)
+; RV64IA-WMO-ZACAS-NEXT:    ret
   %res = cmpxchg ptr %ptr, i32 %cmp, i32 %val acquire acquire
   ret void
 }
 
 define void @cmpxchg_i32_release_monotonic(ptr %ptr, i32 %cmp, i32 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i32_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16

 ; RV32IA-WMO-NEXT:    lr.w a3, (a0)
 ; RV32IA-WMO-NEXT:    bne a3, a1, .LBB23_3
 ; RV32IA-WMO-NEXT:  # %bb.2: # in Loop: Header=BB23_1 Depth=1
 ; RV32IA-WMO-NEXT:    sc.w.rl a4, a2, (a0)