[LoongArch] Add codegen support for atomicrmw umin/umax operation on LA64

[gonglingqin]

Summary

Furthermore, use beqz $rd, .BB instead of beq $rd, $zero, .BB.

[xry111]

Do we need dbar instructions around the ll/sc loop?

llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp

415

I think it's better to use BEQZ here.

[gonglingqin]

In D135525#3845403, @xry111 wrote:

Do we need dbar instructions around the ll/sc loop?

Yes. Thanks for the checking, I will fix this.

llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp

415

Good suggestion! I will modify it. Thanks.

[gonglingqin]

Address @xry111's comments.

[SixWeining]

Some inline comments. Thanks.

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll

77

rd == rj ? Maybe we miss earlyclobber?

86

Ditto.

162

Ditto.

171

Ditto.

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll

23

Had better mention this change in summary.

[gonglingqin]

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll

77

Thanks, I will rebase this patch.

86

Thanks, I will rebase this patch.

162

Thanks, I will rebase this patch.

171

Thanks, I will rebase this patch.

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll

23

Thanks, I will update the summary.

[gonglingqin]

rebase

[SixWeining]

LGTM.

llvm/include/llvm/IR/IntrinsicsLoongArch.td

multiclass MaskedAtomicRMWIntrinsics {
   // i64 @llvm.<name>.i32.<p>(any*, i64, i64, i64 imm);
   def _i64 : MaskedAtomicRMW<llvm_i64_ty>;
 }
 
 defm int_loongarch_masked_atomicrmw_xchg : MaskedAtomicRMWIntrinsics;
 defm int_loongarch_masked_atomicrmw_add : MaskedAtomicRMWIntrinsics;
 defm int_loongarch_masked_atomicrmw_sub : MaskedAtomicRMWIntrinsics;
 defm int_loongarch_masked_atomicrmw_nand : MaskedAtomicRMWIntrinsics;
+defm int_loongarch_masked_atomicrmw_umax : MaskedAtomicRMWIntrinsics;
+defm int_loongarch_masked_atomicrmw_umin : MaskedAtomicRMWIntrinsics;
 } // TargetPrefix = "loongarch"

llvm/lib/Target/LoongArch/LoongArchExpandAtomicPseudoInsts.cpp

 private:
   bool expandMBB(MachineBasicBlock &MBB);
   bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicBinOp(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
                          bool IsMasked, int Width,
                          MachineBasicBlock::iterator &NextMBBI);
+  bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI,
+                            AtomicRMWInst::BinOp, bool IsMasked, int Width,
+                            MachineBasicBlock::iterator &NextMBBI);
 };
 
 char LoongArchExpandAtomicPseudo::ID = 0;
 
 bool LoongArchExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
   TII =
       static_cast<const LoongArchInstrInfo *>(MF.getSubtarget().getInstrInfo());
   bool Modified = false;
bool LoongArchExpandAtomicPseudo::expandMI(
   case LoongArch::PseudoAtomicLoadAnd32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
                              NextMBBI);
   case LoongArch::PseudoAtomicLoadOr32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
   case LoongArch::PseudoAtomicLoadXor32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
                              NextMBBI);
+  case LoongArch::PseudoMaskedAtomicLoadUMax32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
+                                NextMBBI);
+  case LoongArch::PseudoMaskedAtomicLoadUMin32:
+    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
+                                NextMBBI);
   }
   return false;
 }
 
 static void doAtomicBinOpExpansion(const LoongArchInstrInfo *TII,
                                    MachineInstr &MI, DebugLoc DL,
                                    MachineBasicBlock *ThisMBB,
                                    MachineBasicBlock *LoopMBB,
                                    MachineBasicBlock *DoneMBB,
                                    AtomicRMWInst::BinOp BinOp, int Width) {
   Register DestReg = MI.getOperand(0).getReg();
   Register ScratchReg = MI.getOperand(1).getReg();
   Register AddrReg = MI.getOperand(2).getReg();
   Register IncrReg = MI.getOperand(3).getReg();
 
   // .loop:
   //   dbar 0
   //   ll.[w|d] dest, (addr)
   //   binop scratch, dest, val
   //   sc.[w|d] scratch, scratch, (addr)
-  //   beq scratch, zero, loop
+  //   beqz scratch, loop
   BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
   BuildMI(LoopMBB, DL,
           TII->get(Width == 32 ? LoongArch::LL_W : LoongArch::LL_D), DestReg)
       .addReg(AddrReg)
       .addImm(0);
   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
BuildMI(LoopMBB, DL, TII->get(LoongArch::XOR), ScratchReg)
         .addReg(IncrReg);
     break;
   }
   BuildMI(LoopMBB, DL,
           TII->get(Width == 32 ? LoongArch::SC_W : LoongArch::SC_D), ScratchReg)
       .addReg(ScratchReg)
       .addReg(AddrReg)
       .addImm(0);
-  BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQ))
+  BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
       .addReg(ScratchReg)
-      .addReg(LoongArch::R0)
       .addMBB(LoopMBB);
 }
 
 static void insertMaskedMerge(const LoongArchInstrInfo *TII, DebugLoc DL,
                               MachineBasicBlock *MBB, Register DestReg,
                               Register OldValReg, Register NewValReg,
                               Register MaskReg, Register ScratchReg) {
   assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
static void doMaskedAtomicBinOpExpansion(
   // .loop:
   //   dbar 0
   //   ll.w destreg, (alignedaddr)
   //   binop scratch, destreg, incr
   //   xor scratch, destreg, scratch
   //   and scratch, scratch, masktargetdata
   //   xor scratch, destreg, scratch
   //   sc.w scratch, scratch, (alignedaddr)
-  //   beq scratch, zero, loop
+  //   beqz scratch, loop
   BuildMI(LoopMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
   BuildMI(LoopMBB, DL, TII->get(LoongArch::LL_W), DestReg)
       .addReg(AddrReg)
       .addImm(0);
   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
   case AtomicRMWInst::Xchg:
static void doMaskedAtomicBinOpExpansion(
 
   insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
                     ScratchReg);
 
   BuildMI(LoopMBB, DL, TII->get(LoongArch::SC_W), ScratchReg)
       .addReg(ScratchReg)
       .addReg(AddrReg)
       .addImm(0);
-  BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQ))
+  BuildMI(LoopMBB, DL, TII->get(LoongArch::BEQZ))
       .addReg(ScratchReg)
-      .addReg(LoongArch::R0)
       .addMBB(LoopMBB);
 }
 
 bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
     MachineBasicBlock::iterator &NextMBBI) {
   MachineInstr &MI = *MBBI;
bool LoongArchExpandAtomicPseudo::expandAtomicBinOp(
 
   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *LoopMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);
 
   return true;
 }
 
+bool LoongArchExpandAtomicPseudo::expandAtomicMinMaxOp(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
+    MachineBasicBlock::iterator &NextMBBI) {
+  assert(IsMasked == true &&
+         "Should only need to expand masked atomic max/min");
+  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
+
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+  MachineFunction *MF = MBB.getParent();
+  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
+
+  // Insert new MBBs.
+  MF->insert(++MBB.getIterator(), LoopHeadMBB);
+  MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
+  MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
+  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
+
+  // Set up successors and transfer remaining instructions to DoneMBB.
+  LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
+  LoopHeadMBB->addSuccessor(LoopTailMBB);
+  LoopIfBodyMBB->addSuccessor(LoopTailMBB);
+  LoopTailMBB->addSuccessor(LoopHeadMBB);
+  LoopTailMBB->addSuccessor(DoneMBB);
+  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
+  DoneMBB->transferSuccessors(&MBB);
+  MBB.addSuccessor(LoopHeadMBB);
+
+  Register DestReg = MI.getOperand(0).getReg();
+  Register Scratch1Reg = MI.getOperand(1).getReg();
+  Register Scratch2Reg = MI.getOperand(2).getReg();
+  Register AddrReg = MI.getOperand(3).getReg();
+  Register IncrReg = MI.getOperand(4).getReg();
+  Register MaskReg = MI.getOperand(5).getReg();
+
+  //
+  // .loophead:
+  //   dbar 0
+  //   ll.w destreg, (alignedaddr)
+  //   and scratch2, destreg, mask
+  //   move scratch1, destreg
+  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::DBAR)).addImm(0);
+  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::LL_W), DestReg)
+      .addReg(AddrReg)
+      .addImm(0);
+  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::AND), Scratch2Reg)
+      .addReg(DestReg)
+      .addReg(MaskReg);
+  BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::OR), Scratch1Reg)
+      .addReg(DestReg)
+      .addReg(LoongArch::R0);
+
+  switch (BinOp) {
+  default:
+    llvm_unreachable("Unexpected AtomicRMW BinOp");
+  // bgeu scratch2, incr, .looptail
+  case AtomicRMWInst::UMax:
+    BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
+        .addReg(Scratch2Reg)
+        .addReg(IncrReg)
+        .addMBB(LoopTailMBB);
+    break;
+  // bgeu incr, scratch2, .looptail
+  case AtomicRMWInst::UMin:
+    BuildMI(LoopHeadMBB, DL, TII->get(LoongArch::BGEU))
+        .addReg(IncrReg)
+        .addReg(Scratch2Reg)
+        .addMBB(LoopTailMBB);
+    break;
+    // TODO: support other AtomicRMWInst.
+  }
+
+  // .loopifbody:
+  //   xor scratch1, destreg, incr
+  //   and scratch1, scratch1, mask
+  //   xor scratch1, destreg, scratch1
+  insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
+                    MaskReg, Scratch1Reg);
+
+  // .looptail:
+  //   sc.w scratch1, scratch1, (addr)
+  //   beqz scratch1, loop
+  //   dbar 0x700
+  BuildMI(LoopTailMBB, DL, TII->get(LoongArch::SC_W), Scratch1Reg)
+      .addReg(Scratch1Reg)
+      .addReg(AddrReg)
+      .addImm(0);
+  BuildMI(LoopTailMBB, DL, TII->get(LoongArch::BEQZ))
+      .addReg(Scratch1Reg)
+      .addMBB(LoopHeadMBB);
+  BuildMI(LoopTailMBB, DL, TII->get(LoongArch::DBAR)).addImm(0x700);
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+
+  LivePhysRegs LiveRegs;
+  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
+  computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
+  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
+  computeAndAddLiveIns(LiveRegs, *DoneMBB);
+
+  return true;
+}
+
 } // end namespace
 
 INITIALIZE_PASS(LoongArchExpandAtomicPseudo, "loongarch-expand-atomic-pseudo",
                 LoongArch_EXPAND_ATOMIC_PSEUDO_NAME, false, false)
 
 namespace llvm {
 
 FunctionPass *createLoongArchExpandAtomicPseudoPass() {
   return new LoongArchExpandAtomicPseudo();
 }
 
 } // end namespace llvm

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

if (GRLen == 64) {
     case AtomicRMWInst::Xchg:
       return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
     case AtomicRMWInst::Add:
       return Intrinsic::loongarch_masked_atomicrmw_add_i64;
     case AtomicRMWInst::Sub:
       return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
     case AtomicRMWInst::Nand:
       return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
+    case AtomicRMWInst::UMax:
+      return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
+    case AtomicRMWInst::UMin:
+      return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
       // TODO: support other AtomicRMWInst.
     }
   }
 
   if (GRLen == 32) {
     switch (BinOp) {
     default:
       llvm_unreachable("Unexpected AtomicRMW BinOp");

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

 def PseudoAtomicLoadNand32 : PseudoAM;
 def PseudoAtomicLoadNand64 : PseudoAM;
 def PseudoAtomicLoadAdd32 : PseudoAM;
 def PseudoAtomicLoadSub32 : PseudoAM;
 def PseudoAtomicLoadAnd32 : PseudoAM;
 def PseudoAtomicLoadOr32 : PseudoAM;
 def PseudoAtomicLoadXor32 : PseudoAM;
 
+
+class PseudoMaskedAMUMinUMax
+    : Pseudo<(outs GPR:$res, GPR:$scratch1, GPR:$scratch2),
+             (ins GPR:$addr, GPR:$incr, GPR:$mask, grlenimm:$ordering), []> {
+  let Constraints = "@earlyclobber $res,@earlyclobber $scratch1,"
+                    "@earlyclobber $scratch2";
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 0;
+}
+
+def PseudoMaskedAtomicLoadUMax32 : PseudoMaskedAMUMinUMax;
+def PseudoMaskedAtomicLoadUMin32 : PseudoMaskedAMUMinUMax;
+
 class AtomicPat<Intrinsic intrin, Pseudo AMInst>
     : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering),
           (AMInst GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering)>;
 
 let Predicates = [IsLA64] in {
 def : AtomicPat<int_loongarch_masked_atomicrmw_xchg_i64,
                 PseudoMaskedAtomicSwap32>;
 def : Pat<(atomic_swap_32 GPR:$addr, GPR:$incr),

 def : Pat<(atomic_load_or_32 GPR:$rj, GPR:$rk),
           (AMOR_DB_W GPR:$rk, GPR:$rj)>;
 def : Pat<(atomic_load_or_64 GPR:$rj, GPR:$rk),
           (AMOR_DB_D GPR:$rk, GPR:$rj)>;
 def : Pat<(atomic_load_xor_32 GPR:$rj, GPR:$rk),
           (AMXOR_DB_W GPR:$rk, GPR:$rj)>;
 def : Pat<(atomic_load_xor_64 GPR:$rj, GPR:$rk),
           (AMXOR_DB_D GPR:$rk, GPR:$rj)>;
+
+def : Pat<(atomic_load_umin_32 GPR:$rj, GPR:$rk),
+          (AMMIN_DB_WU GPR:$rk, GPR:$rj)>;
+def : Pat<(atomic_load_umin_64 GPR:$rj, GPR:$rk),
+          (AMMIN_DB_DU GPR:$rk, GPR:$rj)>;
+def : Pat<(atomic_load_umax_32 GPR:$rj, GPR:$rk),
+          (AMMAX_DB_WU GPR:$rk, GPR:$rj)>;
+def : Pat<(atomic_load_umax_64 GPR:$rj, GPR:$rk),
+          (AMMAX_DB_DU GPR:$rk, GPR:$rj)>;
+
+def : AtomicPat<int_loongarch_masked_atomicrmw_umax_i64,
+                PseudoMaskedAtomicLoadUMax32>;
+def : AtomicPat<int_loongarch_masked_atomicrmw_umin_i64,
+                PseudoMaskedAtomicLoadUMin32>;
 } // Predicates = [IsLA64]
 
 def : Pat<(atomic_load_nand_32 GPR:$rj, GPR:$rk),
           (PseudoAtomicLoadNand32 GPR:$rk, GPR:$rj)>;
 
 let Predicates = [IsLA32] in {
 def : AtomicPat<int_loongarch_masked_atomicrmw_xchg_i32,
                 PseudoMaskedAtomicSwap32>;

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw-minmax.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
+
+;; TODO: Testing for LA32 architecture will be added later
+
+define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.w $a2, $zero, -4
+; LA64-NEXT:    and $a2, $a0, $a2
+; LA64-NEXT:    slli.d $a0, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a0
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    dbar 0
+; LA64-NEXT:    ll.w $a4, $a2, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB0_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB0_3: # in Loop: Header=BB0_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a2, 0
+; LA64-NEXT:    beqz $a5, .LBB0_1
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a0
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.w $a2, $zero, -4
+; LA64-NEXT:    and $a2, $a0, $a2
+; LA64-NEXT:    slli.d $a0, $a0, 3
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a0
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    dbar 0
+; LA64-NEXT:    ll.w $a4, $a2, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a6, $a1, .LBB1_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB1_3: # in Loop: Header=BB1_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a2, 0
+; LA64-NEXT:    beqz $a5, .LBB1_1
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a0
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umax_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i32_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i32 %b acquire
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umax_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umax_i64_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammax_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umax ptr %a, i64 %b acquire
+  ret i64 %1
+}
+
+define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i8_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.w $a2, $zero, -4
+; LA64-NEXT:    and $a2, $a0, $a2
+; LA64-NEXT:    slli.d $a0, $a0, 3
+; LA64-NEXT:    ori $a3, $zero, 255
+; LA64-NEXT:    sll.w $a3, $a3, $a0
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    andi $a1, $a1, 255
+; LA64-NEXT:    sll.w $a1, $a1, $a0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    dbar 0
+; LA64-NEXT:    ll.w $a4, $a2, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB4_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB4_3: # in Loop: Header=BB4_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a2, 0
+; LA64-NEXT:    beqz $a5, .LBB4_1
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a0
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i8 %b acquire
+  ret i8 %1
+}
+
+define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i16_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.w $a2, $zero, -4
+; LA64-NEXT:    and $a2, $a0, $a2
+; LA64-NEXT:    slli.d $a0, $a0, 3
+; LA64-NEXT:    lu12i.w $a3, 15
+; LA64-NEXT:    ori $a3, $a3, 4095
+; LA64-NEXT:    sll.w $a3, $a3, $a0
+; LA64-NEXT:    addi.w $a3, $a3, 0
+; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
+; LA64-NEXT:    sll.w $a1, $a1, $a0
+; LA64-NEXT:    addi.w $a1, $a1, 0
+; LA64-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; LA64-NEXT:    dbar 0
+; LA64-NEXT:    ll.w $a4, $a2, 0
+; LA64-NEXT:    and $a6, $a4, $a3
+; LA64-NEXT:    move $a5, $a4
+; LA64-NEXT:    bgeu $a1, $a6, .LBB5_3
+; LA64-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; LA64-NEXT:    xor $a5, $a4, $a1
+; LA64-NEXT:    and $a5, $a5, $a3
+; LA64-NEXT:    xor $a5, $a4, $a5
+; LA64-NEXT:  .LBB5_3: # in Loop: Header=BB5_1 Depth=1
+; LA64-NEXT:    sc.w $a5, $a2, 0
+; LA64-NEXT:    beqz $a5, .LBB5_1
+; LA64-NEXT:    dbar 1792
+; LA64-NEXT:  # %bb.4:
+; LA64-NEXT:    srl.w $a0, $a4, $a0
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i16 %b acquire
+  ret i16 %1
+}
+
+define i32 @atomicrmw_umin_i32_acquire(ptr %a, i32 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i32_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.wu $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i32 %b acquire
+  ret i32 %1
+}
+
+define i64 @atomicrmw_umin_i64_acquire(ptr %a, i64 %b) nounwind {
+; LA64-LABEL: atomicrmw_umin_i64_acquire:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ammin_db.du $a2, $a1, $a0
+; LA64-NEXT:    move $a0, $a2
+; LA64-NEXT:    ret
+  %1 = atomicrmw umin ptr %a, i64 %b acquire
+  ret i64 %1
+}

llvm/test/CodeGen/LoongArch/ir-instruction/atomicrmw.ll

 ; LA32-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB0_1
+; LA32-NEXT:    beqz $a5, .LBB0_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xchg_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    ori $a3, $zero, 255
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    andi $a1, $a1, 255
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB0_1
+; LA64-NEXT:    beqz $a5, .LBB0_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw xchg ptr %a, i8 %b acquire
   ret i8 %1
 }
 
 define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {

 ; LA32-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    addi.w $a5, $a1, 0
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB1_1
+; LA32-NEXT:    beqz $a5, .LBB1_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xchg_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    lu12i.w $a3, 15
 ; LA64-NEXT:    ori $a3, $a3, 4095
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    addi.w $a5, $a1, 0
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB1_1
+; LA64-NEXT:    beqz $a5, .LBB1_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw xchg ptr %a, i16 %b acquire
   ret i16 %1
 }
 
 define i32 @atomicrmw_xchg_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xchg_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    move $a3, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB2_1
+; LA32-NEXT:    beqz $a3, .LBB2_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xchg_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amswap_db.w $a2, $a1, $a0
 ; LA64-NEXT:    move $a0, $a2

 ; LA32-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB4_1
+; LA32-NEXT:    beqz $a5, .LBB4_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_add_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    ori $a3, $zero, 255
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    andi $a1, $a1, 255
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB4_1
+; LA64-NEXT:    beqz $a5, .LBB4_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw add ptr %a, i8 %b acquire
   ret i8 %1
 }
 
 define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {

 ; LA32-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    add.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB5_1
+; LA32-NEXT:    beqz $a5, .LBB5_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_add_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    lu12i.w $a3, 15
 ; LA64-NEXT:    ori $a3, $a3, 4095
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    add.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB5_1
+; LA64-NEXT:    beqz $a5, .LBB5_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw add ptr %a, i16 %b acquire
   ret i16 %1
 }
 
 define i32 @atomicrmw_add_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_add_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    add.w $a3, $a2, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB6_1
+; LA32-NEXT:    beqz $a3, .LBB6_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_add_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amadd_db.w $a2, $a1, $a0
 ; LA64-NEXT:    move $a0, $a2

 ; LA32-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB8_1
+; LA32-NEXT:    beqz $a5, .LBB8_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_sub_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    ori $a3, $zero, 255
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    andi $a1, $a1, 255
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB8_1
+; LA64-NEXT:    beqz $a5, .LBB8_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw sub ptr %a, i8 %b acquire
   ret i8 %1
 }
 
 define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {

 ; LA32-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    sub.w $a5, $a4, $a1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB9_1
+; LA32-NEXT:    beqz $a5, .LBB9_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_sub_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    lu12i.w $a3, 15
 ; LA64-NEXT:    ori $a3, $a3, 4095
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    sub.w $a5, $a4, $a1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB9_1
+; LA64-NEXT:    beqz $a5, .LBB9_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw sub ptr %a, i16 %b acquire
   ret i16 %1
 }
 
 define i32 @atomicrmw_sub_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_sub_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    sub.w $a3, $a2, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB10_1
+; LA32-NEXT:    beqz $a3, .LBB10_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_sub_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    sub.w $a2, $zero, $a1
 ; LA64-NEXT:    amadd_db.w $a1, $a2, $a0

 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    xori $a5, $a5, -1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB12_1
+; LA32-NEXT:    beqz $a5, .LBB12_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_nand_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2
 ; LA64-NEXT:    slli.d $a0, $a0, 3
 ; LA64-NEXT:    ori $a3, $zero, 255
 ; LA64-NEXT:    sll.w $a3, $a3, $a0
 ; LA64-NEXT:    addi.w $a3, $a3, 0
 ; LA64-NEXT:    andi $a1, $a1, 255
 ; LA64-NEXT:    sll.w $a1, $a1, $a0
 ; LA64-NEXT:    addi.w $a1, $a1, 0
 ; LA64-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    xori $a5, $a5, -1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB12_1
+; LA64-NEXT:    beqz $a5, .LBB12_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw nand ptr %a, i8 %b acquire
   ret i8 %1
 }
 
 define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {

 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a4, $a2, 0
 ; LA32-NEXT:    and $a5, $a4, $a1
 ; LA32-NEXT:    xori $a5, $a5, -1
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    and $a5, $a5, $a3
 ; LA32-NEXT:    xor $a5, $a4, $a5
 ; LA32-NEXT:    sc.w $a5, $a2, 0
-; LA32-NEXT:    beq $a5, $zero, .LBB13_1
+; LA32-NEXT:    beqz $a5, .LBB13_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a4, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_nand_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2

 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a4, $a2, 0
 ; LA64-NEXT:    and $a5, $a4, $a1
 ; LA64-NEXT:    xori $a5, $a5, -1
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    and $a5, $a5, $a3
 ; LA64-NEXT:    xor $a5, $a4, $a5
 ; LA64-NEXT:    sc.w $a5, $a2, 0
-; LA64-NEXT:    beq $a5, $zero, .LBB13_1
+; LA64-NEXT:    beqz $a5, .LBB13_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    srl.w $a0, $a4, $a0
 ; LA64-NEXT:    ret
   %1 = atomicrmw nand ptr %a, i16 %b acquire
   ret i16 %1
 }
 
 define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_nand_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    and $a3, $a2, $a0
 ; LA32-NEXT:    xori $a3, $a3, -1
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB14_1
+; LA32-NEXT:    beqz $a3, .LBB14_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_nand_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.w $a2, $a1, 0
 ; LA64-NEXT:    and $a3, $a2, $a0
 ; LA64-NEXT:    xori $a3, $a3, -1
 ; LA64-NEXT:    sc.w $a3, $a1, 0
-; LA64-NEXT:    beq $a3, $zero, .LBB14_1
+; LA64-NEXT:    beqz $a3, .LBB14_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    move $a0, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw nand ptr %a, i32 %b acquire
   ret i32 %1
 }
 
 define i64 @atomicrmw_nand_i64_acquire(ptr %a, i64 %b) nounwind {

 ; LA64-LABEL: atomicrmw_nand_i64_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
 ; LA64-NEXT:    dbar 0
 ; LA64-NEXT:    ll.d $a2, $a1, 0
 ; LA64-NEXT:    and $a3, $a2, $a0
 ; LA64-NEXT:    xori $a3, $a3, -1
 ; LA64-NEXT:    sc.d $a3, $a1, 0
-; LA64-NEXT:    beq $a3, $zero, .LBB15_1
+; LA64-NEXT:    beqz $a3, .LBB15_1
 ; LA64-NEXT:  # %bb.2:
 ; LA64-NEXT:    move $a0, $a2
 ; LA64-NEXT:    ret
   %1 = atomicrmw nand ptr %a, i64 %b acquire
   ret i64 %1
 }
 
 define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
 ; LA32-LABEL: atomicrmw_and_i8_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    slli.w $a2, $a0, 3
 ; LA32-NEXT:    ori $a3, $zero, 255
 ; LA32-NEXT:    sll.w $a3, $a3, $a2
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a2
 ; LA32-NEXT:    orn $a1, $a1, $a3
 ; LA32-NEXT:    addi.w $a3, $zero, -4
 ; LA32-NEXT:    and $a0, $a0, $a3
 ; LA32-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a1, 0
 ; LA32-NEXT:    and $a4, $a3, $a0
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB16_1
+; LA32-NEXT:    beqz $a4, .LBB16_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_and_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    slli.d $a2, $a0, 3
 ; LA64-NEXT:    ori $a3, $zero, 255

 ; LA32-NEXT:    orn $a1, $a1, $a2
 ; LA32-NEXT:    addi.w $a2, $zero, -4
 ; LA32-NEXT:    and $a0, $a0, $a2
 ; LA32-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    and $a4, $a2, $a0
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB17_1
+; LA32-NEXT:    beqz $a4, .LBB17_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a2, $a3
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_and_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    lu12i.w $a2, 15
 ; LA64-NEXT:    ori $a2, $a2, 4095

 define i32 @atomicrmw_and_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_and_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    and $a3, $a2, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB18_1
+; LA32-NEXT:    beqz $a3, .LBB18_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_and_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amand_db.w $a2, $a1, $a0
 ; LA64-NEXT:    move $a0, $a2

 ; LA32-NEXT:    slli.w $a0, $a0, 3
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a0
 ; LA32-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a1, 0
 ; LA32-NEXT:    or $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB20_1
+; LA32-NEXT:    beqz $a4, .LBB20_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_or_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2

 ; LA32-NEXT:    slli.w $a0, $a0, 3
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a0
 ; LA32-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a1, 0
 ; LA32-NEXT:    or $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB21_1
+; LA32-NEXT:    beqz $a4, .LBB21_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_or_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2

 define i32 @atomicrmw_or_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_or_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    or $a3, $a2, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB22_1
+; LA32-NEXT:    beqz $a3, .LBB22_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_or_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amor_db.w $a2, $a1, $a0
 ; LA64-NEXT:    move $a0, $a2

 ; LA32-NEXT:    slli.w $a0, $a0, 3
 ; LA32-NEXT:    andi $a1, $a1, 255
 ; LA32-NEXT:    sll.w $a1, $a1, $a0
 ; LA32-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a1, 0
 ; LA32-NEXT:    xor $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB24_1
+; LA32-NEXT:    beqz $a4, .LBB24_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xor_i8_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2

 ; LA32-NEXT:    slli.w $a0, $a0, 3
 ; LA32-NEXT:    bstrpick.w $a1, $a1, 15, 0
 ; LA32-NEXT:    sll.w $a1, $a1, $a0
 ; LA32-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a3, $a1, 0
 ; LA32-NEXT:    xor $a4, $a3, $a2
 ; LA32-NEXT:    sc.w $a4, $a1, 0
-; LA32-NEXT:    beq $a4, $zero, .LBB25_1
+; LA32-NEXT:    beqz $a4, .LBB25_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    srl.w $a0, $a3, $a0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xor_i16_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    addi.w $a2, $zero, -4
 ; LA64-NEXT:    and $a2, $a0, $a2

 define i32 @atomicrmw_xor_i32_acquire(ptr %a, i32 %b) nounwind {
 ; LA32-LABEL: atomicrmw_xor_i32_acquire:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ll.w $a2, $a1, 0
 ; LA32-NEXT:    xor $a3, $a2, $a0
 ; LA32-NEXT:    sc.w $a3, $a1, 0
-; LA32-NEXT:    beq $a3, $zero, .LBB26_1
+; LA32-NEXT:    beqz $a3, .LBB26_1
 ; LA32-NEXT:  # %bb.2:
 ; LA32-NEXT:    move $a0, $a2
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: atomicrmw_xor_i32_acquire:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amxor_db.w $a2, $a1, $a0
 ; LA64-NEXT:    move $a0, $a2