[ARM][AArch64] Add findSpillBefore to TargetRegisterInfo

[tmatheson]

Summary

Due to interactions between RegAllocFast and expansion of atomicrmw at -O0,
both ARM and AArch64 backends would emit stores between ldrex and strex,
which clears the exclusive access monitor.

atomicrmw instructions are expanded to loops, where the main MachineBasicBlock
includes a ldrex/strex pair. It then conditionally branches if this atomic
operation was successful. Because of this, the register loaded by ldrex is
LiveOut, and RegAllocFast therefore spills this register. The issue is that it
spills between the ldrex and strex, which invalidates the monitor.

I tried several ways of fixing this which all have problems:

• Adding a pass after RegAllocFast which moved the str instructions (spills) to after the strex. For more complex sequences like those generated for 64 bit atomics with e.g. nand, this becomes difficult to do.

• Add new pseudo instructions for atomicrmw which are expanded after register allocation. This would involve duplicating all of the loop creation code. Similar approach has been used before for cmpxchg: https://reviews.llvm.org/D16239?id=52861

• Stop FastRegAlloc from spilling these registers for these instructions. However, other instructions between ldrex and strex can spill, and it is hard to catch them all.

• Move the location of the spill to after the strex. This is the approach taken.

To spill after strex, I have added a new function to TargetRegisterInfo which returns an appropriate
point to spill at for a given instruction. For all backends except ARM/AArch64 this just returns the
next instruction. For ARM/AArch64 it returns the strex.

Alternatives:

• A similar approach could have been applied to calcSpillCost instead, to set a high spill cost between ldrex/strex
• Pseudo instructions could have been used instead

It is also possible that the cmpxchg pseudoinstructions could be removed, and the same technique used for them.

[tmatheson]

Duplicate

llvm/include/llvm/CodeGen/TargetRegisterInfo.h

public:
   /// This method is used to decide whether \p VirtReg should use the deferred
   /// spilling stage instead of being spilled right away.
   virtual bool
   shouldUseDeferredSpillingForVirtReg(const MachineFunction &MF,
                                       const LiveInterval &VirtReg) const {
     return false;
   }
 
+  /// On some targets, spilling between certain instructions can invalidate
+  /// semantics used for atomic operations. This function returns a suitable
+  /// point where the spill can be done.
+  virtual MachineBasicBlock::iterator findSpillBefore(MachineInstr &MI) const;
+
   //===--------------------------------------------------------------------===//
   /// Debug information queries.
 
   /// getFrameRegister - This method should return the register used as a base
   /// for values allocated in the current stack frame.
   virtual Register getFrameRegister(const MachineFunction &MF) const = 0;
 
   /// Mark a register and all its aliases as reserved in the given set.

llvm/lib/CodeGen/RegAllocFast.cpp

LLVM_DEBUG(dbgs() << "In def of " << printReg(VirtReg, TRI)
                       << " use existing assignment to "
                       << printReg(LRI->PhysReg, TRI) << '\n');
   }
 
   MCPhysReg PhysReg = LRI->PhysReg;
   assert(PhysReg != 0 && "Register not assigned");
   if (LRI->Reloaded || LRI->LiveOut) {
     if (!MI.isImplicitDef()) {
-      MachineBasicBlock::iterator SpillBefore =
-          std::next((MachineBasicBlock::iterator)MI.getIterator());
+      auto SpillBefore = TRI->findSpillBefore(MI);
       LLVM_DEBUG(dbgs() << "Spill Reason: LO: " << LRI->LiveOut << " RL: "
                         << LRI->Reloaded << '\n');
       bool Kill = LRI->LastUse == nullptr;
       spill(SpillBefore, VirtReg, PhysReg, Kill, LRI->LiveOut);
       LRI->LastUse = nullptr;
     }
     LRI->LiveOut = false;
     LRI->Reloaded = false;

llvm/lib/CodeGen/TargetRegisterInfo.cpp

bool TargetRegisterInfo::shouldRegionSplitForVirtReg(
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   MachineInstr *MI = MRI.getUniqueVRegDef(VirtReg.reg());
   if (MI && TII->isTriviallyReMaterializable(*MI) &&
       VirtReg.size() > HugeSizeForSplit)
     return false;
   return true;
 }
 
+MachineBasicBlock::iterator
+TargetRegisterInfo::findSpillBefore(MachineInstr &MI) const {
+  return std::next((MachineBasicBlock::iterator)MI.getIterator());
+}
+
 void TargetRegisterInfo::markSuperRegs(BitVector &RegisterSet,
                                        MCRegister Reg) const {
   for (MCSuperRegIterator AI(Reg, this, true); AI.isValid(); ++AI)
     RegisterSet.set(*AI);
 }
 
 bool TargetRegisterInfo::checkAllSuperRegsMarked(const BitVector &RegisterSet,
     ArrayRef<MCPhysReg> Exceptions) const {

llvm/lib/Target/AArch64/AArch64RegisterInfo.h

public:
   bool regNeedsCFI(unsigned Reg, unsigned &RegToUseForCFI) const;
 
   /// SrcRC and DstRC will be morphed into NewRC if this returns true
   bool shouldCoalesce(MachineInstr *MI, const TargetRegisterClass *SrcRC,
                       unsigned SubReg, const TargetRegisterClass *DstRC,
                       unsigned DstSubReg, const TargetRegisterClass *NewRC,
                       LiveIntervals &LIS) const override;
 
+  MachineBasicBlock::iterator findSpillBefore(MachineInstr &MI) const override;
+
   void getOffsetOpcodes(const StackOffset &Offset,
                         SmallVectorImpl<uint64_t> &Ops) const override;
 };
 
 } // end namespace llvm
 
 #endif

llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp

unsigned AArch64RegisterInfo::getLocalAddressRegister(
   return getFrameRegister(MF);
 }
 
 /// SrcRC and DstRC will be morphed into NewRC if this returns true
 bool AArch64RegisterInfo::shouldCoalesce(
     MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,
     const TargetRegisterClass *DstRC, unsigned DstSubReg,
     const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
   if (MI->isCopy() &&
       ((DstRC->getID() == AArch64::GPR64RegClassID) ||
        (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&
       MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg())
     // Do not coalesce in the case of a 32-bit subregister copy
     // which implements a 32 to 64 bit zero extension
     // which relies on the upper 32 bits being zeroed.
     return false;
   return true;
 }
+
+/// Select an appropriate instruction to spill before. Normally this is the
+/// instruction right after the current one, but in the case of ldrex/strex
+/// pairs we need to avoid introducing stores between the exclusive
+/// instructions, otherwise we will invalidate them.
+MachineBasicBlock::iterator
+AArch64RegisterInfo::findSpillBefore(MachineInstr &MI) const {
+  auto isExclusiveLoad = [](auto &MI) -> unsigned {
+    switch (MI.getOpcode()) {
+    case AArch64::LDXPW:
+      return AArch64::STXPW;
+    case AArch64::LDXPX:
+      return AArch64::STXPX;
+    case AArch64::LDXRB:
+      return AArch64::STXRB;
+    case AArch64::LDXRH:
+      return AArch64::STXRH;
+    case AArch64::LDXRW:
+      return AArch64::STXRW;
+    case AArch64::LDXRX:
+      return AArch64::STXRX;
+    default:
+      return 0;
+    }
+  };
+
+  MachineBasicBlock &MBB = *MI.getParent();
+  const MachineBasicBlock::reverse_iterator RE = MBB.rend();
+  const MachineBasicBlock::iterator E = MBB.end();
+
+  // Default value, spill before the next instruction
+  auto SpillBefore = std::next((MachineBasicBlock::iterator)MI.getIterator());
+
+  // If we are already at the end of the block, spill here
+  if (SpillBefore == E)
+    return SpillBefore;
+
+  // See if we are between an exclusive load/store pair
+  const auto LoadInst =
+      std::find_if((MachineBasicBlock::reverse_iterator)MI.getReverseIterator(),
+                   RE, isExclusiveLoad);
+
+  // If there is no exclusive load previously in this block, return the default
+  if (LoadInst == RE)
+    return SpillBefore;
+
+  // Find the corresponding exclusive store
+  const unsigned StoreOp = isExclusiveLoad(*LoadInst);
+  while (SpillBefore != E && (*SpillBefore).getOpcode() != StoreOp) {
+    SpillBefore = std::next(SpillBefore);
+  }
+
+  // If we failed to find an exclusive store, return the default
+  if (SpillBefore == E)
+    SpillBefore = std::next((MachineBasicBlock::iterator)MI.getIterator());
+
+  // We want to spill after the store
+  SpillBefore = std::next(SpillBefore);
+
+  return SpillBefore;
+}
 No newline at end of file

llvm/lib/Target/ARM/ARMRegisterInfo.h

 namespace llvm {
 
 class ARMSubtarget;
 
 struct ARMRegisterInfo : public ARMBaseRegisterInfo {
   virtual void anchor();
 public:
   ARMRegisterInfo();
+  MachineBasicBlock::iterator findSpillBefore(MachineInstr &MI) const override;
 };
 
 } // end namespace llvm
 
 #endif

llvm/lib/Target/ARM/ARMRegisterInfo.cpp

 //===----------------------------------------------------------------------===//
 
 #include "ARMRegisterInfo.h"
 using namespace llvm;
 
 void ARMRegisterInfo::anchor() { }
 
 ARMRegisterInfo::ARMRegisterInfo() : ARMBaseRegisterInfo() {}
+
+/// Select an appropriate instruction to spill before. Normally this is the
+/// instruction right after the current one, but in the case of ldrex/strex
+/// pairs we need to avoid introducing stores between the exclusive
+/// instructions, otherwise we will invalidate them.
+MachineBasicBlock::iterator
+ARMRegisterInfo::findSpillBefore(MachineInstr &MI) const {
+  auto isExclusiveLoad = [](const MachineInstr &MI) -> unsigned {
+    switch (MI.getOpcode()) {
+    case ARM::t2LDREX:
+      return ARM::t2STREX;
+    case ARM::t2LDREXB:
+      return ARM::t2STREXB;
+    case ARM::t2LDREXD:
+      return ARM::t2STREXD;
+    case ARM::t2LDREXH:
+      return ARM::t2STREXH;
+    case ARM::LDREX:
+      return ARM::STREX;
+    case ARM::LDREXB:
+      return ARM::STREXB;
+    case ARM::LDREXD:
+      return ARM::STREXD;
+    case ARM::LDREXH:
+      return ARM::STREXH;
+    default:
+      return 0;
+    }
+  };
+
+  MachineBasicBlock &MBB = *MI.getParent();
+  const MachineBasicBlock::reverse_iterator RE = MBB.rend();
+  const MachineBasicBlock::iterator E = MBB.end();
+
+  // Default value, spill before the next instruction
+  auto SpillBefore = std::next((MachineBasicBlock::iterator)MI.getIterator());
+
+  // If we are already at the end of the block, spill here
+  if (SpillBefore == E)
+    return SpillBefore;
+
+  // See if we are between an exclusive load/store pair
+  const auto LoadInst =
+      std::find_if((MachineBasicBlock::reverse_iterator)MI.getReverseIterator(),
+                   RE, isExclusiveLoad);
+
+  // If there is no exclusive load previously in this block, return the default
+  if (LoadInst == RE)
+    return SpillBefore;
+
+  // Find the corresponding exclusive store
+  const unsigned StoreOp = isExclusiveLoad(*LoadInst);
+  while (SpillBefore != E && (*SpillBefore).getOpcode() != StoreOp) {
+    SpillBefore = std::next(SpillBefore);
+  }
+
+  // If we failed to find an exclusive store, return the default
+  if (SpillBefore == E)
+    SpillBefore = std::next((MachineBasicBlock::iterator)MI.getIterator());
+
+  // We want to spill after the store
+  SpillBefore = std::next(SpillBefore);
+
+  return SpillBefore;
+}

llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor.ll

+; TODO arm and aarch64 versions, since bug effects both
+
+; RUN: llc -O0 -o - %s --debug-only=arm-atomic-load-store-fixup --mtriple=armv8-none-eabi   | FileCheck %s --check-prefix=CHECK --check-prefix=ARM
+; RUN: llc -O0 -o - %s --debug-only=arm-atomic-load-store-fixup --mtriple=aarch64-none-eabi | FileCheck %s --check-prefix=CHECK --check-prefix=A64
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@atomic_i8 = external global i8
+@atomic_i16 = external global i16
+@atomic_i32 = external global i32
+@atomic_i64 = external global i64
+
+@atomic_half = external global half
+@atomic_float = external global float
+@atomic_double = external global double  
+
+
+define i8 @test_xchg_i8() {
+entry:
+  %0  = atomicrmw xchg  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_xchg_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_add_i8() {
+entry:
+  %0  = atomicrmw add  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_add_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_sub_i8() {
+entry:
+  %0  = atomicrmw sub  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_sub_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_and_i8() {
+entry:
+  %0  = atomicrmw and  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_and_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_nand_i8() {
+entry:
+  %0  = atomicrmw nand  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_nand_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_or_i8() {
+entry:
+  %0  = atomicrmw or  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_or_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_xor_i8() {
+entry:
+  %0  = atomicrmw xor  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_xor_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_max_i8() {
+entry:
+  %0  = atomicrmw max  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_max_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_min_i8() {
+entry:
+  %0  = atomicrmw min  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_min_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_umax_i8() {
+entry:
+  %0  = atomicrmw umax  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_umax_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+define i8 @test_umin_i8() {
+entry:
+  %0  = atomicrmw umin  i8* @atomic_i8, i8 1 monotonic
+  ; CHECK: test_umin_i8:
+  ; ARM:   ldrexb  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrb   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexb  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrb   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i8 %0
+}
+
+
+define i16 @test_xchg_i16() {
+entry:
+  %0  = atomicrmw xchg  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_xchg_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_add_i16() {
+entry:
+  %0  = atomicrmw add  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_add_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_sub_i16() {
+entry:
+  %0  = atomicrmw sub  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_sub_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_and_i16() {
+entry:
+  %0  = atomicrmw and  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_and_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_nand_i16() {
+entry:
+  %0  = atomicrmw nand  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_nand_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_or_i16() {
+entry:
+  %0  = atomicrmw or  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_or_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_xor_i16() {
+entry:
+  %0  = atomicrmw xor  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_xor_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_max_i16() {
+entry:
+  %0  = atomicrmw max  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_max_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_min_i16() {
+entry:
+  %0  = atomicrmw min  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_min_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_umax_i16() {
+entry:
+  %0  = atomicrmw umax  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_umax_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define i16 @test_umin_i16() {
+entry:
+  %0  = atomicrmw umin  i16* @atomic_i16, i16 1 monotonic
+  ; CHECK: test_umin_i16:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxrh   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxrh   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i16 %0
+}
+define half @test_fadd_half() {
+entry:
+  %0  = atomicrmw fadd  half* @atomic_half, half 1.0 monotonic
+  ; CHECK: test_fadd_half:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxrh  [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxrh  {{w[0-9]+}}, {{w[0-9]+}}, [[ADDR]]
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ret half %0
+}
+define half @test_fsub_half() {
+entry:
+  %0  = atomicrmw fsub  half* @atomic_half, half 1.0 monotonic
+  ; CHECK: test_fsub_half:
+  ; ARM:   ldrexh  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxrh  [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strexh  {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxrh  {{w[0-9]+}}, {{w[0-9]+}}, [[ADDR]]
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ret half %0
+}
+
+
+define i32 @test_xchg_i32() {
+entry:
+  %0  = atomicrmw xchg  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_xchg_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_add_i32() {
+entry:
+  %0  = atomicrmw add  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_add_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_sub_i32() {
+entry:
+  %0  = atomicrmw sub  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_sub_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_and_i32() {
+entry:
+  %0  = atomicrmw and  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_and_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_nand_i32() {
+entry:
+  %0  = atomicrmw nand  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_nand_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_or_i32() {
+entry:
+  %0  = atomicrmw or  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_or_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_xor_i32() {
+entry:
+  %0  = atomicrmw xor  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_xor_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_max_i32() {
+entry:
+  %0  = atomicrmw max  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_max_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_min_i32() {
+entry:
+  %0  = atomicrmw min  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_min_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_umax_i32() {
+entry:
+  %0  = atomicrmw umax  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_umax_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define i32 @test_umin_i32() {
+entry:
+  %0  = atomicrmw umin  i32* @atomic_i32, i32 1 monotonic
+  ; CHECK: test_umin_i32:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr   [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; ARM:   strex  [[RA]], {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr   [[RA]], {{w[0-9]+}}, [[ADDR]]
+  ret i32 %0
+}
+define float @test_fadd_float() {
+entry:
+  %0  = atomicrmw fadd  float* @atomic_float, float 1.0 monotonic
+  ; CHECK: test_fadd_float:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxr  [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ; ARM:   strex  {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxr  {{w[0-9]+}}, {{w[0-9]+}}, [[ADDR]]
+  ret float %0
+}
+define float @test_fsub_float() {
+entry:
+  %0  = atomicrmw fsub  float* @atomic_float, float 1.0 monotonic
+  ; CHECK: test_fsub_float:
+  ; ARM:   ldrex  [[RA:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxr  [[RA:w[0-9]+]], [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT: str
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ; ARM:   strex  {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxr  {{w[0-9]+}}, {{w[0-9]+}}, [[ADDR]]
+  ret float %0
+}
+
+
+
+
+define i64 @test_xchg_i64() {
+entry:
+  %0  = atomicrmw xchg  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_xchg_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_add_i64() {
+entry:
+  %0  = atomicrmw add  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_add_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_sub_i64() {
+entry:
+  %0  = atomicrmw sub  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_sub_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_and_i64() {
+entry:
+  %0  = atomicrmw and  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_and_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_nand_i64() {
+entry:
+  %0  = atomicrmw nand  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_nand_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_or_i64() {
+entry:
+  %0  = atomicrmw or  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_or_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_xor_i64() {
+entry:
+  %0  = atomicrmw xor  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_xor_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_max_i64() {
+entry:
+  %0  = atomicrmw max  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_max_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_min_i64() {
+entry:
+  %0  = atomicrmw min  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_min_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_umax_i64() {
+entry:
+  %0  = atomicrmw umax  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_umax_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define i64 @test_umin_i64() {
+entry:
+  %0  = atomicrmw umin  i64* @atomic_i64, i64 1 monotonic
+  ; CHECK: test_umin_i64:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldxr    [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stxr    {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret i64 %0
+}
+define double @test_fadd_double() {
+entry:
+  %0  = atomicrmw fadd  double* @atomic_double, double 1.0 monotonic
+  ; CHECK: test_fadd_double:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxr   [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxr   {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret double %0
+}
+define double @test_fsub_double() {
+entry:
+  %0  = atomicrmw fsub  double* @atomic_double, double 1.0 monotonic
+  ; CHECK: test_fsub_double:
+  ; ARM:   ldrexd  [[RA:r[0-9]+]], [[RB:r[0-9]+]], [[ADDR:.r[0-9]+.]]
+  ; A64:   ldaxr   [[RA:x[0-9]+]],                 [[ADDR:.x[0-9]+.]]
+  ; CHECK-NOT str FIXME FIXME FIXME
+  ; the strex ends up in the next machine basic block, so regs will be different but address the same
+  ; ARM:   strexd  {{r[0-9]+}}, {{r[0-9]+}}, {{r[0-9]+}}, [[ADDR]]
+  ; A64:   stlxr   {{w[0-9]+}}, {{x[0-9]+}},              [[ADDR]]
+  ret double %0
+}

llvm/test/CodeGen/ARM/cmpxchg-O0.ll

 ; CHECK-DAG: mov [[ADDR:r[0-9]+]], r0
 ; CHECK-DAG: mov [[NEW:r[0-9]+]], r2
 ; CHECK:     dmb ish
 ; CHECK:     uxtb [[DESIRED:r[0-9]+]], [[DESIRED]]
 ; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
 ; CHECK:     ldrexb [[OLD:[lr0-9]+]], {{\[}}[[ADDR]]{{\]}}
 ; CHECK:     cmp [[OLD]], [[DESIRED]]
 ; CHECK:     bne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK-NOT: stri12
 ; CHECK:     strexb [[STATUS:r[0-9]+]], [[NEW]], {{\[}}[[ADDR]]{{\]}}
 ; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
 ; CHECK:     bne [[RETRY]]
 ; CHECK: [[DONE]]:
 ; Materialisation of a boolean is done with sub/clz/lsr
 ; CHECK:     uxtb [[CMP1:r[0-9]+]], [[DESIRED]]
 ; CHECK:     sub{{(\.w|s)?}} [[CMP1]], [[OLD]], [[CMP1]]
 ; CHECK:     clz [[CMP2:r[0-9]+]], [[CMP1]]