diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -1579,8 +1579,7 @@
     MachineInstrBuilder MIB =
         BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
             .addReg(DesiredReg, RegState::Kill);
-    if (!IsThumb)
-      MIB.addImm(0);
+    MIB.addImm(0); // A1/T2 rotate operand
     MIB.add(predOps(ARMCC::AL));
   }
 
@@ -2782,14 +2781,14 @@
     case ARM::CMP_SWAP_8:
       if (STI->isThumb())
         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
-                              ARM::tUXTB, NextMBBI);
+                              ARM::t2UXTB, NextMBBI);
       else
         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
                               ARM::UXTB, NextMBBI);
     case ARM::CMP_SWAP_16:
       if (STI->isThumb())
         return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
-                              ARM::tUXTH, NextMBBI);
+                              ARM::t2UXTH, NextMBBI);
       else
         return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
                               ARM::UXTH, NextMBBI);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -19120,6 +19120,14 @@
   if (AI->isFloatingPointOperation())
     return AtomicExpansionKind::CmpXChg;
 
+  // At -O0, fast-regalloc cannot cope with the live vregs necessary to
+  // implement atomicrmw without spilling. If the target address is also on the
+  // stack and close enough to the spill slot, this can lead to a situation
+  // where the monitor always gets cleared and the atomic operation can never
+  // succeed. So at -O0 lower this operation to a CAS loop.
+  if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
+    return AtomicExpansionKind::CmpXChg;
+
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   bool hasAtomicRMW = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps();
   return (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW)
diff --git a/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_all.ll b/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_all.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/atomicrmw_exclusive_monitor_all.ll
@@ -0,0 +1,4492 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; Test the instruction sequences produced by atomicrmw instructions. In
+; particular, ensure there are no stores/spills inserted between the exclusive
+; load and stores, which would invalidate the exclusive monitor.
+
+; atomicrmw xchg for floating point types are not implemented yet, so the tests
+; are commented.
+
+; RUN: llc -mtriple=armv8-none-linux-gnu -O0 -o - %s | FileCheck %s --check-prefix=CHECK-ARM
+; RUN: llc -mtriple=thumbv7-none-linux-gnu -O0 -o - %s | FileCheck %s --check-prefix=CHECK-THUMB2
+
+@atomic_i8 = external global i8
+@atomic_i16 = external global i16
+@atomic_i32 = external global i32
+@atomic_i64 = external global i64
+
+@atomic_half = external global half
+@atomic_float = external global float
+@atomic_double = external global double
+
+define i8 @test_xchg_i8() {
+; CHECK-ARM-LABEL: test_xchg_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB0_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB0_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB0_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB0_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB0_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB0_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB0_2
+; CHECK-ARM-NEXT:  .LBB0_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB0_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB0_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xchg_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB0_1
+; CHECK-THUMB2-NEXT:  .LBB0_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB0_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB0_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB0_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB0_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB0_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB0_2
+; CHECK-THUMB2-NEXT:  .LBB0_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB0_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB0_1
+; CHECK-THUMB2-NEXT:    b .LBB0_5
+; CHECK-THUMB2-NEXT:  .LBB0_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xchg  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_add_i8() {
+; CHECK-ARM-LABEL: test_add_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB1_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB1_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB1_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB1_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB1_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB1_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB1_2
+; CHECK-ARM-NEXT:  .LBB1_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB1_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB1_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_add_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB1_1
+; CHECK-THUMB2-NEXT:  .LBB1_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB1_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB1_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB1_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB1_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB1_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB1_2
+; CHECK-THUMB2-NEXT:  .LBB1_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB1_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB1_1
+; CHECK-THUMB2-NEXT:    b .LBB1_5
+; CHECK-THUMB2-NEXT:  .LBB1_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw add  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_sub_i8() {
+; CHECK-ARM-LABEL: test_sub_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB2_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB2_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB2_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB2_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB2_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB2_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB2_2
+; CHECK-ARM-NEXT:  .LBB2_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB2_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB2_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_sub_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB2_1
+; CHECK-THUMB2-NEXT:  .LBB2_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB2_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sub.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB2_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB2_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB2_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB2_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB2_2
+; CHECK-THUMB2-NEXT:  .LBB2_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB2_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB2_1
+; CHECK-THUMB2-NEXT:    b .LBB2_5
+; CHECK-THUMB2-NEXT:  .LBB2_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw sub  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_and_i8() {
+; CHECK-ARM-LABEL: test_and_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB3_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB3_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    and r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB3_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB3_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB3_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB3_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB3_2
+; CHECK-ARM-NEXT:  .LBB3_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB3_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB3_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_and_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB3_1
+; CHECK-THUMB2-NEXT:  .LBB3_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB3_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    and r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB3_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB3_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB3_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB3_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB3_2
+; CHECK-THUMB2-NEXT:  .LBB3_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB3_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB3_1
+; CHECK-THUMB2-NEXT:    b .LBB3_5
+; CHECK-THUMB2-NEXT:  .LBB3_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw and  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_nand_i8() {
+; CHECK-ARM-LABEL: test_nand_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB4_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB4_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mvn r0, r1
+; CHECK-ARM-NEXT:    mvn r2, #1
+; CHECK-ARM-NEXT:    orr r12, r0, r2
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB4_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB4_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB4_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB4_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB4_2
+; CHECK-ARM-NEXT:  .LBB4_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB4_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB4_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_nand_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB4_1
+; CHECK-THUMB2-NEXT:  .LBB4_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB4_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mvn r0, #1
+; CHECK-THUMB2-NEXT:    orn r12, r0, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB4_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB4_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB4_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB4_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB4_2
+; CHECK-THUMB2-NEXT:  .LBB4_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB4_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB4_1
+; CHECK-THUMB2-NEXT:    b .LBB4_5
+; CHECK-THUMB2-NEXT:  .LBB4_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw nand  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_or_i8() {
+; CHECK-ARM-LABEL: test_or_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB5_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB5_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    orr r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB5_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB5_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB5_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB5_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB5_2
+; CHECK-ARM-NEXT:  .LBB5_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB5_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB5_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_or_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB5_1
+; CHECK-THUMB2-NEXT:  .LBB5_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB5_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    orr r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB5_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB5_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB5_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB5_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB5_2
+; CHECK-THUMB2-NEXT:  .LBB5_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB5_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB5_1
+; CHECK-THUMB2-NEXT:    b .LBB5_5
+; CHECK-THUMB2-NEXT:  .LBB5_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw or  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_xor_i8() {
+; CHECK-ARM-LABEL: test_xor_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB6_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB6_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    eor r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB6_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB6_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB6_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB6_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB6_2
+; CHECK-ARM-NEXT:  .LBB6_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB6_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB6_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xor_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB6_1
+; CHECK-THUMB2-NEXT:  .LBB6_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB6_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    eor r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB6_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB6_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB6_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB6_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB6_2
+; CHECK-THUMB2-NEXT:  .LBB6_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB6_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB6_1
+; CHECK-THUMB2-NEXT:    b .LBB6_5
+; CHECK-THUMB2-NEXT:  .LBB6_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xor  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_max_i8() {
+; CHECK-ARM-LABEL: test_max_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB7_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB7_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sxtb r0, r1
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r0, #1
+; CHECK-ARM-NEXT:    movgt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB7_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB7_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB7_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB7_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB7_2
+; CHECK-ARM-NEXT:  .LBB7_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB7_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB7_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_max_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB7_1
+; CHECK-THUMB2-NEXT:  .LBB7_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB7_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sxtb r0, r1
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    it gt
+; CHECK-THUMB2-NEXT:    movgt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB7_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB7_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB7_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB7_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB7_2
+; CHECK-THUMB2-NEXT:  .LBB7_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB7_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB7_1
+; CHECK-THUMB2-NEXT:    b .LBB7_5
+; CHECK-THUMB2-NEXT:  .LBB7_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw max  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_min_i8() {
+; CHECK-ARM-LABEL: test_min_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB8_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB8_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sxtb r0, r1
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r0, #2
+; CHECK-ARM-NEXT:    movlt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:  .LBB8_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB8_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB8_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB8_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB8_2
+; CHECK-ARM-NEXT:  .LBB8_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB8_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxtb r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB8_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_min_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB8_1
+; CHECK-THUMB2-NEXT:  .LBB8_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB8_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sxtb r0, r1
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r0, #2
+; CHECK-THUMB2-NEXT:    it lt
+; CHECK-THUMB2-NEXT:    movlt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:  .LBB8_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB8_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB8_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB8_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB8_2
+; CHECK-THUMB2-NEXT:  .LBB8_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB8_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxtb r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB8_1
+; CHECK-THUMB2-NEXT:    b .LBB8_5
+; CHECK-THUMB2-NEXT:  .LBB8_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw min  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_umax_i8() {
+; CHECK-ARM-LABEL: test_umax_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    mov r11, sp
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB9_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB9_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    uxtb r1, r12
+; CHECK-ARM-NEXT:    mov lr, #1
+; CHECK-ARM-NEXT:    cmp r1, #1
+; CHECK-ARM-NEXT:    movhi lr, r12
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r12, r12
+; CHECK-ARM-NEXT:  .LBB9_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB9_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r12
+; CHECK-ARM-NEXT:    bne .LBB9_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB9_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, lr, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB9_2
+; CHECK-ARM-NEXT:  .LBB9_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB9_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB9_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov sp, r11
+; CHECK-ARM-NEXT:    pop {r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_umax_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB9_1
+; CHECK-THUMB2-NEXT:  .LBB9_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB9_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr.w r12, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    uxtb.w r1, r12
+; CHECK-THUMB2-NEXT:    mov.w lr, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    it hi
+; CHECK-THUMB2-NEXT:    movhi lr, r12
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb.w r12, r12
+; CHECK-THUMB2-NEXT:  .LBB9_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB9_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r12
+; CHECK-THUMB2-NEXT:    bne .LBB9_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB9_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, lr, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB9_2
+; CHECK-THUMB2-NEXT:  .LBB9_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB9_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB9_1
+; CHECK-THUMB2-NEXT:    b .LBB9_5
+; CHECK-THUMB2-NEXT:  .LBB9_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw umax  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+define i8 @test_umin_i8() {
+; CHECK-ARM-LABEL: test_umin_i8:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    mov r11, sp
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    ldrb r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB10_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB10_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    uxtb r1, r12
+; CHECK-ARM-NEXT:    mov lr, #1
+; CHECK-ARM-NEXT:    cmp r1, #2
+; CHECK-ARM-NEXT:    movlo lr, r12
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-ARM-NEXT:    uxtb r12, r12
+; CHECK-ARM-NEXT:  .LBB10_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB10_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexb r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r12
+; CHECK-ARM-NEXT:    bne .LBB10_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB10_2 Depth=2
+; CHECK-ARM-NEXT:    strexb r2, lr, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB10_2
+; CHECK-ARM-NEXT:  .LBB10_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB10_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB10_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov sp, r11
+; CHECK-ARM-NEXT:    pop {r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_umin_i8:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    ldrb r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB10_1
+; CHECK-THUMB2-NEXT:  .LBB10_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB10_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr.w r12, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    uxtb.w r1, r12
+; CHECK-THUMB2-NEXT:    mov.w lr, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #2
+; CHECK-THUMB2-NEXT:    it lo
+; CHECK-THUMB2-NEXT:    movlo lr, r12
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i8
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i8
+; CHECK-THUMB2-NEXT:    uxtb.w r12, r12
+; CHECK-THUMB2-NEXT:  .LBB10_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB10_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexb r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r12
+; CHECK-THUMB2-NEXT:    bne .LBB10_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB10_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexb r2, lr, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB10_2
+; CHECK-THUMB2-NEXT:  .LBB10_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB10_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB10_1
+; CHECK-THUMB2-NEXT:    b .LBB10_5
+; CHECK-THUMB2-NEXT:  .LBB10_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw umin  i8* @atomic_i8, i8 1 monotonic
+  ret i8 %0
+}
+
+
+define i16 @test_xchg_i16() {
+; CHECK-ARM-LABEL: test_xchg_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB11_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB11_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB11_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB11_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB11_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB11_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB11_2
+; CHECK-ARM-NEXT:  .LBB11_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB11_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB11_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xchg_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB11_1
+; CHECK-THUMB2-NEXT:  .LBB11_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB11_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB11_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB11_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB11_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB11_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB11_2
+; CHECK-THUMB2-NEXT:  .LBB11_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB11_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB11_1
+; CHECK-THUMB2-NEXT:    b .LBB11_5
+; CHECK-THUMB2-NEXT:  .LBB11_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xchg  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_add_i16() {
+; CHECK-ARM-LABEL: test_add_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB12_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB12_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB12_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB12_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB12_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB12_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB12_2
+; CHECK-ARM-NEXT:  .LBB12_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB12_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB12_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_add_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB12_1
+; CHECK-THUMB2-NEXT:  .LBB12_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB12_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB12_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB12_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB12_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB12_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB12_2
+; CHECK-THUMB2-NEXT:  .LBB12_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB12_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB12_1
+; CHECK-THUMB2-NEXT:    b .LBB12_5
+; CHECK-THUMB2-NEXT:  .LBB12_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw add  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_sub_i16() {
+; CHECK-ARM-LABEL: test_sub_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB13_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB13_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB13_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB13_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB13_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB13_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB13_2
+; CHECK-ARM-NEXT:  .LBB13_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB13_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB13_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_sub_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB13_1
+; CHECK-THUMB2-NEXT:  .LBB13_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB13_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sub.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB13_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB13_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB13_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB13_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB13_2
+; CHECK-THUMB2-NEXT:  .LBB13_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB13_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB13_1
+; CHECK-THUMB2-NEXT:    b .LBB13_5
+; CHECK-THUMB2-NEXT:  .LBB13_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw sub  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_and_i16() {
+; CHECK-ARM-LABEL: test_and_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB14_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB14_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    and r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB14_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB14_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB14_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB14_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB14_2
+; CHECK-ARM-NEXT:  .LBB14_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB14_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB14_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_and_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB14_1
+; CHECK-THUMB2-NEXT:  .LBB14_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB14_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    and r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB14_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB14_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB14_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB14_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB14_2
+; CHECK-THUMB2-NEXT:  .LBB14_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB14_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB14_1
+; CHECK-THUMB2-NEXT:    b .LBB14_5
+; CHECK-THUMB2-NEXT:  .LBB14_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw and  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_nand_i16() {
+; CHECK-ARM-LABEL: test_nand_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB15_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB15_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mvn r0, r1
+; CHECK-ARM-NEXT:    mvn r2, #1
+; CHECK-ARM-NEXT:    orr r12, r0, r2
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB15_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB15_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB15_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB15_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB15_2
+; CHECK-ARM-NEXT:  .LBB15_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB15_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB15_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_nand_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB15_1
+; CHECK-THUMB2-NEXT:  .LBB15_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB15_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mvn r0, #1
+; CHECK-THUMB2-NEXT:    orn r12, r0, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB15_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB15_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB15_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB15_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB15_2
+; CHECK-THUMB2-NEXT:  .LBB15_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB15_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB15_1
+; CHECK-THUMB2-NEXT:    b .LBB15_5
+; CHECK-THUMB2-NEXT:  .LBB15_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw nand  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_or_i16() {
+; CHECK-ARM-LABEL: test_or_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB16_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB16_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    orr r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB16_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB16_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB16_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB16_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB16_2
+; CHECK-ARM-NEXT:  .LBB16_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB16_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB16_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_or_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB16_1
+; CHECK-THUMB2-NEXT:  .LBB16_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB16_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    orr r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB16_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB16_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB16_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB16_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB16_2
+; CHECK-THUMB2-NEXT:  .LBB16_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB16_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB16_1
+; CHECK-THUMB2-NEXT:    b .LBB16_5
+; CHECK-THUMB2-NEXT:  .LBB16_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw or  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_xor_i16() {
+; CHECK-ARM-LABEL: test_xor_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB17_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB17_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    eor r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB17_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB17_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB17_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB17_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB17_2
+; CHECK-ARM-NEXT:  .LBB17_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB17_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB17_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xor_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB17_1
+; CHECK-THUMB2-NEXT:  .LBB17_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB17_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    eor r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB17_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB17_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB17_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB17_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB17_2
+; CHECK-THUMB2-NEXT:  .LBB17_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB17_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB17_1
+; CHECK-THUMB2-NEXT:    b .LBB17_5
+; CHECK-THUMB2-NEXT:  .LBB17_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xor  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_max_i16() {
+; CHECK-ARM-LABEL: test_max_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB18_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB18_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sxth r0, r1
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r0, #1
+; CHECK-ARM-NEXT:    movgt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB18_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB18_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB18_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB18_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB18_2
+; CHECK-ARM-NEXT:  .LBB18_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB18_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB18_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_max_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB18_1
+; CHECK-THUMB2-NEXT:  .LBB18_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB18_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sxth r0, r1
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    it gt
+; CHECK-THUMB2-NEXT:    movgt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB18_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB18_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB18_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB18_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB18_2
+; CHECK-THUMB2-NEXT:  .LBB18_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB18_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB18_1
+; CHECK-THUMB2-NEXT:    b .LBB18_5
+; CHECK-THUMB2-NEXT:  .LBB18_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw max  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_min_i16() {
+; CHECK-ARM-LABEL: test_min_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB19_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB19_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sxth r0, r1
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r0, #2
+; CHECK-ARM-NEXT:    movlt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:  .LBB19_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB19_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB19_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB19_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB19_2
+; CHECK-ARM-NEXT:  .LBB19_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB19_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    uxth r1, r1
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB19_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_min_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB19_1
+; CHECK-THUMB2-NEXT:  .LBB19_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB19_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sxth r0, r1
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r0, #2
+; CHECK-THUMB2-NEXT:    it lt
+; CHECK-THUMB2-NEXT:    movlt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB19_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB19_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB19_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB19_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB19_2
+; CHECK-THUMB2-NEXT:  .LBB19_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB19_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB19_1
+; CHECK-THUMB2-NEXT:    b .LBB19_5
+; CHECK-THUMB2-NEXT:  .LBB19_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw min  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_umax_i16() {
+; CHECK-ARM-LABEL: test_umax_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    mov r11, sp
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB20_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB20_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    uxth r1, r12
+; CHECK-ARM-NEXT:    mov lr, #1
+; CHECK-ARM-NEXT:    cmp r1, #1
+; CHECK-ARM-NEXT:    movhi lr, r12
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r12, r12
+; CHECK-ARM-NEXT:  .LBB20_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB20_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r12
+; CHECK-ARM-NEXT:    bne .LBB20_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB20_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, lr, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB20_2
+; CHECK-ARM-NEXT:  .LBB20_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB20_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB20_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov sp, r11
+; CHECK-ARM-NEXT:    pop {r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_umax_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB20_1
+; CHECK-THUMB2-NEXT:  .LBB20_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB20_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr.w r12, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    uxth.w r1, r12
+; CHECK-THUMB2-NEXT:    mov.w lr, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    it hi
+; CHECK-THUMB2-NEXT:    movhi lr, r12
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth.w r12, r12
+; CHECK-THUMB2-NEXT:  .LBB20_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB20_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r12
+; CHECK-THUMB2-NEXT:    bne .LBB20_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB20_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, lr, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB20_2
+; CHECK-THUMB2-NEXT:  .LBB20_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB20_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB20_1
+; CHECK-THUMB2-NEXT:    b .LBB20_5
+; CHECK-THUMB2-NEXT:  .LBB20_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw umax  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+define i16 @test_umin_i16() {
+; CHECK-ARM-LABEL: test_umin_i16:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r11, lr}
+; CHECK-ARM-NEXT:    mov r11, sp
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB21_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB21_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r12, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    uxth r1, r12
+; CHECK-ARM-NEXT:    mov lr, #1
+; CHECK-ARM-NEXT:    cmp r1, #2
+; CHECK-ARM-NEXT:    movlo lr, r12
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-ARM-NEXT:    uxth r12, r12
+; CHECK-ARM-NEXT:  .LBB21_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB21_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r12
+; CHECK-ARM-NEXT:    bne .LBB21_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB21_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, lr, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB21_2
+; CHECK-ARM-NEXT:  .LBB21_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB21_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB21_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov sp, r11
+; CHECK-ARM-NEXT:    pop {r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_umin_i16:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB21_1
+; CHECK-THUMB2-NEXT:  .LBB21_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB21_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr.w r12, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    uxth.w r1, r12
+; CHECK-THUMB2-NEXT:    mov.w lr, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #2
+; CHECK-THUMB2-NEXT:    it lo
+; CHECK-THUMB2-NEXT:    movlo lr, r12
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i16
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i16
+; CHECK-THUMB2-NEXT:    uxth.w r12, r12
+; CHECK-THUMB2-NEXT:  .LBB21_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB21_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r12
+; CHECK-THUMB2-NEXT:    bne .LBB21_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB21_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, lr, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB21_2
+; CHECK-THUMB2-NEXT:  .LBB21_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB21_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB21_1
+; CHECK-THUMB2-NEXT:    b .LBB21_5
+; CHECK-THUMB2-NEXT:  .LBB21_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw umin  i16* @atomic_i16, i16 1 monotonic
+  ret i16 %0
+}
+
+
+define i32 @test_xchg_i32() {
+; CHECK-ARM-LABEL: test_xchg_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB22_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB22_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:  .LBB22_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB22_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB22_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB22_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB22_2
+; CHECK-ARM-NEXT:  .LBB22_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB22_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB22_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xchg_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB22_1
+; CHECK-THUMB2-NEXT:  .LBB22_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB22_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:  .LBB22_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB22_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB22_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB22_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB22_2
+; CHECK-THUMB2-NEXT:  .LBB22_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB22_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB22_1
+; CHECK-THUMB2-NEXT:    b .LBB22_5
+; CHECK-THUMB2-NEXT:  .LBB22_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xchg  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_add_i32() {
+; CHECK-ARM-LABEL: test_add_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB23_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB23_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB23_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB23_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB23_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB23_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB23_2
+; CHECK-ARM-NEXT:  .LBB23_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB23_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB23_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_add_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB23_1
+; CHECK-THUMB2-NEXT:  .LBB23_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB23_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB23_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB23_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB23_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB23_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB23_2
+; CHECK-THUMB2-NEXT:  .LBB23_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB23_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB23_1
+; CHECK-THUMB2-NEXT:    b .LBB23_5
+; CHECK-THUMB2-NEXT:  .LBB23_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw add  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_sub_i32() {
+; CHECK-ARM-LABEL: test_sub_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB24_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB24_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB24_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB24_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB24_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB24_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB24_2
+; CHECK-ARM-NEXT:  .LBB24_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB24_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB24_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_sub_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB24_1
+; CHECK-THUMB2-NEXT:  .LBB24_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB24_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    sub.w r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB24_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB24_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB24_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB24_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB24_2
+; CHECK-THUMB2-NEXT:  .LBB24_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB24_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB24_1
+; CHECK-THUMB2-NEXT:    b .LBB24_5
+; CHECK-THUMB2-NEXT:  .LBB24_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw sub  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_and_i32() {
+; CHECK-ARM-LABEL: test_and_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB25_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB25_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    and r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB25_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB25_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB25_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB25_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB25_2
+; CHECK-ARM-NEXT:  .LBB25_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB25_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB25_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_and_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB25_1
+; CHECK-THUMB2-NEXT:  .LBB25_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB25_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    and r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB25_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB25_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB25_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB25_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB25_2
+; CHECK-THUMB2-NEXT:  .LBB25_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB25_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB25_1
+; CHECK-THUMB2-NEXT:    b .LBB25_5
+; CHECK-THUMB2-NEXT:  .LBB25_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw and  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_nand_i32() {
+; CHECK-ARM-LABEL: test_nand_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB26_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB26_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mvn r0, r1
+; CHECK-ARM-NEXT:    mvn r2, #1
+; CHECK-ARM-NEXT:    orr r12, r0, r2
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB26_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB26_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB26_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB26_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB26_2
+; CHECK-ARM-NEXT:  .LBB26_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB26_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB26_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_nand_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB26_1
+; CHECK-THUMB2-NEXT:  .LBB26_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB26_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mvn r0, #1
+; CHECK-THUMB2-NEXT:    orn r12, r0, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB26_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB26_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB26_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB26_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB26_2
+; CHECK-THUMB2-NEXT:  .LBB26_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB26_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB26_1
+; CHECK-THUMB2-NEXT:    b .LBB26_5
+; CHECK-THUMB2-NEXT:  .LBB26_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw nand  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_or_i32() {
+; CHECK-ARM-LABEL: test_or_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB27_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB27_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    orr r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB27_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB27_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB27_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB27_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB27_2
+; CHECK-ARM-NEXT:  .LBB27_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB27_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB27_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_or_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB27_1
+; CHECK-THUMB2-NEXT:  .LBB27_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB27_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    orr r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB27_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB27_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB27_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB27_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB27_2
+; CHECK-THUMB2-NEXT:  .LBB27_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB27_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB27_1
+; CHECK-THUMB2-NEXT:    b .LBB27_5
+; CHECK-THUMB2-NEXT:  .LBB27_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw or  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_xor_i32() {
+; CHECK-ARM-LABEL: test_xor_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB28_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB28_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    eor r12, r1, #1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB28_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB28_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB28_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB28_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB28_2
+; CHECK-ARM-NEXT:  .LBB28_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB28_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB28_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_xor_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB28_1
+; CHECK-THUMB2-NEXT:  .LBB28_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB28_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    eor r12, r1, #1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB28_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB28_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB28_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB28_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB28_2
+; CHECK-THUMB2-NEXT:  .LBB28_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB28_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB28_1
+; CHECK-THUMB2-NEXT:    b .LBB28_5
+; CHECK-THUMB2-NEXT:  .LBB28_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw xor  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_max_i32() {
+; CHECK-ARM-LABEL: test_max_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB29_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB29_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r1, #1
+; CHECK-ARM-NEXT:    movgt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB29_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB29_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB29_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB29_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB29_2
+; CHECK-ARM-NEXT:  .LBB29_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB29_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB29_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_max_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB29_1
+; CHECK-THUMB2-NEXT:  .LBB29_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB29_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    it gt
+; CHECK-THUMB2-NEXT:    movgt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB29_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB29_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB29_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB29_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB29_2
+; CHECK-THUMB2-NEXT:  .LBB29_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB29_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB29_1
+; CHECK-THUMB2-NEXT:    b .LBB29_5
+; CHECK-THUMB2-NEXT:  .LBB29_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw max  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_min_i32() {
+; CHECK-ARM-LABEL: test_min_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB30_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB30_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r1, #2
+; CHECK-ARM-NEXT:    movlt r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB30_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB30_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB30_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB30_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB30_2
+; CHECK-ARM-NEXT:  .LBB30_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB30_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB30_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_min_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB30_1
+; CHECK-THUMB2-NEXT:  .LBB30_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB30_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #2
+; CHECK-THUMB2-NEXT:    it lt
+; CHECK-THUMB2-NEXT:    movlt r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB30_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB30_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB30_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB30_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB30_2
+; CHECK-THUMB2-NEXT:  .LBB30_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB30_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB30_1
+; CHECK-THUMB2-NEXT:    b .LBB30_5
+; CHECK-THUMB2-NEXT:  .LBB30_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw min  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_umax_i32() {
+; CHECK-ARM-LABEL: test_umax_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB31_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB31_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r1, #1
+; CHECK-ARM-NEXT:    movhi r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB31_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB31_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB31_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB31_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB31_2
+; CHECK-ARM-NEXT:  .LBB31_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB31_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB31_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_umax_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB31_1
+; CHECK-THUMB2-NEXT:  .LBB31_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB31_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    it hi
+; CHECK-THUMB2-NEXT:    movhi r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB31_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB31_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB31_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB31_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB31_2
+; CHECK-THUMB2-NEXT:  .LBB31_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB31_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB31_1
+; CHECK-THUMB2-NEXT:    b .LBB31_5
+; CHECK-THUMB2-NEXT:  .LBB31_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw umax  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+define i32 @test_umin_i32() {
+; CHECK-ARM-LABEL: test_umin_i32:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-ARM-NEXT:    ldr r0, [r0]
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB32_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB32_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r12, #1
+; CHECK-ARM-NEXT:    cmp r1, #2
+; CHECK-ARM-NEXT:    movlo r12, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-ARM-NEXT:  .LBB32_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB32_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r0, [r3]
+; CHECK-ARM-NEXT:    cmp r0, r1
+; CHECK-ARM-NEXT:    bne .LBB32_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB32_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB32_2
+; CHECK-ARM-NEXT:  .LBB32_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB32_1 Depth=1
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    sub r1, r0, r1
+; CHECK-ARM-NEXT:    clz r1, r1
+; CHECK-ARM-NEXT:    lsr r1, r1, #5
+; CHECK-ARM-NEXT:    tst r1, #1
+; CHECK-ARM-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB32_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_umin_i32:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:    ldr r0, [r0]
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB32_1
+; CHECK-THUMB2-NEXT:  .LBB32_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB32_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov.w r12, #1
+; CHECK-THUMB2-NEXT:    cmp r1, #2
+; CHECK-THUMB2-NEXT:    it lo
+; CHECK-THUMB2-NEXT:    movlo r12, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i32
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i32
+; CHECK-THUMB2-NEXT:  .LBB32_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB32_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB32_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB32_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB32_2
+; CHECK-THUMB2-NEXT:  .LBB32_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB32_1 Depth=1
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    cmp r1, #1
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB32_1
+; CHECK-THUMB2-NEXT:    b .LBB32_5
+; CHECK-THUMB2-NEXT:  .LBB32_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw umin  i32* @atomic_i32, i32 1 monotonic
+  ret i32 %0
+}
+
+
+
+define i64 @test_xchg_i64() {
+; CHECK-ARM-LABEL: test_xchg_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB33_1
+; CHECK-ARM-NEXT:  .LBB33_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB33_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    mov r0, #0
+; CHECK-ARM-NEXT:    mov r8, #1
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:  .LBB33_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB33_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB33_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB33_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB33_2
+; CHECK-ARM-NEXT:  .LBB33_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB33_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB33_1
+; CHECK-ARM-NEXT:    b .LBB33_5
+; CHECK-ARM-NEXT:  .LBB33_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_xchg_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB33_1
+; CHECK-THUMB2-NEXT:  .LBB33_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB33_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    movs r0, #0
+; CHECK-THUMB2-NEXT:    mov.w r8, #1
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:  .LBB33_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB33_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB33_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB33_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB33_2
+; CHECK-THUMB2-NEXT:  .LBB33_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB33_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB33_1
+; CHECK-THUMB2-NEXT:    b .LBB33_5
+; CHECK-THUMB2-NEXT:  .LBB33_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw xchg  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_add_i64() {
+; CHECK-ARM-LABEL: test_add_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB34_1
+; CHECK-ARM-NEXT:  .LBB34_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB34_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    adds r8, r2, #1
+; CHECK-ARM-NEXT:    adc r0, r1, #0
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB34_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB34_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB34_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB34_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB34_2
+; CHECK-ARM-NEXT:  .LBB34_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB34_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB34_1
+; CHECK-ARM-NEXT:    b .LBB34_5
+; CHECK-ARM-NEXT:  .LBB34_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_add_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB34_1
+; CHECK-THUMB2-NEXT:  .LBB34_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB34_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    adds.w r8, r2, #1
+; CHECK-THUMB2-NEXT:    adc r0, r1, #0
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB34_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB34_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB34_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB34_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB34_2
+; CHECK-THUMB2-NEXT:  .LBB34_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB34_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB34_1
+; CHECK-THUMB2-NEXT:    b .LBB34_5
+; CHECK-THUMB2-NEXT:  .LBB34_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw add  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_sub_i64() {
+; CHECK-ARM-LABEL: test_sub_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB35_1
+; CHECK-ARM-NEXT:  .LBB35_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB35_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    subs r8, r2, #1
+; CHECK-ARM-NEXT:    sbc r0, r1, #0
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB35_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB35_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB35_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB35_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB35_2
+; CHECK-ARM-NEXT:  .LBB35_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB35_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB35_1
+; CHECK-ARM-NEXT:    b .LBB35_5
+; CHECK-ARM-NEXT:  .LBB35_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_sub_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB35_1
+; CHECK-THUMB2-NEXT:  .LBB35_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB35_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    subs.w r8, r2, #1
+; CHECK-THUMB2-NEXT:    sbc r0, r1, #0
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB35_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB35_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB35_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB35_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB35_2
+; CHECK-THUMB2-NEXT:  .LBB35_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB35_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB35_1
+; CHECK-THUMB2-NEXT:    b .LBB35_5
+; CHECK-THUMB2-NEXT:  .LBB35_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw sub  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_and_i64() {
+; CHECK-ARM-LABEL: test_and_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB36_1
+; CHECK-ARM-NEXT:  .LBB36_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB36_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    and r8, r2, #1
+; CHECK-ARM-NEXT:    mov r0, #0
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB36_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB36_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB36_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB36_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB36_2
+; CHECK-ARM-NEXT:  .LBB36_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB36_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB36_1
+; CHECK-ARM-NEXT:    b .LBB36_5
+; CHECK-ARM-NEXT:  .LBB36_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_and_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB36_1
+; CHECK-THUMB2-NEXT:  .LBB36_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB36_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    and r8, r2, #1
+; CHECK-THUMB2-NEXT:    movs r0, #0
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB36_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB36_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB36_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB36_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB36_2
+; CHECK-THUMB2-NEXT:  .LBB36_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB36_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB36_1
+; CHECK-THUMB2-NEXT:    b .LBB36_5
+; CHECK-THUMB2-NEXT:  .LBB36_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw and  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_nand_i64() {
+; CHECK-ARM-LABEL: test_nand_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB37_1
+; CHECK-ARM-NEXT:  .LBB37_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB37_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    mvn r0, r2
+; CHECK-ARM-NEXT:    mvn r3, #1
+; CHECK-ARM-NEXT:    orr r8, r0, r3
+; CHECK-ARM-NEXT:    mvn r0, #0
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB37_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB37_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB37_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB37_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB37_2
+; CHECK-ARM-NEXT:  .LBB37_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB37_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB37_1
+; CHECK-ARM-NEXT:    b .LBB37_5
+; CHECK-ARM-NEXT:  .LBB37_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_nand_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB37_1
+; CHECK-THUMB2-NEXT:  .LBB37_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB37_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    mvn r0, #1
+; CHECK-THUMB2-NEXT:    orn r8, r0, r2
+; CHECK-THUMB2-NEXT:    mov.w r0, #-1
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB37_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB37_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB37_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB37_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB37_2
+; CHECK-THUMB2-NEXT:  .LBB37_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB37_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB37_1
+; CHECK-THUMB2-NEXT:    b .LBB37_5
+; CHECK-THUMB2-NEXT:  .LBB37_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw nand  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_or_i64() {
+; CHECK-ARM-LABEL: test_or_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB38_1
+; CHECK-ARM-NEXT:  .LBB38_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB38_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    orr r8, r2, #1
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r1
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB38_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB38_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB38_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB38_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB38_2
+; CHECK-ARM-NEXT:  .LBB38_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB38_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB38_1
+; CHECK-ARM-NEXT:    b .LBB38_5
+; CHECK-ARM-NEXT:  .LBB38_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_or_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB38_1
+; CHECK-THUMB2-NEXT:  .LBB38_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB38_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    orr r8, r2, #1
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r1
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB38_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB38_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB38_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB38_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB38_2
+; CHECK-THUMB2-NEXT:  .LBB38_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB38_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB38_1
+; CHECK-THUMB2-NEXT:    b .LBB38_5
+; CHECK-THUMB2-NEXT:  .LBB38_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw or  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+define i64 @test_xor_i64() {
+; CHECK-ARM-LABEL: test_xor_i64:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-ARM-NEXT:    ldr r1, [r0]
+; CHECK-ARM-NEXT:    ldr r0, [r0, #4]
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB39_1
+; CHECK-ARM-NEXT:  .LBB39_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB39_2 Depth 2
+; CHECK-ARM-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-ARM-NEXT:    eor r8, r2, #1
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r1
+; CHECK-ARM-NEXT:    mov r6, r2
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-ARM-NEXT:  .LBB39_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB39_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB39_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB39_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r0, #0
+; CHECK-ARM-NEXT:    bne .LBB39_2
+; CHECK-ARM-NEXT:  .LBB39_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB39_1 Depth=1
+; CHECK-ARM-NEXT:    mov r0, r5
+; CHECK-ARM-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r3, r0, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    eor r2, r1, r2
+; CHECK-ARM-NEXT:    orr r2, r2, r3
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-ARM-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB39_1
+; CHECK-ARM-NEXT:    b .LBB39_5
+; CHECK-ARM-NEXT:  .LBB39_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_xor_i64:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #16
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:    ldr r1, [r0]
+; CHECK-THUMB2-NEXT:    ldr r0, [r0, #4]
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB39_1
+; CHECK-THUMB2-NEXT:  .LBB39_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB39_2 Depth 2
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    eor r8, r2, #1
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r1
+; CHECK-THUMB2-NEXT:    mov r6, r2
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_i64
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_i64
+; CHECK-THUMB2-NEXT:  .LBB39_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB39_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB39_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB39_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r0, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    bne .LBB39_2
+; CHECK-THUMB2-NEXT:  .LBB39_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB39_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r0, r5
+; CHECK-THUMB2-NEXT:    str r0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eor.w r3, r0, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    eors r2, r1
+; CHECK-THUMB2-NEXT:    orrs r2, r3
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    str r0, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB39_1
+; CHECK-THUMB2-NEXT:    b .LBB39_5
+; CHECK-THUMB2-NEXT:  .LBB39_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    add sp, #16
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw xor  i64* @atomic_i64, i64 1 monotonic
+  ret i64 %0
+}
+
+
+; ; Test floats
+; define half @test_xchg_half() {
+; entry:
+;   %0  = atomicrmw xchg half* @atomic_half, half 1.0 monotonic
+;   ret half %0
+; }
+define half @test_fadd_half() {
+; CHECK-ARM-LABEL: test_fadd_half:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_half
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_half
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    vmov s0, r0
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB40_1
+; CHECK-ARM-NEXT:  .LBB40_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB40_2 Depth 2
+; CHECK-ARM-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    vmov s0, r0
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r12, s0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_half
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_half
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:  .LBB40_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB40_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r1, [r3]
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    bne .LBB40_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB40_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB40_2
+; CHECK-ARM-NEXT:  .LBB40_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB40_1 Depth=1
+; CHECK-ARM-NEXT:    sub r0, r1, r0
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    cmp r0, #1
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB40_1
+; CHECK-ARM-NEXT:    b .LBB40_5
+; CHECK-ARM-NEXT:  .LBB40_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_fadd_half:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #24
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_half
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_half
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    vmov s0, r0
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #20] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB40_1
+; CHECK-THUMB2-NEXT:  .LBB40_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB40_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #20] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    mov r1, r0
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-THUMB2-NEXT:    uxth r0, r0
+; CHECK-THUMB2-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r12, r0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_half
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_half
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB40_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB40_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB40_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB40_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB40_2
+; CHECK-THUMB2-NEXT:  .LBB40_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB40_1 Depth=1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    mov r1, r0
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #16] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #20] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB40_1
+; CHECK-THUMB2-NEXT:    b .LBB40_5
+; CHECK-THUMB2-NEXT:  .LBB40_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #16] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    uxth r0, r0
+; CHECK-THUMB2-NEXT:    add sp, #24
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw fadd half* @atomic_half, half 1.0 monotonic
+  ret half %0
+}
+define half @test_fsub_half() {
+; CHECK-ARM-LABEL: test_fsub_half:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_half
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_half
+; CHECK-ARM-NEXT:    ldrh r0, [r0]
+; CHECK-ARM-NEXT:    vmov s0, r0
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    b .LBB41_1
+; CHECK-ARM-NEXT:  .LBB41_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB41_2 Depth 2
+; CHECK-ARM-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    vmov s0, r0
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vmov.f32 s2, #-1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r12, s0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_half
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_half
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:  .LBB41_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB41_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexh r1, [r3]
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    bne .LBB41_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB41_2 Depth=2
+; CHECK-ARM-NEXT:    strexh r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB41_2
+; CHECK-ARM-NEXT:  .LBB41_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB41_1 Depth=1
+; CHECK-ARM-NEXT:    sub r0, r1, r0
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-ARM-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    cmp r0, #1
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    bne .LBB41_1
+; CHECK-ARM-NEXT:    b .LBB41_5
+; CHECK-ARM-NEXT:  .LBB41_5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vcvtb.f16.f32 s0, s0
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    uxth r0, r0
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_fsub_half:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push {r7, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #24
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_half
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_half
+; CHECK-THUMB2-NEXT:    ldrh r0, [r0]
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    vmov s0, r0
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #20] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB41_1
+; CHECK-THUMB2-NEXT:  .LBB41_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB41_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #20] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    mov r1, r0
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vmov.f32 s2, #-1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-THUMB2-NEXT:    uxth r0, r0
+; CHECK-THUMB2-NEXT:    str r0, [sp, #8] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    mov r12, r0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_half
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_half
+; CHECK-THUMB2-NEXT:    uxth r1, r1
+; CHECK-THUMB2-NEXT:  .LBB41_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB41_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexh r0, [r3]
+; CHECK-THUMB2-NEXT:    cmp r0, r1
+; CHECK-THUMB2-NEXT:    bne .LBB41_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB41_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexh r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB41_2
+; CHECK-THUMB2-NEXT:  .LBB41_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB41_1 Depth=1
+; CHECK-THUMB2-NEXT:    subs r1, r0, r1
+; CHECK-THUMB2-NEXT:    clz r1, r1
+; CHECK-THUMB2-NEXT:    lsrs r1, r1, #5
+; CHECK-THUMB2-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bl __gnu_h2f_ieee
+; CHECK-THUMB2-NEXT:    mov r1, r0
+; CHECK-THUMB2-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #16] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #20] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB41_1
+; CHECK-THUMB2-NEXT:    b .LBB41_5
+; CHECK-THUMB2-NEXT:  .LBB41_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #16] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    bl __gnu_f2h_ieee
+; CHECK-THUMB2-NEXT:    uxth r0, r0
+; CHECK-THUMB2-NEXT:    add sp, #24
+; CHECK-THUMB2-NEXT:    pop {r7, pc}
+entry:
+  %0  = atomicrmw fsub half* @atomic_half, half 1.0 monotonic
+  ret half %0
+}
+; define float @test_xchg_float() {
+; entry:
+;   %0  = atomicrmw xchg float* @atomic_float, float 1.0 monotonic
+;   ret float %0
+; }
+define float @test_fadd_float() {
+; CHECK-ARM-LABEL: test_fadd_float:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_float
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_float
+; CHECK-ARM-NEXT:    vldr s0, [r0]
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB42_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB42_2 Depth 2
+; CHECK-ARM-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f32 s2, s0, s2
+; CHECK-ARM-NEXT:    vmov r12, s2
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_float
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_float
+; CHECK-ARM-NEXT:  .LBB42_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB42_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r1, [r3]
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    bne .LBB42_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB42_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB42_2
+; CHECK-ARM-NEXT:  .LBB42_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB42_1 Depth=1
+; CHECK-ARM-NEXT:    sub r0, r1, r0
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    tst r0, #1
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB42_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_fadd_float:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_float
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_float
+; CHECK-THUMB2-NEXT:    vldr s0, [r0]
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB42_1
+; CHECK-THUMB2-NEXT:  .LBB42_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB42_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov.f32 s2, #1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f32 s2, s0, s2
+; CHECK-THUMB2-NEXT:    vmov r12, s2
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_float
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_float
+; CHECK-THUMB2-NEXT:  .LBB42_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB42_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r1, [r3]
+; CHECK-THUMB2-NEXT:    cmp r1, r0
+; CHECK-THUMB2-NEXT:    bne .LBB42_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB42_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB42_2
+; CHECK-THUMB2-NEXT:  .LBB42_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB42_1 Depth=1
+; CHECK-THUMB2-NEXT:    subs r0, r1, r0
+; CHECK-THUMB2-NEXT:    clz r0, r0
+; CHECK-THUMB2-NEXT:    lsrs r0, r0, #5
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB42_1
+; CHECK-THUMB2-NEXT:    b .LBB42_5
+; CHECK-THUMB2-NEXT:  .LBB42_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw fadd float* @atomic_float, float 1.0 monotonic
+  ret float %0
+}
+define float @test_fsub_float() {
+; CHECK-ARM-LABEL: test_fsub_float:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    sub sp, sp, #8
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_float
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_float
+; CHECK-ARM-NEXT:    vldr s0, [r0]
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:  .LBB43_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB43_2 Depth 2
+; CHECK-ARM-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vmov.f32 s2, #-1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f32 s2, s0, s2
+; CHECK-ARM-NEXT:    vmov r12, s2
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_float
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_float
+; CHECK-ARM-NEXT:  .LBB43_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB43_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrex r1, [r3]
+; CHECK-ARM-NEXT:    cmp r1, r0
+; CHECK-ARM-NEXT:    bne .LBB43_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB43_2 Depth=2
+; CHECK-ARM-NEXT:    strex r2, r12, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB43_2
+; CHECK-ARM-NEXT:  .LBB43_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB43_1 Depth=1
+; CHECK-ARM-NEXT:    sub r0, r1, r0
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov s0, r1
+; CHECK-ARM-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-ARM-NEXT:    tst r0, #1
+; CHECK-ARM-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB43_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-ARM-NEXT:    vmov r0, s0
+; CHECK-ARM-NEXT:    add sp, sp, #8
+; CHECK-ARM-NEXT:    bx lr
+;
+; CHECK-THUMB2-LABEL: test_fsub_float:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    sub sp, #8
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_float
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_float
+; CHECK-THUMB2-NEXT:    vldr s0, [r0]
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB43_1
+; CHECK-THUMB2-NEXT:  .LBB43_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB43_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov.f32 s2, #-1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f32 s2, s0, s2
+; CHECK-THUMB2-NEXT:    vmov r12, s2
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_float
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_float
+; CHECK-THUMB2-NEXT:  .LBB43_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB43_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrex r1, [r3]
+; CHECK-THUMB2-NEXT:    cmp r1, r0
+; CHECK-THUMB2-NEXT:    bne .LBB43_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB43_2 Depth=2
+; CHECK-THUMB2-NEXT:    strex r2, r12, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB43_2
+; CHECK-THUMB2-NEXT:  .LBB43_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB43_1 Depth=1
+; CHECK-THUMB2-NEXT:    subs r0, r1, r0
+; CHECK-THUMB2-NEXT:    clz r0, r0
+; CHECK-THUMB2-NEXT:    lsrs r0, r0, #5
+; CHECK-THUMB2-NEXT:    vmov s0, r1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #1
+; CHECK-THUMB2-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB43_1
+; CHECK-THUMB2-NEXT:    b .LBB43_5
+; CHECK-THUMB2-NEXT:  .LBB43_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr s0, [sp] @ 4-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, s0
+; CHECK-THUMB2-NEXT:    add sp, #8
+; CHECK-THUMB2-NEXT:    bx lr
+entry:
+  %0  = atomicrmw fsub float* @atomic_float, float 1.0 monotonic
+  ret float %0
+}
+; define double @test_xchg_double() {
+; entry:
+;   %0  = atomicrmw xchg double* @atomic_double, double 1.0 monotonic
+;   ret double %0
+; }
+define double @test_fadd_double() {
+; CHECK-ARM-LABEL: test_fadd_double:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_double
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_double
+; CHECK-ARM-NEXT:    vldr d16, [r0]
+; CHECK-ARM-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-ARM-NEXT:  .LBB44_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB44_2 Depth 2
+; CHECK-ARM-NEXT:    vldr d16, [sp, #8] @ 8-byte Reload
+; CHECK-ARM-NEXT:    vmov.f64 d17, #1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f64 d17, d16, d17
+; CHECK-ARM-NEXT:    vmov r8, r0, d17
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    vmov r0, r1, d16
+; CHECK-ARM-NEXT:    mov r6, r0
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_double
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_double
+; CHECK-ARM-NEXT:  .LBB44_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB44_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB44_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB44_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r2, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB44_2
+; CHECK-ARM-NEXT:  .LBB44_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB44_1 Depth=1
+; CHECK-ARM-NEXT:    mov r2, r5
+; CHECK-ARM-NEXT:    eor r3, r2, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    eor r0, r1, r0
+; CHECK-ARM-NEXT:    orr r0, r0, r3
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov d16, r1, r2
+; CHECK-ARM-NEXT:    vstr d16, [sp] @ 8-byte Spill
+; CHECK-ARM-NEXT:    tst r0, #1
+; CHECK-ARM-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB44_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr d16, [sp] @ 8-byte Reload
+; CHECK-ARM-NEXT:    vmov r0, r1, d16
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_fadd_double:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #20
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_double
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_double
+; CHECK-THUMB2-NEXT:    vldr d16, [r0]
+; CHECK-THUMB2-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB44_1
+; CHECK-THUMB2-NEXT:  .LBB44_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB44_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr d16, [sp, #8] @ 8-byte Reload
+; CHECK-THUMB2-NEXT:    vmov.f64 d17, #1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f64 d17, d16, d17
+; CHECK-THUMB2-NEXT:    vmov r8, r0, d17
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    vmov r0, r1, d16
+; CHECK-THUMB2-NEXT:    mov r6, r0
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_double
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_double
+; CHECK-THUMB2-NEXT:  .LBB44_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB44_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB44_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB44_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r2, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB44_2
+; CHECK-THUMB2-NEXT:  .LBB44_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB44_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r2, r5
+; CHECK-THUMB2-NEXT:    eor.w r3, r2, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    eors r0, r1
+; CHECK-THUMB2-NEXT:    orrs r0, r3
+; CHECK-THUMB2-NEXT:    vmov d16, r1, r2
+; CHECK-THUMB2-NEXT:    vstr d16, [sp] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB44_1
+; CHECK-THUMB2-NEXT:    b .LBB44_5
+; CHECK-THUMB2-NEXT:  .LBB44_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr d16, [sp] @ 8-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, r1, d16
+; CHECK-THUMB2-NEXT:    add sp, #20
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw fadd double* @atomic_double, double 1.0 monotonic
+  ret double %0
+}
+define double @test_fsub_double() {
+; CHECK-ARM-LABEL: test_fsub_double:
+; CHECK-ARM:       @ %bb.0: @ %entry
+; CHECK-ARM-NEXT:    push {r4, r5, r6, r7, r8, r9, r11, lr}
+; CHECK-ARM-NEXT:    add r11, sp, #24
+; CHECK-ARM-NEXT:    sub sp, sp, #16
+; CHECK-ARM-NEXT:    movw r0, :lower16:atomic_double
+; CHECK-ARM-NEXT:    movt r0, :upper16:atomic_double
+; CHECK-ARM-NEXT:    vldr d16, [r0]
+; CHECK-ARM-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-ARM-NEXT:  .LBB45_1: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-ARM-NEXT:    @ Child Loop BB45_2 Depth 2
+; CHECK-ARM-NEXT:    vldr d16, [sp, #8] @ 8-byte Reload
+; CHECK-ARM-NEXT:    vmov.f64 d17, #-1.000000e+00
+; CHECK-ARM-NEXT:    vadd.f64 d17, d16, d17
+; CHECK-ARM-NEXT:    vmov r8, r0, d17
+; CHECK-ARM-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-ARM-NEXT:    mov r9, r0
+; CHECK-ARM-NEXT:    vmov r0, r1, d16
+; CHECK-ARM-NEXT:    mov r6, r0
+; CHECK-ARM-NEXT:    mov r7, r1
+; CHECK-ARM-NEXT:    movw r3, :lower16:atomic_double
+; CHECK-ARM-NEXT:    movt r3, :upper16:atomic_double
+; CHECK-ARM-NEXT:  .LBB45_2: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ Parent Loop BB45_1 Depth=1
+; CHECK-ARM-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-ARM-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-ARM-NEXT:    cmp r4, r6
+; CHECK-ARM-NEXT:    cmpeq r5, r7
+; CHECK-ARM-NEXT:    bne .LBB45_4
+; CHECK-ARM-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB45_2 Depth=2
+; CHECK-ARM-NEXT:    strexd r2, r8, r9, [r3]
+; CHECK-ARM-NEXT:    cmp r2, #0
+; CHECK-ARM-NEXT:    bne .LBB45_2
+; CHECK-ARM-NEXT:  .LBB45_4: @ %atomicrmw.start
+; CHECK-ARM-NEXT:    @ in Loop: Header=BB45_1 Depth=1
+; CHECK-ARM-NEXT:    mov r2, r5
+; CHECK-ARM-NEXT:    eor r3, r2, r1
+; CHECK-ARM-NEXT:    mov r1, r4
+; CHECK-ARM-NEXT:    eor r0, r1, r0
+; CHECK-ARM-NEXT:    orr r0, r0, r3
+; CHECK-ARM-NEXT:    clz r0, r0
+; CHECK-ARM-NEXT:    lsr r0, r0, #5
+; CHECK-ARM-NEXT:    vmov d16, r1, r2
+; CHECK-ARM-NEXT:    vstr d16, [sp] @ 8-byte Spill
+; CHECK-ARM-NEXT:    tst r0, #1
+; CHECK-ARM-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-ARM-NEXT:    beq .LBB45_1
+; CHECK-ARM-NEXT:  @ %bb.5: @ %atomicrmw.end
+; CHECK-ARM-NEXT:    vldr d16, [sp] @ 8-byte Reload
+; CHECK-ARM-NEXT:    vmov r0, r1, d16
+; CHECK-ARM-NEXT:    sub sp, r11, #24
+; CHECK-ARM-NEXT:    pop {r4, r5, r6, r7, r8, r9, r11, pc}
+;
+; CHECK-THUMB2-LABEL: test_fsub_double:
+; CHECK-THUMB2:       @ %bb.0: @ %entry
+; CHECK-THUMB2-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-THUMB2-NEXT:    sub sp, #20
+; CHECK-THUMB2-NEXT:    movw r0, :lower16:atomic_double
+; CHECK-THUMB2-NEXT:    movt r0, :upper16:atomic_double
+; CHECK-THUMB2-NEXT:    vldr d16, [r0]
+; CHECK-THUMB2-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    b .LBB45_1
+; CHECK-THUMB2-NEXT:  .LBB45_1: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ =>This Loop Header: Depth=1
+; CHECK-THUMB2-NEXT:    @ Child Loop BB45_2 Depth 2
+; CHECK-THUMB2-NEXT:    vldr d16, [sp, #8] @ 8-byte Reload
+; CHECK-THUMB2-NEXT:    vmov.f64 d17, #-1.000000e+00
+; CHECK-THUMB2-NEXT:    vadd.f64 d17, d16, d17
+; CHECK-THUMB2-NEXT:    vmov r8, r0, d17
+; CHECK-THUMB2-NEXT:    @ kill: def $r8 killed $r8 def $r8_r9
+; CHECK-THUMB2-NEXT:    mov r9, r0
+; CHECK-THUMB2-NEXT:    vmov r0, r1, d16
+; CHECK-THUMB2-NEXT:    mov r6, r0
+; CHECK-THUMB2-NEXT:    mov r7, r1
+; CHECK-THUMB2-NEXT:    movw r3, :lower16:atomic_double
+; CHECK-THUMB2-NEXT:    movt r3, :upper16:atomic_double
+; CHECK-THUMB2-NEXT:  .LBB45_2: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ Parent Loop BB45_1 Depth=1
+; CHECK-THUMB2-NEXT:    @ => This Inner Loop Header: Depth=2
+; CHECK-THUMB2-NEXT:    ldrexd r4, r5, [r3]
+; CHECK-THUMB2-NEXT:    cmp r4, r6
+; CHECK-THUMB2-NEXT:    it eq
+; CHECK-THUMB2-NEXT:    cmpeq r5, r7
+; CHECK-THUMB2-NEXT:    bne .LBB45_4
+; CHECK-THUMB2-NEXT:  @ %bb.3: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB45_2 Depth=2
+; CHECK-THUMB2-NEXT:    strexd r2, r8, r9, [r3]
+; CHECK-THUMB2-NEXT:    cmp r2, #0
+; CHECK-THUMB2-NEXT:    bne .LBB45_2
+; CHECK-THUMB2-NEXT:  .LBB45_4: @ %atomicrmw.start
+; CHECK-THUMB2-NEXT:    @ in Loop: Header=BB45_1 Depth=1
+; CHECK-THUMB2-NEXT:    mov r2, r5
+; CHECK-THUMB2-NEXT:    eor.w r3, r2, r1
+; CHECK-THUMB2-NEXT:    mov r1, r4
+; CHECK-THUMB2-NEXT:    eors r0, r1
+; CHECK-THUMB2-NEXT:    orrs r0, r3
+; CHECK-THUMB2-NEXT:    vmov d16, r1, r2
+; CHECK-THUMB2-NEXT:    vstr d16, [sp] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    cmp r0, #0
+; CHECK-THUMB2-NEXT:    vstr d16, [sp, #8] @ 8-byte Spill
+; CHECK-THUMB2-NEXT:    bne .LBB45_1
+; CHECK-THUMB2-NEXT:    b .LBB45_5
+; CHECK-THUMB2-NEXT:  .LBB45_5: @ %atomicrmw.end
+; CHECK-THUMB2-NEXT:    vldr d16, [sp] @ 8-byte Reload
+; CHECK-THUMB2-NEXT:    vmov r0, r1, d16
+; CHECK-THUMB2-NEXT:    add sp, #20
+; CHECK-THUMB2-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+entry:
+  %0  = atomicrmw fsub double* @atomic_double, double 1.0 monotonic
+  ret double %0
+}