[AtomicExpandPass] Always copy pcsections Metadata to expanded atomics

[melver]

Summary

When expanding IR atomics to target-specific atomics, copy all
!pcsections Metadata to expanded atomics automatically.

Depends on D130884

[melver]

Rebase.

[melver]

Rebase.

[vitalybuka]

I guess this patch can be tested with regual IR test?

[melver]

Add IR test.

PTAL.

[vitalybuka]

llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll

4310

I don't know how stable the order
often in such generated test I had to replace -NEXT with -DAG

9eadad51bafb741203e21d917a43d45c99fabe55
and
c0ee9e1b98cac2cc4b3369dbcf52486e4d294060
When it was -NEXT it failed bunch of bots, but I can't reproduce locally even with LLVM_REVERSE_ITERATION or LLVM_ENABLE_EXPENSIVE_CHECKS (I just noticed, that i didn't try gcc, which is used on those bots)

[arsenm]

llvm/lib/CodeGen/AtomicExpandPass.cpp

130–133

This seems too specific. Why do we need a new IRBuilder subclass just for this? Isn't there already an existing helper for metadata? We could also preserve a lot more metadata, like any aliasing info

[melver]

llvm/lib/CodeGen/AtomicExpandPass.cpp

130–133

There's no existing helper (except CollectMetadataToCopy()) AFAIK.

The alternative is to explicitly add a Builder.CollectMetadataToCopy everywhere, but I felt that this is more concise given it's done everywhere.

This subclass is only available to AtomicExpandPass. I tried to have a local BuildIRBuilder() function, but IR builder doesn't like being copied. Having a subclass that does something extra in the constructor seemed just as elegant and simple.

llvm/lib/CodeGen/AtomicExpandPass.cpp

private:
   void expandAtomicRMWToLibcall(AtomicRMWInst *I);
   void expandAtomicCASToLibcall(AtomicCmpXchgInst *I);
 
   friend bool
   llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
                                  CreateCmpXchgInstFun CreateCmpXchg);
 };
 
+// IRBuilder to be used for replacement atomic instructions.
+struct ReplacementIRBuilder : IRBuilder<> {
+  // Preserves the DebugLoc from I, and preserves still valid metadata.
+  explicit ReplacementIRBuilder(Instruction *I) : IRBuilder<>(I) {
+    this->CollectMetadataToCopy(I, {LLVMContext::MD_pcsections});
+  }
+};
+
 } // end anonymous namespace
 
 char AtomicExpand::ID = 0;
 
 char &llvm::AtomicExpandID = AtomicExpand::ID;
 
 INITIALIZE_PASS(AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false,
                 false)
else if (RMWI) {
       }
     } else if (CASI)
       MadeChange |= tryExpandAtomicCmpXchg(CASI);
   }
   return MadeChange;
 }
 
 bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order) {
-  IRBuilder<> Builder(I);
+  ReplacementIRBuilder Builder(I);
 
   auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);
 
   auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);
   // We have a guard here because not every atomic operation generates a
   // trailing fence.
   if (TrailingFence)
     TrailingFence->moveAfter(I);

 
 /// Convert an atomic load of a non-integral type to an integer load of the
 /// equivalent bitwidth.  See the function comment on
 /// convertAtomicStoreToIntegerType for background.
 LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
   auto *M = LI->getModule();
   Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout());
 
-  IRBuilder<> Builder(LI);
+  ReplacementIRBuilder Builder(LI);
 
   Value *Addr = LI->getPointerOperand();
   Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace());
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);
 
   auto *NewLI = Builder.CreateLoad(NewTy, NewAddr);
   NewLI->setAlignment(LI->getAlign());
   NewLI->setVolatile(LI->isVolatile());
   NewLI->setAtomic(LI->getOrdering(), LI->getSyncScopeID());
   LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
 
   Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType());
   LI->replaceAllUsesWith(NewVal);
   LI->eraseFromParent();
   return NewLI;
 }
 
 AtomicRMWInst *
 AtomicExpand::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {
   auto *M = RMWI->getModule();
   Type *NewTy =
       getCorrespondingIntegerType(RMWI->getType(), M->getDataLayout());
 
-  IRBuilder<> Builder(RMWI);
+  ReplacementIRBuilder Builder(RMWI);
 
   Value *Addr = RMWI->getPointerOperand();
   Value *Val = RMWI->getValOperand();
   Type *PT = PointerType::get(NewTy, RMWI->getPointerAddressSpace());
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);
   Value *NewVal = Val->getType()->isPointerTy()
                       ? Builder.CreatePtrToInt(Val, NewTy)
                       : Builder.CreateBitCast(Val, NewTy);
case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
     SI->setAtomic(AtomicOrdering::NotAtomic);
     return true;
   default:
     llvm_unreachable("Unhandled case in tryExpandAtomicStore");
   }
 }
 
 bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) {
-  IRBuilder<> Builder(LI);
+  ReplacementIRBuilder Builder(LI);
 
   // On some architectures, load-linked instructions are atomic for larger
   // sizes than normal loads. For example, the only 64-bit load guaranteed
   // to be single-copy atomic by ARM is an ldrexd (A3.5.3).
   Value *Val = TLI->emitLoadLinked(Builder, LI->getType(),
                                    LI->getPointerOperand(), LI->getOrdering());
   TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
 
   LI->replaceAllUsesWith(Val);
   LI->eraseFromParent();
 
   return true;
 }
 
 bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) {
-  IRBuilder<> Builder(LI);
+  ReplacementIRBuilder Builder(LI);
   AtomicOrdering Order = LI->getOrdering();
   if (Order == AtomicOrdering::Unordered)
     Order = AtomicOrdering::Monotonic;
 
   Value *Addr = LI->getPointerOperand();
   Type *Ty = LI->getType();
   Constant *DummyVal = Constant::getNullValue(Ty);
 

 /// equivalent bitwidth.  We used to not support floating point or vector
 /// atomics in the IR at all.  The backends learned to deal with the bitcast
 /// idiom because that was the only way of expressing the notion of a atomic
 /// float or vector store.  The long term plan is to teach each backend to
 /// instruction select from the original atomic store, but as a migration
 /// mechanism, we convert back to the old format which the backends understand.
 /// Each backend will need individual work to recognize the new format.
 StoreInst *AtomicExpand::convertAtomicStoreToIntegerType(StoreInst *SI) {
-  IRBuilder<> Builder(SI);
+  ReplacementIRBuilder Builder(SI);
   auto *M = SI->getModule();
   Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(),
                                             M->getDataLayout());
   Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy);
 
   Value *Addr = SI->getPointerOperand();
   Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace());
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);

 
 void AtomicExpand::expandAtomicStore(StoreInst *SI) {
   // This function is only called on atomic stores that are too large to be
   // atomic if implemented as a native store. So we replace them by an
   // atomic swap, that can be implemented for example as a ldrex/strex on ARM
   // or lock cmpxchg8/16b on X86, as these are atomic for larger sizes.
   // It is the responsibility of the target to only signal expansion via
   // shouldExpandAtomicRMW in cases where this is required and possible.
-  IRBuilder<> Builder(SI);
+  ReplacementIRBuilder Builder(SI);
   AtomicOrdering Ordering = SI->getOrdering();
   assert(Ordering != AtomicOrdering::NotAtomic);
   AtomicOrdering RMWOrdering = Ordering == AtomicOrdering::Unordered
                                    ? AtomicOrdering::Monotonic
                                    : Ordering;
   AtomicRMWInst *AI = Builder.CreateAtomicRMW(
       AtomicRMWInst::Xchg, SI->getPointerOperand(), SI->getValueOperand(),
       SI->getAlign(), RMWOrdering);

 /// way as a typical atomicrmw expansion. The only difference here is
 /// that the operation inside of the loop may operate upon only a
 /// part of the value.
 void AtomicExpand::expandPartwordAtomicRMW(
     AtomicRMWInst *AI, TargetLoweringBase::AtomicExpansionKind ExpansionKind) {
   AtomicOrdering MemOpOrder = AI->getOrdering();
   SyncScope::ID SSID = AI->getSyncScopeID();
 
-  IRBuilder<> Builder(AI);
+  ReplacementIRBuilder Builder(AI);
 
   PartwordMaskValues PMV =
       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
 
   Value *ValOperand_Shifted =
       Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
                         PMV.ShiftAmt, "ValOperand_Shifted");
void AtomicExpand::expandPartwordAtomicRMW(
 
   Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
   AI->replaceAllUsesWith(FinalOldResult);
   AI->eraseFromParent();
 }
 
 // Widen the bitwise atomicrmw (or/xor/and) to the minimum supported width.
 AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
-  IRBuilder<> Builder(AI);
+  ReplacementIRBuilder Builder(AI);
   AtomicRMWInst::BinOp Op = AI->getOperation();
 
   assert((Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
           Op == AtomicRMWInst::And) &&
          "Unable to widen operation");
 
   PartwordMaskValues PMV =
       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
   //    %Res = insertvalue { i8, i1 } %25, i1 %Success, 1
 
   Value *Addr = CI->getPointerOperand();
   Value *Cmp = CI->getCompareOperand();
   Value *NewVal = CI->getNewValOperand();
 
   BasicBlock *BB = CI->getParent();
   Function *F = BB->getParent();
-  IRBuilder<> Builder(CI);
+  ReplacementIRBuilder Builder(CI);
   LLVMContext &Ctx = Builder.getContext();
 
   BasicBlock *EndBB =
       BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
   auto FailureBB =
       BasicBlock::Create(Ctx, "partword.cmpxchg.failure", F, EndBB);
   auto LoopBB = BasicBlock::Create(Ctx, "partword.cmpxchg.loop", F, FailureBB);
 
bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
   CI->eraseFromParent();
   return true;
 }
 
 void AtomicExpand::expandAtomicOpToLLSC(
     Instruction *I, Type *ResultType, Value *Addr, Align AddrAlign,
     AtomicOrdering MemOpOrder,
     function_ref<Value *(IRBuilder<> &, Value *)> PerformOp) {
-  IRBuilder<> Builder(I);
+  ReplacementIRBuilder Builder(I);
   Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,
                                     MemOpOrder, PerformOp);
 
   I->replaceAllUsesWith(Loaded);
   I->eraseFromParent();
 }
 
 void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
-  IRBuilder<> Builder(AI);
+  ReplacementIRBuilder Builder(AI);
 
   PartwordMaskValues PMV =
       createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
                        AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
 
   // The value operand must be sign-extended for signed min/max so that the
   // target's signed comparison instructions can be used. Otherwise, just
   // zero-ext.
Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(
       Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,
       AI->getOrdering());
   Value *FinalOldResult = extractMaskedValue(Builder, OldResult, PMV);
   AI->replaceAllUsesWith(FinalOldResult);
   AI->eraseFromParent();
 }
 
 void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {
-  IRBuilder<> Builder(CI);
+  ReplacementIRBuilder Builder(CI);
 
   PartwordMaskValues PMV = createMaskInstrs(
       Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
       CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
 
   Value *CmpVal_Shifted = Builder.CreateShl(
       Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
       "CmpVal_Shifted");

 /// way to represent a pointer cmpxchg so that we can update backends one by
 /// one.
 AtomicCmpXchgInst *
 AtomicExpand::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {
   auto *M = CI->getModule();
   Type *NewTy = getCorrespondingIntegerType(CI->getCompareOperand()->getType(),
                                             M->getDataLayout());
 
-  IRBuilder<> Builder(CI);
+  ReplacementIRBuilder Builder(CI);
 
   Value *Addr = CI->getPointerOperand();
   Type *PT = PointerType::get(NewTy, Addr->getType()->getPointerAddressSpace());
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);
 
   Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
   Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
 
bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
   auto ReleasedLoadBB =
       BasicBlock::Create(Ctx, "cmpxchg.releasedload", F, SuccessBB);
   auto TryStoreBB =
       BasicBlock::Create(Ctx, "cmpxchg.trystore", F, ReleasedLoadBB);
   auto ReleasingStoreBB =
       BasicBlock::Create(Ctx, "cmpxchg.fencedstore", F, TryStoreBB);
   auto StartBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, ReleasingStoreBB);
 
-  // This grabs the DebugLoc from CI
-  IRBuilder<> Builder(CI);
+  ReplacementIRBuilder Builder(CI);
 
   // The split call above "helpfully" added a branch at the end of BB (to the
   // wrong place), but we might want a fence too. It's easiest to just remove
   // the branch entirely.
   std::prev(BB->end())->eraseFromParent();
   Builder.SetInsertPoint(BB);
   if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
     TLI->emitLeadingFence(Builder, CI, SuccessOrder);
bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
   case TargetLoweringBase::AtomicExpansionKind::NotAtomic:
     return lowerAtomicCmpXchgInst(CI);
   }
 }
 
 // Note: This function is exposed externally by AtomicExpandUtils.h
 bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
                                     CreateCmpXchgInstFun CreateCmpXchg) {
-  IRBuilder<> Builder(AI);
+  ReplacementIRBuilder Builder(AI);
   Value *Loaded = AtomicExpand::insertRMWCmpXchgLoop(
       Builder, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
       AI->getOrdering(), AI->getSyncScopeID(),
       [&](IRBuilder<> &Builder, Value *Loaded) {
         return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,
                                    AI->getValOperand());
       },
       CreateCmpXchg);

llvm/test/Transforms/AtomicExpand/AArch64/pcsections.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=aarch64-linux-gnu -atomic-expand %s | FileCheck %s
+
+define i8 @atomic8_load_unordered(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_load_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, i8* [[A:%.*]] unordered, align 1, !pcsections !0
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %0 = load atomic i8, i8* %a unordered, align 1, !pcsections !0
+  ret i8 %0
+}
+
+define i8 @atomic8_load_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_load_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, i8* [[A:%.*]] monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %0 = load atomic i8, i8* %a monotonic, align 1, !pcsections !0
+  ret i8 %0
+}
+
+define i8 @atomic8_load_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_load_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, i8* [[A:%.*]] acquire, align 1, !pcsections !0
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %0 = load atomic i8, i8* %a acquire, align 1, !pcsections !0
+  ret i8 %0
+}
+
+define i8 @atomic8_load_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_load_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8, i8* [[A:%.*]] seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret i8 [[TMP0]]
+;
+entry:
+  %0 = load atomic i8, i8* %a seq_cst, align 1, !pcsections !0
+  ret i8 %0
+}
+
+define void @atomic8_store_unordered(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_store_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i8 0, i8* [[A:%.*]] unordered, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i8 0, i8* %a unordered, align 1, !pcsections !0
+  ret void
+}
+
+define void @atomic8_store_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_store_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i8 0, i8* [[A:%.*]] monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i8 0, i8* %a monotonic, align 1, !pcsections !0
+  ret void
+}
+
+define void @atomic8_store_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_store_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i8 0, i8* [[A:%.*]] release, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i8 0, i8* %a release, align 1, !pcsections !0
+  ret void
+}
+
+define void @atomic8_store_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_store_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i8 0, i8* [[A:%.*]] seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i8 0, i8* %a seq_cst, align 1, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xchg_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xchg_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_add_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_add_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_sub_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_sub_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_and_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_and_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_or_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_or_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xor_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xor_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_nand_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_nand_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i8* %a, i8 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xchg_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xchg_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_add_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_add_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_sub_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_sub_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_and_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_and_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_or_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_or_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xor_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xor_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_nand_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_nand_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i8* %a, i8 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xchg_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xchg_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_add_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_add_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_sub_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_sub_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_and_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_and_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_or_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_or_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xor_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xor_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_nand_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_nand_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i8* %a, i8 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xchg_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xchg_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_add_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_add_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_sub_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_sub_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_and_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_and_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_or_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_or_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xor_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xor_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_nand_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_nand_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i8* %a, i8 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xchg_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xchg_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 0 seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_add_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_add_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_sub_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_sub_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_and_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_and_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_or_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_or_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_xor_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_xor_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_nand_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_nand_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, i8* [[A:%.*]], align 1, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i8 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 [[LOADED]], i8 [[NEW]] seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i8* %a, i8 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_cas_monotonic(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_cas_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i8* [[A:%.*]], i8 0, i8 1 monotonic monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 monotonic acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 monotonic seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 monotonic acquire, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 monotonic seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_cas_acquire(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_cas_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i8* [[A:%.*]], i8 0, i8 1 acquire monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 acquire acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 acquire seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 acquire monotonic, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 acquire acquire, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 acquire seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_cas_release(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_cas_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i8* [[A:%.*]], i8 0, i8 1 release monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 release acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 release seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 release monotonic, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 release acquire, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 release seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_cas_acq_rel(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_cas_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i8* [[A:%.*]], i8 0, i8 1 acq_rel monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 acq_rel acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 acq_rel seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel monotonic, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic8_cas_seq_cst(i8* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic8_cas_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i8* [[A:%.*]], i8 0, i8 1 seq_cst monotonic, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 seq_cst acquire, align 1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i8* [[A]], i8 0, i8 1 seq_cst seq_cst, align 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst monotonic, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst acquire, !pcsections !0
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+define i16 @atomic16_load_unordered(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_load_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, i16* [[A:%.*]] unordered, align 2, !pcsections !0
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %0 = load atomic i16, i16* %a unordered, align 2, !pcsections !0
+  ret i16 %0
+}
+
+define i16 @atomic16_load_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_load_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, i16* [[A:%.*]] monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %0 = load atomic i16, i16* %a monotonic, align 2, !pcsections !0
+  ret i16 %0
+}
+
+define i16 @atomic16_load_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_load_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, i16* [[A:%.*]] acquire, align 2, !pcsections !0
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %0 = load atomic i16, i16* %a acquire, align 2, !pcsections !0
+  ret i16 %0
+}
+
+define i16 @atomic16_load_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_load_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i16, i16* [[A:%.*]] seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret i16 [[TMP0]]
+;
+entry:
+  %0 = load atomic i16, i16* %a seq_cst, align 2, !pcsections !0
+  ret i16 %0
+}
+
+define void @atomic16_store_unordered(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_store_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i16 0, i16* [[A:%.*]] unordered, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i16 0, i16* %a unordered, align 2, !pcsections !0
+  ret void
+}
+
+define void @atomic16_store_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_store_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i16 0, i16* [[A:%.*]] monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i16 0, i16* %a monotonic, align 2, !pcsections !0
+  ret void
+}
+
+define void @atomic16_store_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_store_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i16 0, i16* [[A:%.*]] release, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i16 0, i16* %a release, align 2, !pcsections !0
+  ret void
+}
+
+define void @atomic16_store_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_store_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i16 0, i16* [[A:%.*]] seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i16 0, i16* %a seq_cst, align 2, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xchg_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xchg_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_add_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_add_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_sub_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_sub_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_and_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_and_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_or_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_or_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xor_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xor_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_nand_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_nand_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i16* %a, i16 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xchg_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xchg_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_add_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_add_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_sub_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_sub_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_and_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_and_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_or_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_or_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xor_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xor_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_nand_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_nand_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i16* %a, i16 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xchg_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xchg_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_add_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_add_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_sub_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_sub_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_and_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_and_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_or_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_or_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xor_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xor_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_nand_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_nand_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i16* %a, i16 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xchg_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xchg_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_add_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_add_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_sub_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_sub_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_and_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_and_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_or_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_or_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xor_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xor_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_nand_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_nand_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i16* %a, i16 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xchg_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xchg_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 0 seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_add_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_add_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_sub_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_sub_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_and_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_and_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_or_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_or_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_xor_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_xor_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_nand_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_nand_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i16, i16* [[A:%.*]], align 2, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i16 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i16 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 [[LOADED]], i16 [[NEW]] seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i16* %a, i16 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_cas_monotonic(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_cas_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i16* [[A:%.*]], i16 0, i16 1 monotonic monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 monotonic acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 monotonic seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 monotonic acquire, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 monotonic seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_cas_acquire(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_cas_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i16* [[A:%.*]], i16 0, i16 1 acquire monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 acquire acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 acquire seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 acquire monotonic, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 acquire acquire, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 acquire seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_cas_release(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_cas_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i16* [[A:%.*]], i16 0, i16 1 release monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 release acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 release seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 release monotonic, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 release acquire, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 release seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_cas_acq_rel(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_cas_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i16* [[A:%.*]], i16 0, i16 1 acq_rel monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 acq_rel acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 acq_rel seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel monotonic, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic16_cas_seq_cst(i16* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic16_cas_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i16* [[A:%.*]], i16 0, i16 1 seq_cst monotonic, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 seq_cst acquire, align 2, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i16* [[A]], i16 0, i16 1 seq_cst seq_cst, align 2, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst monotonic, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst acquire, !pcsections !0
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+define i32 @atomic32_load_unordered(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_load_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, i32* [[A:%.*]] unordered, align 4, !pcsections !0
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %0 = load atomic i32, i32* %a unordered, align 4, !pcsections !0
+  ret i32 %0
+}
+
+define i32 @atomic32_load_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_load_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, i32* [[A:%.*]] monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %0 = load atomic i32, i32* %a monotonic, align 4, !pcsections !0
+  ret i32 %0
+}
+
+define i32 @atomic32_load_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_load_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, i32* [[A:%.*]] acquire, align 4, !pcsections !0
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %0 = load atomic i32, i32* %a acquire, align 4, !pcsections !0
+  ret i32 %0
+}
+
+define i32 @atomic32_load_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_load_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i32, i32* [[A:%.*]] seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret i32 [[TMP0]]
+;
+entry:
+  %0 = load atomic i32, i32* %a seq_cst, align 4, !pcsections !0
+  ret i32 %0
+}
+
+define void @atomic32_store_unordered(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_store_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i32 0, i32* [[A:%.*]] unordered, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i32 0, i32* %a unordered, align 4, !pcsections !0
+  ret void
+}
+
+define void @atomic32_store_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_store_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i32 0, i32* [[A:%.*]] monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i32 0, i32* %a monotonic, align 4, !pcsections !0
+  ret void
+}
+
+define void @atomic32_store_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_store_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i32 0, i32* [[A:%.*]] release, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i32 0, i32* %a release, align 4, !pcsections !0
+  ret void
+}
+
+define void @atomic32_store_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_store_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i32 0, i32* [[A:%.*]] seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i32 0, i32* %a seq_cst, align 4, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xchg_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xchg_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_add_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_add_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_sub_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_sub_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_and_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_and_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_or_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_or_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xor_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xor_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_nand_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_nand_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i32* %a, i32 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xchg_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xchg_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_add_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_add_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_sub_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_sub_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_and_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_and_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_or_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_or_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xor_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xor_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_nand_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_nand_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i32* %a, i32 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xchg_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xchg_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_add_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_add_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_sub_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_sub_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_and_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_and_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_or_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_or_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xor_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xor_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_nand_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_nand_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i32* %a, i32 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xchg_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xchg_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_add_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_add_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_sub_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_sub_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_and_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_and_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_or_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_or_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xor_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xor_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_nand_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_nand_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i32* %a, i32 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xchg_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xchg_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 0 seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_add_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_add_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_sub_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_sub_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_and_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_and_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_or_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_or_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_xor_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_xor_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_nand_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_nand_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[A:%.*]], align 4, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i32* %a, i32 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_cas_monotonic(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_cas_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32* [[A:%.*]], i32 0, i32 1 monotonic monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 monotonic acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 monotonic seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 monotonic acquire, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 monotonic seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_cas_acquire(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_cas_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32* [[A:%.*]], i32 0, i32 1 acquire monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 acquire acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 acquire seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 acquire monotonic, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 acquire acquire, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 acquire seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_cas_release(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_cas_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32* [[A:%.*]], i32 0, i32 1 release monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 release acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 release seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 release monotonic, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 release acquire, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 release seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_cas_acq_rel(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_cas_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32* [[A:%.*]], i32 0, i32 1 acq_rel monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 acq_rel acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 acq_rel seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel monotonic, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic32_cas_seq_cst(i32* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic32_cas_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32* [[A:%.*]], i32 0, i32 1 seq_cst monotonic, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 seq_cst acquire, align 4, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i32* [[A]], i32 0, i32 1 seq_cst seq_cst, align 4, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst monotonic, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst acquire, !pcsections !0
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+define i64 @atomic64_load_unordered(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_load_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, i64* [[A:%.*]] unordered, align 8, !pcsections !0
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %0 = load atomic i64, i64* %a unordered, align 8, !pcsections !0
+  ret i64 %0
+}
+
+define i64 @atomic64_load_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_load_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, i64* [[A:%.*]] monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %0 = load atomic i64, i64* %a monotonic, align 8, !pcsections !0
+  ret i64 %0
+}
+
+define i64 @atomic64_load_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_load_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, i64* [[A:%.*]] acquire, align 8, !pcsections !0
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %0 = load atomic i64, i64* %a acquire, align 8, !pcsections !0
+  ret i64 %0
+}
+
+define i64 @atomic64_load_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_load_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i64, i64* [[A:%.*]] seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret i64 [[TMP0]]
+;
+entry:
+  %0 = load atomic i64, i64* %a seq_cst, align 8, !pcsections !0
+  ret i64 %0
+}
+
+define i8* @atomic64_load_seq_cst_ptr_ty(i8** %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_load_seq_cst_ptr_ty(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load atomic i8*, i8** [[A:%.*]] seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret i8* [[TMP0]]
+;
+entry:
+  %0 = load atomic i8*, i8** %a seq_cst, align 8, !pcsections !0
+  ret i8* %0
+}
+
+define void @atomic64_store_unordered(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_store_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i64 0, i64* [[A:%.*]] unordered, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i64 0, i64* %a unordered, align 8, !pcsections !0
+  ret void
+}
+
+define void @atomic64_store_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_store_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i64 0, i64* [[A:%.*]] monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i64 0, i64* %a monotonic, align 8, !pcsections !0
+  ret void
+}
+
+define void @atomic64_store_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_store_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i64 0, i64* [[A:%.*]] release, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i64 0, i64* %a release, align 8, !pcsections !0
+  ret void
+}
+
+define void @atomic64_store_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_store_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i64 0, i64* [[A:%.*]] seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i64 0, i64* %a seq_cst, align 8, !pcsections !0
+  ret void
+}
+
+define void @atomic64_store_seq_cst_ptr_ty(i8** %a, i8* %v) nounwind uwtable {
+; CHECK-LABEL: @atomic64_store_seq_cst_ptr_ty(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store atomic i8* [[V:%.*]], i8** [[A:%.*]] seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i8* %v, i8** %a seq_cst, align 8, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xchg_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xchg_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_add_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_add_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_sub_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_sub_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_and_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_and_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_or_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_or_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xor_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xor_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_nand_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_nand_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i64* %a, i64 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xchg_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xchg_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_add_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_add_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_sub_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_sub_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_and_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_and_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_or_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_or_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xor_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xor_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_nand_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_nand_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i64* %a, i64 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xchg_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xchg_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_add_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_add_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_sub_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_sub_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_and_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_and_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_or_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_or_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xor_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xor_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_nand_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_nand_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i64* %a, i64 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xchg_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xchg_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_add_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_add_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_sub_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_sub_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_and_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_and_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_or_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_or_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xor_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xor_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_nand_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_nand_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i64* %a, i64 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xchg_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xchg_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 0 seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_add_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_add_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_sub_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_sub_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_and_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_and_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_or_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_or_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_xor_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_xor_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_nand_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_nand_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[A:%.*]], align 8, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i64 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i64* %a, i64 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_monotonic(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i64* [[A:%.*]], i64 0, i64 1 monotonic monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 monotonic acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 monotonic seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 monotonic acquire, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 monotonic seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_acquire(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i64* [[A:%.*]], i64 0, i64 1 acquire monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 acquire acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 acquire seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 acquire monotonic, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 acquire acquire, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 acquire seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_release(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i64* [[A:%.*]], i64 0, i64 1 release monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 release acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 release seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 release monotonic, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 release acquire, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 release seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_acq_rel(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i64* [[A:%.*]], i64 0, i64 1 acq_rel monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 acq_rel acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 acq_rel seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel monotonic, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_seq_cst(i64* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i64* [[A:%.*]], i64 0, i64 1 seq_cst monotonic, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 seq_cst acquire, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i64* [[A]], i64 0, i64 1 seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst monotonic, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst acquire, !pcsections !0
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic64_cas_seq_cst_ptr_ty(i8** %a, i8* %v1, i8* %v2) nounwind uwtable {
+; CHECK-LABEL: @atomic64_cas_seq_cst_ptr_ty(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8** [[A:%.*]] to i64*, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8* [[V1:%.*]] to i64, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint i8* [[V2:%.*]] to i64, !pcsections !0
+; CHECK-NEXT:    [[TMP3:%.*]] = cmpxchg i64* [[TMP0]], i64 [[TMP1]], i64 [[TMP2]] seq_cst seq_cst, align 8, !pcsections !0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { i64, i1 } [[TMP3]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1, !pcsections !0
+; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP4]] to i8*, !pcsections !0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8*, i1 } undef, i8* [[TMP6]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { i8*, i1 } [[TMP7]], i1 [[TMP5]], 1, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i8** %a, i8* %v1, i8* %v2 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+define i128 @atomic128_load_unordered(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_load_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 0 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP0]], 0, !pcsections !0
+; CHECK-NEXT:    ret i128 [[LOADED]]
+;
+entry:
+  %0 = load atomic i128, i128* %a unordered, align 16, !pcsections !0
+  ret i128 %0
+}
+
+define i128 @atomic128_load_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_load_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 0 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP0]], 0, !pcsections !0
+; CHECK-NEXT:    ret i128 [[LOADED]]
+;
+entry:
+  %0 = load atomic i128, i128* %a monotonic, align 16, !pcsections !0
+  ret i128 %0
+}
+
+define i128 @atomic128_load_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_load_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 0 acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP0]], 0, !pcsections !0
+; CHECK-NEXT:    ret i128 [[LOADED]]
+;
+entry:
+  %0 = load atomic i128, i128* %a acquire, align 16, !pcsections !0
+  ret i128 %0
+}
+
+define i128 @atomic128_load_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_load_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 0 seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[LOADED:%.*]] = extractvalue { i128, i1 } [[TMP0]], 0, !pcsections !0
+; CHECK-NEXT:    ret i128 [[LOADED]]
+;
+entry:
+  %0 = load atomic i128, i128* %a seq_cst, align 16, !pcsections !0
+  ret i128 %0
+}
+
+define void @atomic128_store_unordered(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_store_unordered(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i128 0, i128* %a unordered, align 16, !pcsections !0
+  ret void
+}
+
+define void @atomic128_store_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_store_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i128 0, i128* %a monotonic, align 16, !pcsections !0
+  ret void
+}
+
+define void @atomic128_store_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_store_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i128 0, i128* %a release, align 16, !pcsections !0
+  ret void
+}
+
+define void @atomic128_store_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_store_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  store atomic i128 0, i128* %a seq_cst, align 16, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xchg_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xchg_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_add_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_add_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_sub_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_sub_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_and_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_and_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_or_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_or_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xor_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xor_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_nand_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_nand_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i128* %a, i128 0 monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xchg_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xchg_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_add_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_add_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_sub_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_sub_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_and_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_and_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_or_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_or_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xor_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xor_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_nand_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_nand_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i128* %a, i128 0 acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xchg_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xchg_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_add_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_add_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_sub_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_sub_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_and_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_and_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_or_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_or_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xor_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xor_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_nand_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_nand_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i128* %a, i128 0 release, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xchg_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xchg_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_add_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_add_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_sub_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_sub_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_and_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_and_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_or_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_or_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xor_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xor_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_nand_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_nand_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i128* %a, i128 0 acq_rel, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xchg_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xchg_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 0 seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xchg i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_add_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_add_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = add i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw add i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_sub_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_sub_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = sub i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw sub i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_and_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_and_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw and i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_or_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_or_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = or i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw or i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_xor_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_xor_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP1]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP1]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw xor i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_nand_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_nand_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i128, i128* [[A:%.*]], align 16, !pcsections !0
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]], !pcsections !0
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i128 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ], !pcsections !0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[LOADED]], 0, !pcsections !0
+; CHECK-NEXT:    [[NEW:%.*]] = xor i128 [[TMP1]], -1, !pcsections !0
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg i128* [[A]], i128 [[LOADED]], i128 [[NEW]] seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i128, i1 } [[TMP2]], 1, !pcsections !0
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i128, i1 } [[TMP2]], 0, !pcsections !0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]], !pcsections !0
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  atomicrmw nand i128* %a, i128 0 seq_cst, !pcsections !0
+  ret void
+}
+
+define void @atomic128_cas_monotonic(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_cas_monotonic(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 1 monotonic monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 monotonic monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_cas_acquire(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_cas_acquire(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 1 acquire acquire, align 16, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 acquire acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_cas_release(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_cas_release(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 1 release monotonic, align 16, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 release monotonic, !pcsections !0
+  ret void
+}
+
+define void @atomic128_cas_acq_rel(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_cas_acq_rel(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 1 acq_rel acquire, align 16, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 acq_rel acquire, !pcsections !0
+  ret void
+}
+
+define void @atomic128_cas_seq_cst(i128* %a) nounwind uwtable {
+; CHECK-LABEL: @atomic128_cas_seq_cst(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i128* [[A:%.*]], i128 0, i128 1 seq_cst seq_cst, align 16, !pcsections !0
+; CHECK-NEXT:    ret void
+;
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 seq_cst seq_cst, !pcsections !0
+  ret void
+}
+
+!0 = !{!"foo"}