[SanitizerBinaryMetadata] Introduce SanitizerBinaryMetadata instrumentation pass

[melver]

Summary

Introduces the SanitizerBinaryMetadata instrumentation pass which uses
the new MD_pcsections metadata kinds to instrument certain types of
instructions and functions required for breakpoint-based sanitizers.

The first intended user of the binary metadata emitted will be a variant
of GWP-TSan [1]. GWP-TSan will require information about atomic
accesses; to unambiguously determine if an access is atomic or not, we
also require "covered" information which code has been compiled with
SanitizerBinaryMetadata instrumentation enabled.

[1] https://llvm.org/devmtg/2020-09/slides/Morehouse-GWP-Tsan.pdf

Depends on D130879

[Enna1]

Hi, I'm interested about GWP-TSan. Is there any plan about open source GWP-TSan? :)

[melver]

In D130887#3690479, @Enna1 wrote:

Hi, I'm interested about GWP-TSan. Is there any plan about open source GWP-TSan? :)

Thanks for the interest - yes definitely, but we're only at the beginning and have no stable version yet. The compiler support is a strong dependency and pre-requisite for us to get to a stable version.

Also see https://reviews.llvm.org/D130888 and the RFC: https://discourse.llvm.org/t/rfc-pc-keyed-metadata-at-runtime/64191

[melver]

Add IR test.

[melver]

Comment wording.

[melver]

Rebase.

[melver]

• Change MDNode format to take aux constants as another MDNode.
• Change createPCSections interface.

[melver]

Rebase.

llvm/include/llvm/Transforms/Instrumentation.h

struct SanitizerCoverageOptions {
   bool NoPrune = false;
   bool StackDepth = false;
   bool TraceLoads = false;
   bool TraceStores = false;
 
   SanitizerCoverageOptions() = default;
 };
 
+/// Options for SanitizerBinaryMetadata.
+struct SanitizerBinaryMetadataOptions {
+  bool Covered = false;
+  bool Atomics = false;
+  SanitizerBinaryMetadataOptions() = default;
+};
+
 /// Calculate what to divide by to scale counts.
 ///
 /// Given the maximum count, calculate a divisor that will scale all the
 /// weights to strictly less than std::numeric_limits<uint32_t>::max().
 static inline uint64_t calculateCountScale(uint64_t MaxCount) {
   return MaxCount < std::numeric_limits<uint32_t>::max()
              ? 1
              : MaxCount / std::numeric_limits<uint32_t>::max() + 1;

llvm/include/llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h

+//===------- Definition of the SanitizerBinaryMetadata class ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the SanitizerBinaryMetadata pass.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERBINARYMETADATA_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_SANITIZERBINARYMETADATA_H
+
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Instrumentation.h"
+
+namespace llvm {
+
+/// Public interface to the SanitizerBinaryMetadata module pass for emitting
+/// metadata for binary analysis sanitizers.
+//
+/// The pass should be inserted after optimizations.
+class SanitizerBinaryMetadataPass
+    : public PassInfoMixin<SanitizerBinaryMetadataPass> {
+public:
+  explicit SanitizerBinaryMetadataPass(
+      SanitizerBinaryMetadataOptions Opts = {});
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+  static bool isRequired() { return true; }
+
+private:
+  const SanitizerBinaryMetadataOptions Options;
+};
+
+} // namespace llvm
+
+#endif

llvm/lib/Passes/PassBuilder.cpp

 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 #include "llvm/Transforms/Instrumentation/MemProfiler.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
+#include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h"
 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 #include "llvm/Transforms/ObjCARC.h"
 #include "llvm/Transforms/Scalar/ADCE.h"
 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
 #include "llvm/Transforms/Scalar/AnnotationRemarks.h"
 #include "llvm/Transforms/Scalar/BDCE.h"
 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"

llvm/lib/Passes/PassRegistry.def

 MODULE_PASS("trigger-crash", TriggerCrashPass())
 MODULE_PASS("verify", VerifierPass())
 MODULE_PASS("view-callgraph", CallGraphViewerPass())
 MODULE_PASS("wholeprogramdevirt", WholeProgramDevirtPass())
 MODULE_PASS("dfsan", DataFlowSanitizerPass())
 MODULE_PASS("module-inline", ModuleInlinerPass())
 MODULE_PASS("tsan-module", ModuleThreadSanitizerPass())
 MODULE_PASS("sancov-module", SanitizerCoveragePass())
+MODULE_PASS("sanmd-module", SanitizerBinaryMetadataPass())
 MODULE_PASS("memprof-module", ModuleMemProfilerPass())
 MODULE_PASS("poison-checking", PoisonCheckingPass())
 MODULE_PASS("pseudo-probe-update", PseudoProbeUpdatePass())
 #undef MODULE_PASS
 
 #ifndef MODULE_PASS_WITH_PARAMS
 #define MODULE_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif

llvm/lib/Transforms/Instrumentation/CMakeLists.txt

add_llvm_component_library(LLVMInstrumentation
   IndirectCallPromotion.cpp
   Instrumentation.cpp
   InstrOrderFile.cpp
   InstrProfiling.cpp
   PGOInstrumentation.cpp
   PGOMemOPSizeOpt.cpp
   PoisonChecking.cpp
   SanitizerCoverage.cpp
+  SanitizerBinaryMetadata.cpp
   ValueProfileCollector.cpp
   ThreadSanitizer.cpp
   HWAddressSanitizer.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms
 
   DEPENDS

llvm/lib/Transforms/Instrumentation/SanitizerBinaryMetadata.cpp

+//===- SanitizerBinaryMetadata.cpp - binary analysis sanitizers metadata --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of SanitizerBinaryMetadata.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+#include <array>
+#include <cstdint>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "sanmd"
+
+namespace {
+
+//===--- Constants --------------------------------------------------------===//
+
+constexpr uint32_t kVersionBase = 1;                // occupies lower 16 bits
+constexpr uint32_t kVersionPtrSizeRel = (1u << 16); // offsets are pointer-sized
+constexpr int kCtorDtorPriority = 2;
+
+// Pairs of names of initialization callback functions and which section
+// contains the relevant metadata.
+class MetadataInfo {
+public:
+  const StringRef FunctionPrefix;
+  const StringRef SectionSuffix;
+  const uint32_t FeatureMask;
+
+  static const MetadataInfo Covered;
+  static const MetadataInfo Atomics;
+
+private:
+  // Forbid construction elsewhere.
+  explicit constexpr MetadataInfo(StringRef FunctionPrefix,
+                                  StringRef SectionSuffix, int Feature)
+      : FunctionPrefix(FunctionPrefix), SectionSuffix(SectionSuffix),
+        FeatureMask(Feature != -1 ? (1u << Feature) : 0) {}
+};
+const MetadataInfo MetadataInfo::Covered{"__sanitizer_metadata_covered",
+                                         "sanmd_covered", -1};
+const MetadataInfo MetadataInfo::Atomics{"__sanitizer_metadata_atomics",
+                                         "sanmd_atomics", 0};
+
+// The only instances of MetadataInfo are the constants above, so a set of
+// them may simply store pointers to them. To deterministically generate code,
+// we need to use a set with stable iteration order, such as SetVector.
+using MetadataInfoSet = SetVector<const MetadataInfo *>;
+
+//===--- Command-line options ---------------------------------------------===//
+
+cl::opt<bool> ClEmitCovered("sanitizer-metadata-covered",
+                            cl::desc("Emit PCs for covered functions."),
+                            cl::Hidden, cl::init(false));
+cl::opt<bool> ClEmitAtomics("sanitizer-metadata-atomics",
+                            cl::desc("Emit PCs for atomic operations."),
+                            cl::Hidden, cl::init(false));
+
+//===--- Statistics -------------------------------------------------------===//
+
+STATISTIC(NumMetadataCovered, "Metadata attached to covered functions");
+STATISTIC(NumMetadataAtomics, "Metadata attached to atomics");
+
+//===----------------------------------------------------------------------===//
+
+// Apply opt overrides.
+SanitizerBinaryMetadataOptions &&
+transformOptionsFromCl(SanitizerBinaryMetadataOptions &&Opts) {
+  Opts.Covered |= ClEmitCovered;
+  Opts.Atomics |= ClEmitAtomics;
+  return std::move(Opts);
+}
+
+class SanitizerBinaryMetadata {
+public:
+  SanitizerBinaryMetadata(Module &M, SanitizerBinaryMetadataOptions Opts)
+      : Mod(M), Options(transformOptionsFromCl(std::move(Opts))),
+        TargetTriple(M.getTargetTriple()), IRB(M.getContext()) {
+    // FIXME: Make it work with other formats.
+    assert(TargetTriple.isOSBinFormatELF() && "ELF only");
+  }
+
+  bool run();
+
+private:
+  // Return enabled feature mask of per-instruction metadata.
+  uint32_t getEnabledPerInstructionFeature() const {
+    uint32_t FeatureMask = 0;
+    if (Options.Atomics)
+      FeatureMask |= MetadataInfo::Atomics.FeatureMask;
+    return FeatureMask;
+  }
+
+  uint32_t getVersion() const {
+    uint32_t Version = kVersionBase;
+    const auto CM = Mod.getCodeModel();
+    if (CM.has_value() && (*CM == CodeModel::Medium || *CM == CodeModel::Large))
+      Version |= kVersionPtrSizeRel;
+    return Version;
+  }
+
+  void runOn(Function &F, MetadataInfoSet &MIS);
+
+  // Determines which set of metadata to collect for this instruction.
+  //
+  // Returns true if covered metadata is required to unambiguously interpret
+  // other metadata. For example, if we are interested in atomics metadata, any
+  // function with memory operations (atomic or not) requires covered metadata
+  // to determine if a memory operation is atomic or not in modules compiled
+  // with SanitizerBinaryMetadata.
+  bool runOn(Instruction &I, MetadataInfoSet &MIS, MDBuilder &MDB);
+
+  // Get start/end section marker pointer.
+  GlobalVariable *getSectionMarker(const Twine &MarkerName, Type *Ty);
+
+  // Create a 0-sized object in a section, so that the section is not discarded
+  // if all inputs have been discarded.
+  void createZeroSizedObjectInSection(Type *Ty, StringRef SectionSuffix);
+
+  // Returns the target-dependent section name.
+  StringRef getSectionName(StringRef SectionSuffix);
+
+  // Returns the section start marker name.
+  Twine getSectionStart(StringRef SectionSuffix);
+
+  // Returns the section end marker name.
+  Twine getSectionEnd(StringRef SectionSuffix);
+
+  Module &Mod;
+  const SanitizerBinaryMetadataOptions Options;
+  const Triple TargetTriple;
+  IRBuilder<> IRB;
+};
+
+bool SanitizerBinaryMetadata::run() {
+  MetadataInfoSet MIS;
+
+  for (Function &F : Mod)
+    runOn(F, MIS);
+
+  if (MIS.empty())
+    return false;
+
+  //
+  // Setup constructors and call all initialization functions for requested
+  // metadata features.
+  //
+
+  auto *Int8PtrTy = IRB.getInt8PtrTy();
+  auto *Int8PtrPtrTy = PointerType::getUnqual(Int8PtrTy);
+  auto *Int32Ty = IRB.getInt32Ty();
+  const std::array<Type *, 3> InitTypes = {Int32Ty, Int8PtrPtrTy, Int8PtrPtrTy};
+  auto *Version = ConstantInt::get(Int32Ty, getVersion());
+
+  for (const MetadataInfo *MI : MIS) {
+    const std::array<Value *, InitTypes.size()> InitArgs = {
+        Version,
+        getSectionMarker(getSectionStart(MI->SectionSuffix), Int8PtrTy),
+        getSectionMarker(getSectionEnd(MI->SectionSuffix), Int8PtrTy),
+    };
+    Function *Ctor =
+        createSanitizerCtorAndInitFunctions(
+            Mod, (MI->FunctionPrefix + ".module_ctor").str(),
+            (MI->FunctionPrefix + "_add").str(), InitTypes, InitArgs)
+            .first;
+    Function *Dtor =
+        createSanitizerCtorAndInitFunctions(
+            Mod, (MI->FunctionPrefix + ".module_dtor").str(),
+            (MI->FunctionPrefix + "_del").str(), InitTypes, InitArgs)
+            .first;
+    Constant *CtorData = nullptr;
+    Constant *DtorData = nullptr;
+    if (TargetTriple.supportsCOMDAT()) {
+      // Use COMDAT to deduplicate constructor/destructor function.
+      Ctor->setComdat(Mod.getOrInsertComdat(Ctor->getName()));
+      Dtor->setComdat(Mod.getOrInsertComdat(Dtor->getName()));
+      CtorData = Ctor;
+      DtorData = Dtor;
+    }
+    appendToGlobalCtors(Mod, Ctor, kCtorDtorPriority, CtorData);
+    appendToGlobalDtors(Mod, Dtor, kCtorDtorPriority, DtorData);
+    createZeroSizedObjectInSection(Int8PtrTy, MI->SectionSuffix);
+  }
+
+  return true;
+}
+
+void SanitizerBinaryMetadata::runOn(Function &F, MetadataInfoSet &MIS) {
+  if (F.empty())
+    return;
+  if (F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
+    return;
+  // Don't touch available_externally functions, their actual body is elsewhere.
+  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
+    return;
+
+  MDBuilder MDB(F.getContext());
+
+  // The metadata features enabled for this function, stored along covered
+  // metadata (if enabled).
+  uint32_t PerInstrFeatureMask = getEnabledPerInstructionFeature();
+  // Don't emit unnecessary covered metadata for all functions to save space.
+  bool RequiresCovered = false;
+  if (PerInstrFeatureMask) {
+    for (BasicBlock &BB : F)
+      for (Instruction &I : BB)
+        RequiresCovered |= runOn(I, MIS, MDB);
+  }
+
+  // Covered metadata is always emitted if explicitly requested, otherwise only
+  // if some other metadata requires it to unambiguously interpret it for
+  // modules compiled with SanitizerBinaryMetadata.
+  if (Options.Covered || RequiresCovered) {
+    NumMetadataCovered++;
+    const auto *MI = &MetadataInfo::Covered;
+    MIS.insert(MI);
+    const StringRef Section = getSectionName(MI->SectionSuffix);
+    // The feature mask will be placed after the size (32 bit) of the function,
+    // so in total one covered entry will use `sizeof(void*) + 4 + 4`.
+    Constant *CFM = IRB.getInt32(PerInstrFeatureMask);
+    F.setMetadata(LLVMContext::MD_pcsections,
+                  MDB.createPCSections({{Section, {CFM}}}));
+  }
+}
+
+bool SanitizerBinaryMetadata::runOn(Instruction &I, MetadataInfoSet &MIS,
+                                    MDBuilder &MDB) {
+  SmallVector<const MetadataInfo *, 1> InstMetadata;
+  bool RequiresCovered = false;
+
+  if (Options.Atomics && I.mayReadOrWriteMemory()) {
+    auto SSID = getAtomicSyncScopeID(&I);
+    if (SSID.has_value() && SSID.value() != SyncScope::SingleThread) {
+      NumMetadataAtomics++;
+      InstMetadata.push_back(&MetadataInfo::Atomics);
+    }
+    RequiresCovered = true;
+  }
+
+  // Attach MD_pcsections to instruction.
+  if (!InstMetadata.empty()) {
+    MIS.insert(InstMetadata.begin(), InstMetadata.end());
+    SmallVector<MDBuilder::PCSection, 1> Sections;
+    for (const auto &MI : InstMetadata)
+      Sections.push_back({getSectionName(MI->SectionSuffix), {}});
+    I.setMetadata(LLVMContext::MD_pcsections, MDB.createPCSections(Sections));
+  }
+
+  return RequiresCovered;
+}
+
+GlobalVariable *
+SanitizerBinaryMetadata::getSectionMarker(const Twine &MarkerName, Type *Ty) {
+  auto *Marker = new GlobalVariable(Mod, Ty, /*isConstant=*/false,
+                                    GlobalVariable::ExternalLinkage,
+                                    /*Initializer=*/nullptr, MarkerName);
+  Marker->setVisibility(GlobalValue::HiddenVisibility);
+  return Marker;
+}
+
+void SanitizerBinaryMetadata::createZeroSizedObjectInSection(
+    Type *Ty, StringRef SectionSuffix) {
+  auto *DummyInit = ConstantAggregateZero::get(ArrayType::get(Ty, 0));
+  auto *DummyEntry = new GlobalVariable(Mod, DummyInit->getType(), true,
+                                        GlobalVariable::ExternalLinkage,
+                                        DummyInit, "__dummy_" + SectionSuffix);
+  DummyEntry->setSection(getSectionName(SectionSuffix));
+  DummyEntry->setVisibility(GlobalValue::HiddenVisibility);
+  if (TargetTriple.supportsCOMDAT())
+    DummyEntry->setComdat(Mod.getOrInsertComdat(DummyEntry->getName()));
+  // Make sure the section isn't discarded by gc-sections.
+  appendToUsed(Mod, DummyEntry);
+}
+
+StringRef SanitizerBinaryMetadata::getSectionName(StringRef SectionSuffix) {
+  // FIXME: Other TargetTriple (req. string pool)
+  return SectionSuffix;
+}
+
+Twine SanitizerBinaryMetadata::getSectionStart(StringRef SectionSuffix) {
+  return "__start_" + SectionSuffix;
+}
+
+Twine SanitizerBinaryMetadata::getSectionEnd(StringRef SectionSuffix) {
+  return "__stop_" + SectionSuffix;
+}
+
+} // namespace
+
+SanitizerBinaryMetadataPass::SanitizerBinaryMetadataPass(
+    SanitizerBinaryMetadataOptions Opts)
+    : Options(std::move(Opts)) {}
+
+PreservedAnalyses
+SanitizerBinaryMetadataPass::run(Module &M, AnalysisManager<Module> &AM) {
+  SanitizerBinaryMetadata Pass(M, Options);
+  if (Pass.run())
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}

llvm/test/Instrumentation/SanitizerBinaryMetadata/atomics.ll

+; RUN: opt < %s -passes='module(sanmd-module)' -sanitizer-metadata-atomics -S | FileCheck %s
+
+; Check that atomic memory operations receive PC sections metadata.
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+define i8 @atomic8_load_unordered(i8* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i8, i8* %a unordered, align 1
+  ret i8 %0
+}
+; CHECK-LABEL: atomic8_load_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i8 @atomic8_load_monotonic(i8* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i8, i8* %a monotonic, align 1
+  ret i8 %0
+}
+; CHECK-LABEL: atomic8_load_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i8 @atomic8_load_acquire(i8* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i8, i8* %a acquire, align 1
+  ret i8 %0
+}
+; CHECK-LABEL: atomic8_load_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i8 @atomic8_load_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i8, i8* %a seq_cst, align 1
+  ret i8 %0
+}
+; CHECK-LABEL: atomic8_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_store_unordered(i8* %a) nounwind uwtable {
+entry:
+  store atomic i8 0, i8* %a unordered, align 1
+  ret void
+}
+; CHECK-LABEL: atomic8_store_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_store_monotonic(i8* %a) nounwind uwtable {
+entry:
+  store atomic i8 0, i8* %a monotonic, align 1
+  ret void
+}
+; CHECK-LABEL: atomic8_store_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_store_release(i8* %a) nounwind uwtable {
+entry:
+  store atomic i8 0, i8* %a release, align 1
+  ret void
+}
+; CHECK-LABEL: atomic8_store_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_store_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  store atomic i8 0, i8* %a seq_cst, align 1
+  ret void
+}
+; CHECK-LABEL: atomic8_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xchg_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_xchg_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_add_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw add i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_add_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_sub_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_sub_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_and_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw and i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_and_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_or_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw or i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_or_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xor_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_xor_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_nand_monotonic(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i8* %a, i8 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic8_nand_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xchg_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_xchg_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_add_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw add i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_add_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_sub_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_sub_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_and_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw and i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_and_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_or_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw or i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_or_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xor_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_xor_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_nand_acquire(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i8* %a, i8 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic8_nand_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xchg_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_xchg_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_add_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw add i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_add_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_sub_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_sub_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_and_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw and i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_and_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_or_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw or i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_or_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xor_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_xor_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_nand_release(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i8* %a, i8 0 release
+  ret void
+}
+; CHECK-LABEL: atomic8_nand_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xchg_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_xchg_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_add_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw add i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_add_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_sub_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_sub_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_and_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw and i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_and_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_or_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw or i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_or_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xor_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_xor_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_nand_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i8* %a, i8 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic8_nand_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xchg_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_xchg_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_add_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw add i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_add_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_sub_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_sub_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_and_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw and i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_and_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_or_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw or i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_or_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_xor_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_xor_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_nand_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i8* %a, i8 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_nand_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic8_cas_monotonic(i8* %a) nounwind uwtable {
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic
+  cmpxchg i8* %a, i8 0, i8 1 monotonic acquire
+  cmpxchg i8* %a, i8 0, i8 1 monotonic seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_cas_monotonic{{.*}}!pcsections !0
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 monotonic monotonic, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 monotonic acquire, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 monotonic seq_cst, align 1, !pcsections !2
+
+define void @atomic8_cas_acquire(i8* %a) nounwind uwtable {
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 acquire monotonic
+  cmpxchg i8* %a, i8 0, i8 1 acquire acquire
+  cmpxchg i8* %a, i8 0, i8 1 acquire seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_cas_acquire{{.*}}!pcsections !0
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acquire monotonic, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acquire acquire, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acquire seq_cst, align 1, !pcsections !2
+
+define void @atomic8_cas_release(i8* %a) nounwind uwtable {
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 release monotonic
+  cmpxchg i8* %a, i8 0, i8 1 release acquire
+  cmpxchg i8* %a, i8 0, i8 1 release seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_cas_release{{.*}}!pcsections !0
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 release monotonic, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 release acquire, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 release seq_cst, align 1, !pcsections !2
+
+define void @atomic8_cas_acq_rel(i8* %a) nounwind uwtable {
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel monotonic
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire
+  cmpxchg i8* %a, i8 0, i8 1 acq_rel seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_cas_acq_rel{{.*}}!pcsections !0
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acq_rel monotonic, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acq_rel acquire, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 acq_rel seq_cst, align 1, !pcsections !2
+
+define void @atomic8_cas_seq_cst(i8* %a) nounwind uwtable {
+entry:
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst monotonic
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst acquire
+  cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic8_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 seq_cst monotonic, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 seq_cst acquire, align 1, !pcsections !2
+; CHECK: cmpxchg i8* %a, i8 0, i8 1 seq_cst seq_cst, align 1, !pcsections !2
+
+define i16 @atomic16_load_unordered(i16* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i16, i16* %a unordered, align 2
+  ret i16 %0
+}
+; CHECK-LABEL: atomic16_load_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i16 @atomic16_load_monotonic(i16* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i16, i16* %a monotonic, align 2
+  ret i16 %0
+}
+; CHECK-LABEL: atomic16_load_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i16 @atomic16_load_acquire(i16* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i16, i16* %a acquire, align 2
+  ret i16 %0
+}
+; CHECK-LABEL: atomic16_load_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i16 @atomic16_load_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i16, i16* %a seq_cst, align 2
+  ret i16 %0
+}
+; CHECK-LABEL: atomic16_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_store_unordered(i16* %a) nounwind uwtable {
+entry:
+  store atomic i16 0, i16* %a unordered, align 2
+  ret void
+}
+; CHECK-LABEL: atomic16_store_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_store_monotonic(i16* %a) nounwind uwtable {
+entry:
+  store atomic i16 0, i16* %a monotonic, align 2
+  ret void
+}
+; CHECK-LABEL: atomic16_store_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_store_release(i16* %a) nounwind uwtable {
+entry:
+  store atomic i16 0, i16* %a release, align 2
+  ret void
+}
+; CHECK-LABEL: atomic16_store_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_store_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  store atomic i16 0, i16* %a seq_cst, align 2
+  ret void
+}
+; CHECK-LABEL: atomic16_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xchg_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_xchg_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_add_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw add i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_add_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_sub_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_sub_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_and_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw and i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_and_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_or_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw or i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_or_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xor_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_xor_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_nand_monotonic(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i16* %a, i16 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic16_nand_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xchg_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_xchg_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_add_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw add i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_add_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_sub_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_sub_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_and_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw and i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_and_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_or_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw or i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_or_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xor_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_xor_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_nand_acquire(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i16* %a, i16 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic16_nand_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xchg_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_xchg_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_add_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw add i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_add_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_sub_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_sub_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_and_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw and i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_and_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_or_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw or i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_or_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xor_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_xor_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_nand_release(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i16* %a, i16 0 release
+  ret void
+}
+; CHECK-LABEL: atomic16_nand_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xchg_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_xchg_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_add_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw add i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_add_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_sub_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_sub_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_and_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw and i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_and_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_or_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw or i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_or_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xor_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_xor_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_nand_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i16* %a, i16 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic16_nand_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xchg_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_xchg_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_add_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw add i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_add_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_sub_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_sub_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_and_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw and i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_and_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_or_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw or i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_or_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_xor_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_xor_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_nand_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i16* %a, i16 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_nand_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic16_cas_monotonic(i16* %a) nounwind uwtable {
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic
+  cmpxchg i16* %a, i16 0, i16 1 monotonic acquire
+  cmpxchg i16* %a, i16 0, i16 1 monotonic seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_cas_monotonic{{.*}}!pcsections !0
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 monotonic monotonic, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 monotonic acquire, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 monotonic seq_cst, align 2, !pcsections !2
+
+define void @atomic16_cas_acquire(i16* %a) nounwind uwtable {
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 acquire monotonic
+  cmpxchg i16* %a, i16 0, i16 1 acquire acquire
+  cmpxchg i16* %a, i16 0, i16 1 acquire seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_cas_acquire{{.*}}!pcsections !0
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acquire monotonic, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acquire acquire, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acquire seq_cst, align 2, !pcsections !2
+
+define void @atomic16_cas_release(i16* %a) nounwind uwtable {
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 release monotonic
+  cmpxchg i16* %a, i16 0, i16 1 release acquire
+  cmpxchg i16* %a, i16 0, i16 1 release seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_cas_release{{.*}}!pcsections !0
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 release monotonic, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 release acquire, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 release seq_cst, align 2, !pcsections !2
+
+define void @atomic16_cas_acq_rel(i16* %a) nounwind uwtable {
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel monotonic
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire
+  cmpxchg i16* %a, i16 0, i16 1 acq_rel seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_cas_acq_rel{{.*}}!pcsections !0
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acq_rel monotonic, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acq_rel acquire, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 acq_rel seq_cst, align 2, !pcsections !2
+
+define void @atomic16_cas_seq_cst(i16* %a) nounwind uwtable {
+entry:
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst monotonic
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst acquire
+  cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic16_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 seq_cst monotonic, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 seq_cst acquire, align 2, !pcsections !2
+; CHECK: cmpxchg i16* %a, i16 0, i16 1 seq_cst seq_cst, align 2, !pcsections !2
+
+define i32 @atomic32_load_unordered(i32* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i32, i32* %a unordered, align 4
+  ret i32 %0
+}
+; CHECK-LABEL: atomic32_load_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i32 @atomic32_load_monotonic(i32* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i32, i32* %a monotonic, align 4
+  ret i32 %0
+}
+; CHECK-LABEL: atomic32_load_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i32 @atomic32_load_acquire(i32* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i32, i32* %a acquire, align 4
+  ret i32 %0
+}
+; CHECK-LABEL: atomic32_load_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i32 @atomic32_load_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i32, i32* %a seq_cst, align 4
+  ret i32 %0
+}
+; CHECK-LABEL: atomic32_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_store_unordered(i32* %a) nounwind uwtable {
+entry:
+  store atomic i32 0, i32* %a unordered, align 4
+  ret void
+}
+; CHECK-LABEL: atomic32_store_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_store_monotonic(i32* %a) nounwind uwtable {
+entry:
+  store atomic i32 0, i32* %a monotonic, align 4
+  ret void
+}
+; CHECK-LABEL: atomic32_store_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_store_release(i32* %a) nounwind uwtable {
+entry:
+  store atomic i32 0, i32* %a release, align 4
+  ret void
+}
+; CHECK-LABEL: atomic32_store_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_store_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  store atomic i32 0, i32* %a seq_cst, align 4
+  ret void
+}
+; CHECK-LABEL: atomic32_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xchg_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_xchg_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_add_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw add i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_add_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_sub_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_sub_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_and_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw and i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_and_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_or_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw or i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_or_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xor_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_xor_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_nand_monotonic(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i32* %a, i32 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic32_nand_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xchg_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_xchg_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_add_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw add i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_add_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_sub_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_sub_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_and_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw and i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_and_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_or_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw or i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_or_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xor_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_xor_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_nand_acquire(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i32* %a, i32 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic32_nand_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xchg_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_xchg_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_add_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw add i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_add_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_sub_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_sub_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_and_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw and i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_and_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_or_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw or i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_or_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xor_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_xor_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_nand_release(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i32* %a, i32 0 release
+  ret void
+}
+; CHECK-LABEL: atomic32_nand_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xchg_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_xchg_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_add_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw add i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_add_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_sub_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_sub_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_and_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw and i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_and_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_or_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw or i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_or_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xor_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_xor_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_nand_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i32* %a, i32 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic32_nand_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xchg_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_xchg_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_add_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw add i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_add_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_sub_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_sub_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_and_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw and i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_and_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_or_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw or i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_or_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_xor_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_xor_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_nand_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i32* %a, i32 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_nand_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic32_cas_monotonic(i32* %a) nounwind uwtable {
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic
+  cmpxchg i32* %a, i32 0, i32 1 monotonic acquire
+  cmpxchg i32* %a, i32 0, i32 1 monotonic seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_cas_monotonic{{.*}}!pcsections !0
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 monotonic monotonic, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 monotonic acquire, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 monotonic seq_cst, align 4, !pcsections !2
+
+define void @atomic32_cas_acquire(i32* %a) nounwind uwtable {
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 acquire monotonic
+  cmpxchg i32* %a, i32 0, i32 1 acquire acquire
+  cmpxchg i32* %a, i32 0, i32 1 acquire seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_cas_acquire{{.*}}!pcsections !0
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acquire monotonic, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acquire acquire, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acquire seq_cst, align 4, !pcsections !2
+
+define void @atomic32_cas_release(i32* %a) nounwind uwtable {
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 release monotonic
+  cmpxchg i32* %a, i32 0, i32 1 release acquire
+  cmpxchg i32* %a, i32 0, i32 1 release seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_cas_release{{.*}}!pcsections !0
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 release monotonic, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 release acquire, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 release seq_cst, align 4, !pcsections !2
+
+define void @atomic32_cas_acq_rel(i32* %a) nounwind uwtable {
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel monotonic
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire
+  cmpxchg i32* %a, i32 0, i32 1 acq_rel seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_cas_acq_rel{{.*}}!pcsections !0
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acq_rel monotonic, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acq_rel acquire, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 acq_rel seq_cst, align 4, !pcsections !2
+
+define void @atomic32_cas_seq_cst(i32* %a) nounwind uwtable {
+entry:
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst monotonic
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst acquire
+  cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic32_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 seq_cst monotonic, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 seq_cst acquire, align 4, !pcsections !2
+; CHECK: cmpxchg i32* %a, i32 0, i32 1 seq_cst seq_cst, align 4, !pcsections !2
+
+define i64 @atomic64_load_unordered(i64* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i64, i64* %a unordered, align 8
+  ret i64 %0
+}
+; CHECK-LABEL: atomic64_load_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i64 @atomic64_load_monotonic(i64* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i64, i64* %a monotonic, align 8
+  ret i64 %0
+}
+; CHECK-LABEL: atomic64_load_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i64 @atomic64_load_acquire(i64* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i64, i64* %a acquire, align 8
+  ret i64 %0
+}
+; CHECK-LABEL: atomic64_load_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i64 @atomic64_load_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i64, i64* %a seq_cst, align 8
+  ret i64 %0
+}
+; CHECK-LABEL: atomic64_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i8* @atomic64_load_seq_cst_ptr_ty(i8** %a) nounwind uwtable {
+entry:
+  %0 = load atomic i8*, i8** %a seq_cst, align 8
+  ret i8* %0
+}
+; CHECK-LABEL: atomic64_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_store_unordered(i64* %a) nounwind uwtable {
+entry:
+  store atomic i64 0, i64* %a unordered, align 8
+  ret void
+}
+; CHECK-LABEL: atomic64_store_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_store_monotonic(i64* %a) nounwind uwtable {
+entry:
+  store atomic i64 0, i64* %a monotonic, align 8
+  ret void
+}
+; CHECK-LABEL: atomic64_store_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_store_release(i64* %a) nounwind uwtable {
+entry:
+  store atomic i64 0, i64* %a release, align 8
+  ret void
+}
+; CHECK-LABEL: atomic64_store_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_store_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  store atomic i64 0, i64* %a seq_cst, align 8
+  ret void
+}
+; CHECK-LABEL: atomic64_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_store_seq_cst_ptr_ty(i8** %a, i8* %v) nounwind uwtable {
+entry:
+  store atomic i8* %v, i8** %a seq_cst, align 8
+  ret void
+}
+; CHECK-LABEL: atomic64_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xchg_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_xchg_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_add_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw add i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_add_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_sub_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_sub_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_and_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw and i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_and_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_or_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw or i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_or_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xor_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_xor_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_nand_monotonic(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i64* %a, i64 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic64_nand_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xchg_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_xchg_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_add_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw add i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_add_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_sub_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_sub_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_and_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw and i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_and_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_or_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw or i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_or_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xor_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_xor_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_nand_acquire(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i64* %a, i64 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic64_nand_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xchg_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_xchg_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_add_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw add i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_add_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_sub_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_sub_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_and_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw and i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_and_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_or_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw or i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_or_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xor_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_xor_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_nand_release(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i64* %a, i64 0 release
+  ret void
+}
+; CHECK-LABEL: atomic64_nand_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xchg_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_xchg_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_add_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw add i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_add_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_sub_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_sub_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_and_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw and i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_and_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_or_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw or i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_or_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xor_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_xor_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_nand_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i64* %a, i64 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic64_nand_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xchg_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_xchg_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_add_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw add i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_add_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_sub_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_sub_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_and_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw and i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_and_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_or_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw or i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_or_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_xor_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_xor_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_nand_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i64* %a, i64 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_nand_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic64_cas_monotonic(i64* %a) nounwind uwtable {
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic
+  cmpxchg i64* %a, i64 0, i64 1 monotonic acquire
+  cmpxchg i64* %a, i64 0, i64 1 monotonic seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_monotonic{{.*}}!pcsections !0
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 monotonic monotonic, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 monotonic acquire, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 monotonic seq_cst, align 8, !pcsections !2
+
+define void @atomic64_cas_acquire(i64* %a) nounwind uwtable {
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 acquire monotonic
+  cmpxchg i64* %a, i64 0, i64 1 acquire acquire
+  cmpxchg i64* %a, i64 0, i64 1 acquire seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_acquire{{.*}}!pcsections !0
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acquire monotonic, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acquire acquire, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acquire seq_cst, align 8, !pcsections !2
+
+define void @atomic64_cas_release(i64* %a) nounwind uwtable {
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 release monotonic
+  cmpxchg i64* %a, i64 0, i64 1 release acquire
+  cmpxchg i64* %a, i64 0, i64 1 release seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_release{{.*}}!pcsections !0
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 release monotonic, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 release acquire, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 release seq_cst, align 8, !pcsections !2
+
+define void @atomic64_cas_acq_rel(i64* %a) nounwind uwtable {
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel monotonic
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire
+  cmpxchg i64* %a, i64 0, i64 1 acq_rel seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_acq_rel{{.*}}!pcsections !0
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acq_rel monotonic, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acq_rel acquire, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 acq_rel seq_cst, align 8, !pcsections !2
+
+define void @atomic64_cas_seq_cst(i64* %a) nounwind uwtable {
+entry:
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst monotonic
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst acquire
+  cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 seq_cst monotonic, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 seq_cst acquire, align 8, !pcsections !2
+; CHECK: cmpxchg i64* %a, i64 0, i64 1 seq_cst seq_cst, align 8, !pcsections !2
+
+define void @atomic64_cas_seq_cst_ptr_ty(i8** %a, i8* %v1, i8* %v2) nounwind uwtable {
+entry:
+  cmpxchg i8** %a, i8* %v1, i8* %v2 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic64_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i128 @atomic128_load_unordered(i128* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i128, i128* %a unordered, align 16
+  ret i128 %0
+}
+; CHECK-LABEL: atomic128_load_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i128 @atomic128_load_monotonic(i128* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i128, i128* %a monotonic, align 16
+  ret i128 %0
+}
+; CHECK-LABEL: atomic128_load_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i128 @atomic128_load_acquire(i128* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i128, i128* %a acquire, align 16
+  ret i128 %0
+}
+; CHECK-LABEL: atomic128_load_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define i128 @atomic128_load_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  %0 = load atomic i128, i128* %a seq_cst, align 16
+  ret i128 %0
+}
+; CHECK-LABEL: atomic128_load_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_store_unordered(i128* %a) nounwind uwtable {
+entry:
+  store atomic i128 0, i128* %a unordered, align 16
+  ret void
+}
+; CHECK-LABEL: atomic128_store_unordered{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_store_monotonic(i128* %a) nounwind uwtable {
+entry:
+  store atomic i128 0, i128* %a monotonic, align 16
+  ret void
+}
+; CHECK-LABEL: atomic128_store_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_store_release(i128* %a) nounwind uwtable {
+entry:
+  store atomic i128 0, i128* %a release, align 16
+  ret void
+}
+; CHECK-LABEL: atomic128_store_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_store_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  store atomic i128 0, i128* %a seq_cst, align 16
+  ret void
+}
+; CHECK-LABEL: atomic128_store_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xchg_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_xchg_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_add_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw add i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_add_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_sub_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_sub_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_and_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw and i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_and_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_or_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw or i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_or_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xor_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_xor_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_nand_monotonic(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i128* %a, i128 0 monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_nand_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xchg_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_xchg_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_add_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw add i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_add_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_sub_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_sub_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_and_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw and i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_and_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_or_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw or i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_or_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xor_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_xor_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_nand_acquire(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i128* %a, i128 0 acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_nand_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xchg_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_xchg_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_add_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw add i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_add_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_sub_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_sub_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_and_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw and i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_and_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_or_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw or i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_or_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xor_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_xor_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_nand_release(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i128* %a, i128 0 release
+  ret void
+}
+; CHECK-LABEL: atomic128_nand_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xchg_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_xchg_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_add_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw add i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_add_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_sub_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_sub_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_and_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw and i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_and_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_or_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw or i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_or_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xor_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_xor_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_nand_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i128* %a, i128 0 acq_rel
+  ret void
+}
+; CHECK-LABEL: atomic128_nand_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xchg_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xchg i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_xchg_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_add_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw add i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_add_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_sub_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw sub i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_sub_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_and_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw and i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_and_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_or_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw or i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_or_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_xor_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw xor i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_xor_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_nand_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  atomicrmw nand i128* %a, i128 0 seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_nand_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_cas_monotonic(i128* %a) nounwind uwtable {
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 monotonic monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_cas_monotonic{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_cas_acquire(i128* %a) nounwind uwtable {
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 acquire acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_cas_acquire{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_cas_release(i128* %a) nounwind uwtable {
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 release monotonic
+  ret void
+}
+; CHECK-LABEL: atomic128_cas_release{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_cas_acq_rel(i128* %a) nounwind uwtable {
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 acq_rel acquire
+  ret void
+}
+; CHECK-LABEL: atomic128_cas_acq_rel{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+define void @atomic128_cas_seq_cst(i128* %a) nounwind uwtable {
+entry:
+  cmpxchg i128* %a, i128 0, i128 1 seq_cst seq_cst
+  ret void
+}
+; CHECK-LABEL: atomic128_cas_seq_cst{{.*}}!pcsections !0
+; CHECK: !pcsections !2
+
+; Check that callbacks are emitted.
+
+; CHECK-LABEL: __sanitizer_metadata_atomics.module_ctor
+; CHECK: call void @__sanitizer_metadata_atomics_add(i32 1, i8** @__start_sanmd_atomics, i8** @__stop_sanmd_atomics)
+
+; CHECK-LABEL: __sanitizer_metadata_atomics.module_dtor
+; CHECK: call void @__sanitizer_metadata_atomics_del(i32 1, i8** @__start_sanmd_atomics, i8** @__stop_sanmd_atomics)
+
+; CHECK-LABEL: __sanitizer_metadata_covered.module_ctor
+; CHECK: call void @__sanitizer_metadata_covered_add(i32 1, i8** @__start_sanmd_covered, i8** @__stop_sanmd_covered)
+
+; CHECK-LABEL: __sanitizer_metadata_covered.module_dtor
+; CHECK: call void @__sanitizer_metadata_covered_del(i32 1, i8** @__start_sanmd_covered, i8** @__stop_sanmd_covered)
+
+; CHECK: !0 = !{!"sanmd_covered", !1}
+; CHECK: !1 = !{i32 1}
+; CHECK: !2 = !{!"sanmd_atomics"}