Index: clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- clang/lib/CodeGen/BackendUtil.cpp +++ clang/lib/CodeGen/BackendUtil.cpp @@ -653,6 +653,11 @@ InstrProfOptions Options; Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput; + + // TODO: Surface the option to emit atomic profile counter increments at + // the driver level. + Options.Atomic = LangOpts.Sanitize.has(SanitizerKind::Thread); + MPM.add(createInstrProfilingLegacyPass(Options)); } if (CodeGenOpts.hasProfileIRInstr()) { Index: clang/test/CodeGen/tsan-instrprof-atomic.c =================================================================== --- /dev/null +++ clang/test/CodeGen/tsan-instrprof-atomic.c @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 %s -emit-llvm -fprofile-instrument=clang -fsanitize=thread -o - | FileCheck %s + +// CHECK: define void @foo +// CHECK-NOT: load {{.*}}foo +// CHECK: ret void +void foo() {} Index: llvm/include/llvm/Transforms/Instrumentation.h =================================================================== --- llvm/include/llvm/Transforms/Instrumentation.h +++ llvm/include/llvm/Transforms/Instrumentation.h @@ -111,6 +111,9 @@ // Do counter register promotion bool DoCounterPromotion = false; + // Use atomic profile counter increments. + bool Atomic = false; + // Name of the profile file to use as output std::string InstrProfileOutput; Index: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp =================================================================== --- llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -96,6 +96,11 @@ // is usually smaller than 2. cl::init(1.0)); +cl::opt AtomicCounterUpdateAll( + "instrprof-atomic-counter-update-all", cl::ZeroOrMore, + cl::desc("Make all profile counter updates atomic (for testing only)"), + cl::init(false)); + cl::opt AtomicCounterUpdatePromoted( "atomic-counter-update-promoted", cl::ZeroOrMore, cl::desc("Do counter update using atomic fetch add " @@ -597,12 +602,17 @@ IRBuilder<> Builder(Inc); uint64_t Index = Inc->getIndex()->getZExtValue(); Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index); - Value *Load = Builder.CreateLoad(Addr, "pgocount"); - auto *Count = Builder.CreateAdd(Load, Inc->getStep()); - auto *Store = Builder.CreateStore(Count, Addr); - Inc->replaceAllUsesWith(Store); - if (isCounterPromotionEnabled()) - PromotionCandidates.emplace_back(cast(Load), Store); + + if (Options.Atomic || AtomicCounterUpdateAll) { + Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(), + AtomicOrdering::Monotonic); + } else { + Value *Load = Builder.CreateLoad(Addr, "pgocount"); + auto *Count = Builder.CreateAdd(Load, Inc->getStep()); + auto *Store = Builder.CreateStore(Count, Addr); + if (isCounterPromotionEnabled()) + PromotionCandidates.emplace_back(cast(Load), Store); + } Inc->eraseFromParent(); } Index: llvm/test/Instrumentation/InstrProfiling/atomic-updates.ll =================================================================== --- /dev/null +++ llvm/test/Instrumentation/InstrProfiling/atomic-updates.ll @@ -0,0 +1,14 @@ +; RUN: opt < %s -S -instrprof -instrprof-atomic-counter-update-all | FileCheck %s + +target triple = "x86_64-apple-macosx10.10.0" + +@__profn_foo = hidden constant [3 x i8] c"foo" + +; CHECK-LABEL: define void @foo +; CHECK-NEXT: atomicrmw add i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i64 0, i64 0), i64 1 monotonic +define void @foo() { + call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0) + ret void +} + +declare void @llvm.instrprof.increment(i8*, i64, i32, i32)