diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -159,6 +159,10 @@ return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); } +inline StringRef getInstrProfCounterDynamicBiasFuncName() { + return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_DYNAMIC_BIAS_FUNC); +} + /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -665,6 +665,7 @@ #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_COUNTER_DYNAMIC_BIAS_FUNC __llvm_profile_counter_dynamic_bias /* The variable that holds the name of the profile data * specified via command line. */ diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -59,15 +59,29 @@ namespace { +constexpr StringRef kCountersBiasInstName = "prof_counters_bias"; + cl::opt DoHashBasedCounterSplit( "hash-based-counter-split", cl::desc("Rename counter variable of a comdat function based on cfg hash"), cl::init(true)); -cl::opt - RuntimeCounterRelocation("runtime-counter-relocation", - cl::desc("Enable relocating counters at runtime."), - cl::init(false)); +enum class RuntimeCounterRelocationTy { + kNone, + kVariable, + kFunction, +}; + +cl::opt RuntimeCounterRelocation( + "runtime-counter-relocation", + cl::desc("Enable relocating counters at runtime."), + cl::values(clEnumValN(RuntimeCounterRelocationTy::kNone, "none", + "Non-relocatable counters."), + clEnumValN(RuntimeCounterRelocationTy::kVariable, "variable", + "Counters relocation bias from a global variable."), + clEnumValN(RuntimeCounterRelocationTy::kFunction, "function", + "Counters relocation bias from a function call.")), + cl::init(RuntimeCounterRelocationTy::kNone)); cl::opt ValueProfileStaticAlloc( "vp-static-alloc", @@ -472,7 +486,7 @@ return false; if (RuntimeCounterRelocation.getNumOccurrences() > 0) - return RuntimeCounterRelocation; + return RuntimeCounterRelocation != RuntimeCounterRelocationTy::kNone; // Fuchsia uses runtime counter relocation by default. return TT.isOSFuchsia(); @@ -701,8 +715,18 @@ Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext()); Function *Fn = Inc->getParent()->getParent(); Instruction &I = Fn->getEntryBlock().front(); - LoadInst *LI = dyn_cast(&I); - if (!LI) { + Instruction *BiasInst = nullptr; + if (I.getName() == kCountersBiasInstName) { + // Reuse the first materialization of the bias done in this function. + BiasInst = &I; + } else if (RuntimeCounterRelocation == + RuntimeCounterRelocationTy::kFunction) { + IRBuilder<> Builder(&I); + BiasInst = Builder.CreateCall( + M->getOrInsertFunction(getInstrProfCounterDynamicBiasFuncName(), + Int64Ty), + None, kCountersBiasInstName); + } else { IRBuilder<> Builder(&I); GlobalVariable *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); @@ -721,9 +745,10 @@ if (TT.supportsCOMDAT()) Bias->setComdat(M->getOrInsertComdat(Bias->getName())); } - LI = Builder.CreateLoad(Int64Ty, Bias); + BiasInst = Builder.CreateLoad(Int64Ty, Bias, kCountersBiasInstName); } - auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI); + auto *Add = + Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasInst); Addr = Builder.CreateIntToPtr(Add, Int64PtrTy); } diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll --- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll +++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll @@ -1,26 +1,30 @@ ; RUN: opt < %s -S -instrprof | FileCheck %s -; RUN: opt < %s -S -instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s +; RUN: opt < %s -S -instrprof -runtime-counter-relocation=variable | FileCheck -check-prefixes=RELOC,RELOCVAR %s +; RUN: opt < %s -S -instrprof -runtime-counter-relocation=function | FileCheck -check-prefixes=RELOC,RELOCFUNC %s target triple = "x86_64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" -; RELOC: $__llvm_profile_counter_bias = comdat any -; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat +; RELOCVAR: $__llvm_profile_counter_bias = comdat any +; RELOCVAR: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat ; CHECK-LABEL: define void @foo ; CHECK-NEXT: %pgocount = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i32 0, i32 0) ; CHECK-NEXT: %1 = add i64 %pgocount, 1 ; CHECK-NEXT: store i64 %1, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i32 0, i32 0) ; RELOC-LABEL: define void @foo -; RELOC-NEXT: %1 = load i64, i64* @__llvm_profile_counter_bias -; RELOC-NEXT: %2 = add i64 ptrtoint ([1 x i64]* @__profc_foo to i64), %1 -; RELOC-NEXT: %3 = inttoptr i64 %2 to i64* -; RELOC-NEXT: %pgocount = load i64, i64* %3 -; RELOC-NEXT: %4 = add i64 %pgocount, 1 -; RELOC-NEXT: store i64 %4, i64* %3 +; RELOCVAR-NEXT: %prof_counters_bias = load i64, i64* @__llvm_profile_counter_bias +; RELOCFUNC-NEXT: %prof_counters_bias = call i64 @__llvm_profile_counter_dynamic_bias() +; RELOC-NEXT: %1 = add i64 ptrtoint ([1 x i64]* @__profc_foo to i64), %prof_counters_bias +; RELOC-NEXT: %2 = inttoptr i64 %1 to i64* +; RELOC-NEXT: %pgocount = load i64, i64* %2 +; RELOC-NEXT: %3 = add i64 %pgocount, 1 +; RELOC-NEXT: store i64 %3, i64* %2 define void @foo() { call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0) ret void } declare void @llvm.instrprof.increment(i8*, i64, i32, i32) + +; RELOCFUNC: declare i64 @__llvm_profile_counter_dynamic_bias()