diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -665,6 +665,7 @@ #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_COUNTER_DYNAMIC_BIAS_FUNC __llvm_profile_get_counter_bias /* The variable that holds the name of the profile data * specified via command line. */ diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -159,6 +159,10 @@ return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR); } +inline StringRef getInstrProfCounterDynamicBiasFuncName() { + return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_DYNAMIC_BIAS_FUNC); +} + /// Return the marker used to separate PGO names during serialization. inline StringRef getInstrProfNameSeparator() { return "\01"; } diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -665,6 +665,7 @@ #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_COUNTER_DYNAMIC_BIAS_FUNC __llvm_profile_get_counter_bias /* The variable that holds the name of the profile data * specified via command line. */ diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -57,6 +57,9 @@ } }; DenseMap ProfileDataMap; + // If runtime relocation is enabled, this maps functions to the instruction + // that produces the profile relocation bias. + DenseMap FunctionToProfileBiasMap; std::vector CompilerUsedVars; std::vector UsedVars; std::vector ReferencedNames; diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -64,10 +64,22 @@ cl::desc("Rename counter variable of a comdat function based on cfg hash"), cl::init(true)); -cl::opt - RuntimeCounterRelocation("runtime-counter-relocation", - cl::desc("Enable relocating counters at runtime."), - cl::init(false)); +enum class RuntimeCounterRelocationTy { + kNone, + kVariable, + kFunction, +}; + +cl::opt RuntimeCounterRelocation( + "runtime-counter-relocation", + cl::desc("Enable relocating counters at runtime."), + cl::values(clEnumValN(RuntimeCounterRelocationTy::kNone, "none", + "Non-relocatable counters."), + clEnumValN(RuntimeCounterRelocationTy::kVariable, "variable", + "Counters relocation bias from a global variable."), + clEnumValN(RuntimeCounterRelocationTy::kFunction, "function", + "Counters relocation bias from a function call.")), + cl::init(RuntimeCounterRelocationTy::kNone)); cl::opt ValueProfileStaticAlloc( "vp-static-alloc", @@ -472,7 +484,7 @@ return false; if (RuntimeCounterRelocation.getNumOccurrences() > 0) - return RuntimeCounterRelocation; + return RuntimeCounterRelocation != RuntimeCounterRelocationTy::kNone; // Fuchsia uses runtime counter relocation by default. return TT.isOSFuchsia(); @@ -700,30 +712,38 @@ Type *Int64Ty = Type::getInt64Ty(M->getContext()); Type *Int64PtrTy = Type::getInt64PtrTy(M->getContext()); Function *Fn = Inc->getParent()->getParent(); - Instruction &I = Fn->getEntryBlock().front(); - LoadInst *LI = dyn_cast(&I); - if (!LI) { - IRBuilder<> Builder(&I); - GlobalVariable *Bias = - M->getGlobalVariable(getInstrProfCounterBiasVarName()); - if (!Bias) { - // Compiler must define this variable when runtime counter relocation - // is being used. Runtime has a weak external reference that is used - // to check whether that's the case or not. - Bias = new GlobalVariable( - *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, - Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName()); - Bias->setVisibility(GlobalVariable::HiddenVisibility); - // A definition that's weak (linkonce_odr) without being in a COMDAT - // section wouldn't lead to link errors, but it would lead to a dead - // data word from every TU but one. Putting it in COMDAT ensures there - // will be exactly one data slot in the link. - if (TT.supportsCOMDAT()) - Bias->setComdat(M->getOrInsertComdat(Bias->getName())); + Instruction *&BiasInst = FunctionToProfileBiasMap[Fn]; + if (!BiasInst) { + IRBuilder<> Builder(&Fn->getEntryBlock().front()); + if (RuntimeCounterRelocation == RuntimeCounterRelocationTy::kFunction) { + BiasInst = Builder.CreateCall( + M->getOrInsertFunction(getInstrProfCounterDynamicBiasFuncName(), + Int64Ty), + None, "prof_counters_bias"); + } else { + GlobalVariable *Bias = + M->getGlobalVariable(getInstrProfCounterBiasVarName()); + if (!Bias) { + // Compiler must define this variable when runtime counter relocation + // is being used. Runtime has a weak external reference that is used + // to check whether that's the case or not. + Bias = new GlobalVariable(*M, Int64Ty, false, + GlobalValue::LinkOnceODRLinkage, + Constant::getNullValue(Int64Ty), + getInstrProfCounterBiasVarName()); + Bias->setVisibility(GlobalVariable::HiddenVisibility); + // A definition that's weak (linkonce_odr) without being in a COMDAT + // section wouldn't lead to link errors, but it would lead to a dead + // data word from every TU but one. Putting it in COMDAT ensures + // there will be exactly one data slot in the link. + if (TT.supportsCOMDAT()) + Bias->setComdat(M->getOrInsertComdat(Bias->getName())); + } + BiasInst = Builder.CreateLoad(Int64Ty, Bias, "prof_counters_bias"); } - LI = Builder.CreateLoad(Int64Ty, Bias); } - auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), LI); + auto *Add = + Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasInst); Addr = Builder.CreateIntToPtr(Add, Int64PtrTy); } diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll --- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll +++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll @@ -1,26 +1,30 @@ ; RUN: opt < %s -S -instrprof | FileCheck %s -; RUN: opt < %s -S -instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s +; RUN: opt < %s -S -instrprof -runtime-counter-relocation=variable | FileCheck -check-prefixes=RELOC,RELOCVAR %s +; RUN: opt < %s -S -instrprof -runtime-counter-relocation=function | FileCheck -check-prefixes=RELOC,RELOCFUNC %s target triple = "x86_64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" -; RELOC: $__llvm_profile_counter_bias = comdat any -; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat +; RELOCVAR: $__llvm_profile_counter_bias = comdat any +; RELOCVAR: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat ; CHECK-LABEL: define void @foo ; CHECK-NEXT: %pgocount = load i64, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i32 0, i32 0) ; CHECK-NEXT: %1 = add i64 %pgocount, 1 ; CHECK-NEXT: store i64 %1, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc_foo, i32 0, i32 0) ; RELOC-LABEL: define void @foo -; RELOC-NEXT: %1 = load i64, i64* @__llvm_profile_counter_bias -; RELOC-NEXT: %2 = add i64 ptrtoint ([1 x i64]* @__profc_foo to i64), %1 -; RELOC-NEXT: %3 = inttoptr i64 %2 to i64* -; RELOC-NEXT: %pgocount = load i64, i64* %3 -; RELOC-NEXT: %4 = add i64 %pgocount, 1 -; RELOC-NEXT: store i64 %4, i64* %3 +; RELOCVAR-NEXT: %prof_counters_bias = load i64, i64* @__llvm_profile_counter_bias +; RELOCFUNC-NEXT: %prof_counters_bias = call i64 @__llvm_profile_counter_dynamic_bias() +; RELOC-NEXT: %1 = add i64 ptrtoint ([1 x i64]* @__profc_foo to i64), %prof_counters_bias +; RELOC-NEXT: %2 = inttoptr i64 %1 to i64* +; RELOC-NEXT: %pgocount = load i64, i64* %2 +; RELOC-NEXT: %3 = add i64 %pgocount, 1 +; RELOC-NEXT: store i64 %3, i64* %2 define void @foo() { call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0) ret void } declare void @llvm.instrprof.increment(i8*, i64, i32, i32) + +; RELOCFUNC: declare i64 @__llvm_profile_counter_dynamic_bias()