Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -25,9 +25,12 @@ // // This file contains two passes: // (1) Pass PGOInstrumentationGen which instruments the IR to generate edge -// count profile, and +// count profile, and generates the instrumentation for indirect call +// profiling. // (2) Pass PGOInstrumentationUse which reads the edge count profile and -// annotates the branch weights. +// annotates the branch weights. It also reads the indirect call value +// profiling records and annotate the indirect call instructions. +// // To get the precise counter information, These two passes need to invoke at // the same compilation point (so they see the same IR). For pass // PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For @@ -83,7 +86,7 @@ STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts."); STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile."); STATISTIC(NumOfPGOMissing, "Number of functions without profile."); -STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentation."); +STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations."); // Command line option to specify the file to read profile from. This is // mainly used for testing. @@ -94,7 +97,7 @@ "mainly for test purpose.")); // Command line options to disable value profiling. The default is false: -// i.e. vaule profiling is enabled by default. This is for debug purpose. +// i.e. value profiling is enabled by default. This is for debug purpose. static cl::opt DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, @@ -462,6 +465,9 @@ // compilation. uint64_t ProgramMaxCount; + // ProfileRecord for this function. + InstrProfRecord ProfileRecords; + // Find the Instrumented BB and set the value. void setInstrumentedCounts(const std::vector &CountFromProfile); @@ -501,6 +507,9 @@ // Set the branch weights based on the count values. void setBranchWeights(); + + // Annotate the indirect call sites. + void annotateIndirectCallSites(); }; // Visit all the edges and assign the count value for the instrumented @@ -559,9 +568,9 @@ // Return true if the profile are successfully read, and false on errors. bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) { auto &Ctx = M->getContext(); - ErrorOr Result = + ErrorOr Records = PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash); - if (std::error_code EC = Result.getError()) { + if (std::error_code EC = Records.getError()) { if (EC == instrprof_error::unknown_function) NumOfPGOMissing++; else if (EC == instrprof_error::hash_mismatch || @@ -573,7 +582,8 @@ DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning)); return false; } - std::vector &CountFromProfile = Result.get().Counts; + ProfileRecords = std::move(Records.get()); + std::vector &CountFromProfile = ProfileRecords.Counts; NumOfPGOFunc++; DEBUG(dbgs() << CountFromProfile.size() << " counts\n"); @@ -711,6 +721,27 @@ dbgs() << "\n";); } } + +// Traverse all the indirect callsites and annotate the instructions. +void PGOUseFunc::annotateIndirectCallSites() { + if (DisableValueProfiling) + return; + + unsigned IndirectCallSiteIndex = 0; + PGOIndirectCallSiteVisitor ICV; + ICV.visit(F); + unsigned NumTargets = + ProfileRecords.getNumValueSites(IPVK_IndirectCallTarget); + if (NumTargets == 0) + return; + for (auto &I : ICV.IndirectCallInsts) { + DEBUG(dbgs() << "Read one indirect call instrumentation: Index=" + << IndirectCallSiteIndex << " out of " << NumTargets << "\n"); + annotateValueSite(*M, *I, ProfileRecords, IPVK_IndirectCallTarget, + IndirectCallSiteIndex); + IndirectCallSiteIndex++; + } +} } // end anonymous namespace bool PGOInstrumentationGen::runOnModule(Module &M) { @@ -731,6 +762,7 @@ if (Func.readCounters(PGOReader)) { Func.populateCounters(); Func.setBranchWeights(); + Func.annotateIndirectCallSites(); } } Index: test/Transforms/PGOProfile/Inputs/indirect_call.proftext =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/Inputs/indirect_call.proftext @@ -0,0 +1,42 @@ +bar +# Func Hash: +12884901887 +# Num Counters: +1 +# Counter Values: +140 +# Num Value Kinds: +1 +# ValueKind = IPVK_IndirectCallTarget: +0 +# NumValueSites: +1 +3 +func2:80 +func1:40 +func3:20 + +func1 +# Func Hash: +12884901887 +# Num Counters: +1 +# Counter Values: +40 + +func2 +# Func Hash: +12884901887 +# Num Counters: +1 +# Counter Values: +80 + +func3 +# Func Hash: +12884901887 +# Num Counters: +1 +# Counter Values: +20 + Index: test/Transforms/PGOProfile/indirect_call_annotation.ll =================================================================== --- /dev/null +++ test/Transforms/PGOProfile/indirect_call_annotation.ll @@ -0,0 +1,35 @@ +; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata +; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@foo = common global i32 (i32)* null, align 8 + +define i32 @func1(i32 %x) { +entry: + ret i32 %x +} + +define i32 @func2(i32 %x) { +entry: + %add = add nsw i32 %x, 1 + ret i32 %add +} + +define i32 @func3(i32 %x) { +entry: + %add = add nsw i32 %x, 3 + ret i32 %add +} + +define i32 @bar(i32 %i) { +entry: + %tmp = load i32 (i32)*, i32 (i32)** @foo, align 8 + %call = call i32 %tmp(i32 %i) +; VP-ANNOTATION: %call = call i32 %tmp(i32 %i) +; VP-ANNOTATION-SAME: !prof ![[VP:[0-9]+]] +; VP-ANNOTATION: ![[VP]] = !{!"VP", i32 0, i64 140, i64 -4377547752858689819, i64 80, i64 -2545542355363006406, i64 40, i64 -6929281286627296573, i64 20} + ret i32 %call +} + +