[PGO] Indirect-call profile annotation in IR level

[xur]

Summary

This patch read the indirect call value records in the profile and make the annotation in the indirectcall instructions. This is for IR level instrumentation.

Thanks,

-Rong

[davidxl]

lib/Transforms/Instrumentation/PGOInstrumentation.cpp

469

nit: ProfileRecords --> ProfileRecord

571

Seems a unnecessary change.

[xur]

Integrated David's comments

[xur]

ping

[betulb]

Looks fine to me except a naming nit.

lib/Transforms/Instrumentation/PGOInstrumentation.cpp

733

NumTargets -> NumValueSites.

[davidxl]

lib/Transforms/Instrumentation/PGOInstrumentation.cpp

734

Better check ICV's number of icall instructions matches the number of NumValueSites. If not give warning and return.

[xur]

Integrated with Betul and David's review comments.

Thanks,

-Rong

[davidxl]

lgtm

lib/Transforms/Instrumentation/PGOInstrumentation.cpp

 //
 // The minimal spanning tree here is actually a maximum weight tree -- on-tree
 // edges have higher frequencies (more likely to execute). The idea is to
 // instrument those less frequently executed edges to reduce the runtime
 // overhead of instrumented binaries.
 //
 // This file contains two passes:
 // (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
-// count profile, and
+// count profile, and generates the instrumentation for indirect call
+// profiling.
 // (2) Pass PGOInstrumentationUse which reads the edge count profile and
-// annotates the branch weights.
+// annotates the branch weights. It also reads the indirect call value
+// profiling records and annotate the indirect call instructions.
+//
 // To get the precise counter information, These two passes need to invoke at
 // the same compilation point (so they see the same IR). For pass
 // PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For
 // pass PGOInstrumentationUse, the real work in done in class PGOUseFunc and
 // the profile is opened in module level and passed to each PGOUseFunc instance.
 // The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put
 // in class FuncPGOInstrumentation.
 //

 
 STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");
 STATISTIC(NumOfPGOEdge, "Number of edges.");
 STATISTIC(NumOfPGOBB, "Number of basic-blocks.");
 STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");
 STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");
 STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
 STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
-STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentation.");
+STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");
 
 // Command line option to specify the file to read profile from. This is
 // mainly used for testing.
 static cl::opt<std::string>
     PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden,
                        cl::value_desc("filename"),
                        cl::desc("Specify the path of profile data file. This is"
                                 "mainly for test purpose."));
 
 // Command line options to disable value profiling. The default is false:
-// i.e. vaule profiling is enabled by default. This is for debug purpose.
+// i.e. value profiling is enabled by default. This is for debug purpose.
 static cl::opt<bool>
 DisableValueProfiling("disable-vp", cl::init(false),
                                     cl::Hidden,
                                     cl::desc("Disable Value Profiling"));
 
 namespace {
 class PGOInstrumentationGen : public ModulePass {
 public:
private:
   UseBBInfo &getBBInfo(const BasicBlock *BB) const {
     return FuncInfo.getBBInfo(BB);
   }
 
   // The maximum count value in the profile. This is only used in PGO use
   // compilation.
   uint64_t ProgramMaxCount;
 
+  // ProfileRecord for this function.
+  InstrProfRecord ProfileRecord;
+
   // Find the Instrumented BB and set the value.
   void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
 
   // Set the edge counter value for the unknown edge -- there should be only
   // one unknown edge.
   void setEdgeCount(DirectEdges &Edges, uint64_t Value);
 
   // Return FuncName string;
public:
   // Read counts for the instrumented BB from profile.
   bool readCounters(IndexedInstrProfReader *PGOReader);
 
   // Populate the counts for all BBs.
   void populateCounters();
 
   // Set the branch weights based on the count values.
   void setBranchWeights();
+
+  // Annotate the indirect call sites.
+  void annotateIndirectCallSites();
 };
 
 // Visit all the edges and assign the count value for the instrumented
 // edges and the BB.
 void PGOUseFunc::setInstrumentedCounts(
     const std::vector<uint64_t> &CountFromProfile) {
 
   // Use a worklist as we will update the vector during the iteration.
void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
   llvm_unreachable("Cannot find the unknown count edge");
 }
 
 // Read the profile from ProfileFileName and assign the value to the
 // instrumented BB and the edges. This function also updates ProgramMaxCount.
 // Return true if the profile are successfully read, and false on errors.
 bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
   auto &Ctx = M->getContext();
   ErrorOr<InstrProfRecord> Result =
       PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
   if (std::error_code EC = Result.getError()) {
     if (EC == instrprof_error::unknown_function)
       NumOfPGOMissing++;
     else if (EC == instrprof_error::hash_mismatch ||
              EC == llvm::instrprof_error::malformed)
       NumOfPGOMismatch++;
 
     std::string Msg = EC.message() + std::string(" ") + F.getName().str();
     Ctx.diagnose(
         DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
     return false;
   }
-  std::vector<uint64_t> &CountFromProfile = Result.get().Counts;
+  ProfileRecord = std::move(Result.get());
+  std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
 
   NumOfPGOFunc++;
   DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
   uint64_t ValueSum = 0;
   for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
     DEBUG(dbgs() << "  " << I << ": " << CountFromProfile[I] << "\n");
     ValueSum += CountFromProfile[I];
   }
for (auto &BB : F) {
 
     TI->setMetadata(llvm::LLVMContext::MD_prof,
                     MDB.createBranchWeights(Weights));
     DEBUG(dbgs() << "Weight is: ";
           for (const auto &W : Weights) { dbgs() << W << " "; }
           dbgs() << "\n";);
   }
 }
+
+// Traverse all the indirect callsites and annotate the instructions.
+void PGOUseFunc::annotateIndirectCallSites() {
+  if (DisableValueProfiling)
+    return;
+
+  unsigned IndirectCallSiteIndex = 0;
+  PGOIndirectCallSiteVisitor ICV;
+  ICV.visit(F);
+  unsigned NumTargets =
+      ProfileRecord.getNumValueSites(IPVK_IndirectCallTarget);
+  if (NumTargets == 0)
+    return;
+  for (auto &I : ICV.IndirectCallInsts) {
+    DEBUG(dbgs() << "Read one indirect call instrumentation: Index="
+                 << IndirectCallSiteIndex << " out of " << NumTargets << "\n");
+    annotateValueSite(*M, *I, ProfileRecord, IPVK_IndirectCallTarget,
+                      IndirectCallSiteIndex);
+    IndirectCallSiteIndex++;
+  }
+}
 } // end anonymous namespace
 
 bool PGOInstrumentationGen::runOnModule(Module &M) {
   for (auto &F : M) {
     if (F.isDeclaration())
       continue;
     BranchProbabilityInfo *BPI =
         &(getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI());
     BlockFrequencyInfo *BFI =
         &(getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI());
     instrumentOneFunc(F, &M, BPI, BFI);
   }
   return true;
 }
 
 static void setPGOCountOnFunc(PGOUseFunc &Func,
                               IndexedInstrProfReader *PGOReader) {
   if (Func.readCounters(PGOReader)) {
     Func.populateCounters();
     Func.setBranchWeights();
+    Func.annotateIndirectCallSites();
   }
 }
 
 bool PGOInstrumentationUse::runOnModule(Module &M) {
   DEBUG(dbgs() << "Read in profile counters: ");
   auto &Ctx = M.getContext();
   // Read the counter array from file.
   auto ReaderOrErr = IndexedInstrProfReader::create(ProfileFileName);

test/Transforms/PGOProfile/Inputs/indirect_call.proftext

+bar
+# Func Hash:
[phone]
+# Num Counters:
+1
+# Counter Values:
+140
+# Num Value Kinds:
+1
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+1
+3
+func2:80
+func1:40
+func3:20
+
+func1
+# Func Hash:
[phone]
+# Num Counters:
+1
+# Counter Values:
+40
+
+func2
+# Func Hash:
[phone]
+# Num Counters:
+1
+# Counter Values:
+80
+
+func3
+# Func Hash:
[phone]
+# Num Counters:
+1
+# Counter Values:
+20
+

test/Transforms/PGOProfile/indirect_call_annotation.ll

+; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 (i32)* null, align 8
+
+define i32 @func1(i32 %x) {
+entry:
+  ret i32 %x
+}
+
+define i32 @func2(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 1
+  ret i32 %add
+}
+
+define i32 @func3(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 3
+  ret i32 %add
+}
+
+define i32 @bar(i32 %i) {
+entry:
+  %tmp = load i32 (i32)*, i32 (i32)** @foo, align 8
+  %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION: %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION-SAME: !prof ![[VP:[0-9]+]]
+; VP-ANNOTATION: ![[VP]] = !{!"VP", i32 0, i64 140, i64 -4377547752858689819, i64 80, i64 -2545542355363006406, i64 40, i64 -6929281286627296573, i64 20}
+  ret i32 %call
+}
+
+