Fix update VP metadata after inlining for instrumentation PGO

[tejohnson]

Summary

With instrumentation profiling, when updating the VP metadata after
an inline, VP metadata on the inlined copy was inadvertantly having
all counts zeroed out. This was causing indirect calls from code inlined
during the call step to be marked as cold in the ThinLTO summaries and
not imported.

The CallerBFI needs to be passed down so that the CallSiteCount can be
computed from the profile summary info. With Sample PGO this was working
since the count is extracted from the branch weight metadata on the
call being inlined (even before we stopped looking at metadata for
non-sample PGO in r302844 this largely wasn't working for instrumentation
PGO since only promoted indirect calls would be getting inlined and have
the metadata).

Added an instrumentation PGO test and renamed the sample PGO test.

[eraman]

LGTM

lib/Transforms/Utils/InlineFunction.cpp

CallerBFI->setBlockFreqAndScale(
       EntryClone, CallerBFI->getBlockFreq(CallSiteBlock).getFrequency(),
       ClonedBBs);
 }
 
 /// Update the branch metadata for cloned call instructions.
 static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap,
                               const Optional<uint64_t> &CalleeEntryCount,
                               const Instruction *TheCall,
-                              ProfileSummaryInfo *PSI) {
+                              ProfileSummaryInfo *PSI,
+                              BlockFrequencyInfo *CallerBFI) {
   if (!CalleeEntryCount.hasValue() || CalleeEntryCount.getValue() < 1)
     return;
   Optional<uint64_t> CallSiteCount =
-      PSI ? PSI->getProfileCount(TheCall, nullptr) : None;
+      PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None;
   uint64_t CallCount =
       std::min(CallSiteCount.hasValue() ? CallSiteCount.getValue() : 0,
                CalleeEntryCount.getValue());
 
   for (auto const &Entry : VMap)
     if (isa<CallInst>(Entry.first))
       if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
         CI->updateProfWeight(CallCount, CalleeEntryCount.getValue());
Function::iterator FirstNewBlock;
     FirstNewBlock = LastBlock; ++FirstNewBlock;
 
     if (IFI.CallerBFI != nullptr && IFI.CalleeBFI != nullptr)
       // Update the BFI of blocks cloned into the caller.
       updateCallerBFI(OrigBB, VMap, IFI.CallerBFI, IFI.CalleeBFI,
                       CalledFunc->front());
 
     updateCallProfile(CalledFunc, VMap, CalledFunc->getEntryCount(), TheCall,
-                      IFI.PSI);
+                      IFI.PSI, IFI.CallerBFI);
     // Update the profile count of callee.
     updateCalleeCount(IFI.CallerBFI, OrigBB, TheCall, CalledFunc, IFI.PSI);
 
     // Inject byval arguments initialization.
     for (std::pair<Value*, Value*> &Init : ByValInit)
       HandleByValArgumentInit(Init.first, Init.second, Caller->getParent(),
                               &*FirstNewBlock, IFI);
 

test/Transforms/Inline/prof-update-instr.ll

-; RUN: opt < %s -inline -S | FileCheck %s
-; Checks if inliner updates branch_weights annotation for call instructions.
+; RUN: opt < %s -passes='require<profile-summary>,cgscc(inline)' -S | FileCheck %s
+; Checks if inliner updates VP metadata for indrect call instructions
+; with instrumentation based profile.
 
-declare void @ext();
-declare void @ext1();
 @func = global void ()* null
+@func2 = global void ()* null
 
 ; CHECK: define void @callee(i32 %n) !prof ![[ENTRY_COUNT:[0-9]*]]
 define void  @callee(i32 %n) !prof !15 {
   %cond = icmp sle i32 %n, 10
-  br i1 %cond, label %cond_true, label %cond_false
+  br i1 %cond, label %cond_true, label %cond_false, !prof !20
 cond_true:
-; ext1 is optimized away, thus not updated.
-; CHECK: call void @ext1(), !prof ![[COUNT_CALLEE1:[0-9]*]]
-  call void @ext1(), !prof !16
+; f2 is optimized away, thus not updated.
+  %f2 = load void ()*, void ()** @func2
+; CHECK: call void %f2(), !prof ![[COUNT_IND_CALLEE1:[0-9]*]]
+  call void %f2(), !prof !19
   ret void
 cond_false:
-; ext is cloned and updated.
-; CHECK: call void @ext(), !prof ![[COUNT_CALLEE:[0-9]*]]
-  call void @ext(), !prof !16
   %f = load void ()*, void ()** @func
-; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]] 
+; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]]
   call void %f(), !prof !18
   ret void
 }
 
 ; CHECK: define void @caller()
-define void @caller() {
-; CHECK: call void @ext(), !prof ![[COUNT_CALLER:[0-9]*]]
+define void @caller() !prof !21 {
 ; CHECK: call void %f.i(), !prof ![[COUNT_IND_CALLER:[0-9]*]]
-  call void @callee(i32 15), !prof !17
+  call void @callee(i32 15)
   ret void
 }
 
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
-!3 = !{!"ProfileFormat", !"SampleProfile"}
+!3 = !{!"ProfileFormat", !"InstrProf"}
 !4 = !{!"TotalCount", i64 10000}
 !5 = !{!"MaxCount", i64 10}
 !6 = !{!"MaxInternalCount", i64 1}
 !7 = !{!"MaxFunctionCount", i64 2000}
 !8 = !{!"NumCounts", i64 2}
 !9 = !{!"NumFunctions", i64 2}
 !10 = !{!"DetailedSummary", !11}
 !11 = !{!12, !13, !14}
 !12 = !{i32 10000, i64 100, i32 1}
 !13 = !{i32 999000, i64 100, i32 1}
 !14 = !{i32 999999, i64 1, i32 2}
 !15 = !{!"function_entry_count", i64 1000}
 !16 = !{!"branch_weights", i64 2000}
-!17 = !{!"branch_weights", i64 400}
 !18 = !{!"VP", i32 0, i64 140, i64 111, i64 80, i64 222, i64 40, i64 333, i64 20}
+!19 = !{!"VP", i32 0, i64 200, i64 111, i64 100, i64 222, i64 60, i64 333, i64 40}
+!20 = !{!"branch_weights", i32 1000, i32 1000}
+!21 = !{!"function_entry_count", i64 400}
 attributes #0 = { alwaysinline }
 ; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 600}
-; CHECK: ![[COUNT_CALLEE1]] = !{!"branch_weights", i64 2000}
-; CHECK: ![[COUNT_CALLEE]] = !{!"branch_weights", i64 1200}
+; CHECK: ![[COUNT_IND_CALLEE1]] = !{!"VP", i32 0, i64 200, i64 111, i64 100, i64 222, i64 60, i64 333, i64 40}
 ; CHECK: ![[COUNT_IND_CALLEE]] = !{!"VP", i32 0, i64 84, i64 111, i64 48, i64 222, i64 24, i64 333, i64 12}
-; CHECK: ![[COUNT_CALLER]] = !{!"branch_weights", i64 800}
 ; CHECK: ![[COUNT_IND_CALLER]] = !{!"VP", i32 0, i64 56, i64 111, i64 32, i64 222, i64 16, i64 333, i64 8}

test/Transforms/Inline/prof-update-sample.ll

 ; RUN: opt < %s -inline -S | FileCheck %s
 ; Checks if inliner updates branch_weights annotation for call instructions.
 
 declare void @ext();
 declare void @ext1();
 @func = global void ()* null
 
 ; CHECK: define void @callee(i32 %n) !prof ![[ENTRY_COUNT:[0-9]*]]
 define void  @callee(i32 %n) !prof !15 {
   %cond = icmp sle i32 %n, 10
   br i1 %cond, label %cond_true, label %cond_false
 cond_true:
 ; ext1 is optimized away, thus not updated.
 ; CHECK: call void @ext1(), !prof ![[COUNT_CALLEE1:[0-9]*]]
   call void @ext1(), !prof !16
   ret void
 cond_false:
 ; ext is cloned and updated.
 ; CHECK: call void @ext(), !prof ![[COUNT_CALLEE:[0-9]*]]
   call void @ext(), !prof !16
   %f = load void ()*, void ()** @func
 ; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]] 
   call void %f(), !prof !18
   ret void
 }
 
 ; CHECK: define void @caller()
 define void @caller() {
 ; CHECK: call void @ext(), !prof ![[COUNT_CALLER:[0-9]*]]
 ; CHECK: call void %f.i(), !prof ![[COUNT_IND_CALLER:[0-9]*]]
   call void @callee(i32 15), !prof !17
   ret void
 }
 
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
 !3 = !{!"ProfileFormat", !"SampleProfile"}
 !4 = !{!"TotalCount", i64 10000}
 !5 = !{!"MaxCount", i64 10}
 !6 = !{!"MaxInternalCount", i64 1}
 !7 = !{!"MaxFunctionCount", i64 2000}
 !8 = !{!"NumCounts", i64 2}
 !9 = !{!"NumFunctions", i64 2}
 !10 = !{!"DetailedSummary", !11}
 !11 = !{!12, !13, !14}
 !12 = !{i32 10000, i64 100, i32 1}
 !13 = !{i32 999000, i64 100, i32 1}
 !14 = !{i32 999999, i64 1, i32 2}
 !15 = !{!"function_entry_count", i64 1000}
 !16 = !{!"branch_weights", i64 2000}
 !17 = !{!"branch_weights", i64 400}
 !18 = !{!"VP", i32 0, i64 140, i64 111, i64 80, i64 222, i64 40, i64 333, i64 20}
 attributes #0 = { alwaysinline }
 ; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 600}
 ; CHECK: ![[COUNT_CALLEE1]] = !{!"branch_weights", i64 2000}
 ; CHECK: ![[COUNT_CALLEE]] = !{!"branch_weights", i64 1200}
 ; CHECK: ![[COUNT_IND_CALLEE]] = !{!"VP", i32 0, i64 84, i64 111, i64 48, i64 222, i64 24, i64 333, i64 12}
 ; CHECK: ![[COUNT_CALLER]] = !{!"branch_weights", i64 800}
 ; CHECK: ![[COUNT_IND_CALLER]] = !{!"VP", i32 0, i64 56, i64 111, i64 32, i64 222, i64 16, i64 333, i64 8}

test/Transforms/Inline/prof-update.ll

-; RUN: opt < %s -inline -S | FileCheck %s
-; Checks if inliner updates branch_weights annotation for call instructions.
-
-declare void @ext();
-declare void @ext1();
-@func = global void ()* null
-
-; CHECK: define void @callee(i32 %n) !prof ![[ENTRY_COUNT:[0-9]*]]
-define void  @callee(i32 %n) !prof !15 {
-  %cond = icmp sle i32 %n, 10
-  br i1 %cond, label %cond_true, label %cond_false
-cond_true:
-; ext1 is optimized away, thus not updated.
-; CHECK: call void @ext1(), !prof ![[COUNT_CALLEE1:[0-9]*]]
-  call void @ext1(), !prof !16
-  ret void
-cond_false:
-; ext is cloned and updated.
-; CHECK: call void @ext(), !prof ![[COUNT_CALLEE:[0-9]*]]
-  call void @ext(), !prof !16
-  %f = load void ()*, void ()** @func
-; CHECK: call void %f(), !prof ![[COUNT_IND_CALLEE:[0-9]*]] 
-  call void %f(), !prof !18
-  ret void
-}
-
-; CHECK: define void @caller()
-define void @caller() {
-; CHECK: call void @ext(), !prof ![[COUNT_CALLER:[0-9]*]]
-; CHECK: call void %f.i(), !prof ![[COUNT_IND_CALLER:[0-9]*]]
-  call void @callee(i32 15), !prof !17
-  ret void
-}
-
-!llvm.module.flags = !{!1}
-!1 = !{i32 1, !"ProfileSummary", !2}
-!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
-!3 = !{!"ProfileFormat", !"SampleProfile"}
-!4 = !{!"TotalCount", i64 10000}
-!5 = !{!"MaxCount", i64 10}
-!6 = !{!"MaxInternalCount", i64 1}
-!7 = !{!"MaxFunctionCount", i64 2000}
-!8 = !{!"NumCounts", i64 2}
-!9 = !{!"NumFunctions", i64 2}
-!10 = !{!"DetailedSummary", !11}
-!11 = !{!12, !13, !14}
-!12 = !{i32 10000, i64 100, i32 1}
-!13 = !{i32 999000, i64 100, i32 1}
-!14 = !{i32 999999, i64 1, i32 2}
-!15 = !{!"function_entry_count", i64 1000}
-!16 = !{!"branch_weights", i64 2000}
-!17 = !{!"branch_weights", i64 400}
-!18 = !{!"VP", i32 0, i64 140, i64 111, i64 80, i64 222, i64 40, i64 333, i64 20}
-attributes #0 = { alwaysinline }
-; CHECK: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 600}
-; CHECK: ![[COUNT_CALLEE1]] = !{!"branch_weights", i64 2000}
-; CHECK: ![[COUNT_CALLEE]] = !{!"branch_weights", i64 1200}
-; CHECK: ![[COUNT_IND_CALLEE]] = !{!"VP", i32 0, i64 84, i64 111, i64 48, i64 222, i64 24, i64 333, i64 12}
-; CHECK: ![[COUNT_CALLER]] = !{!"branch_weights", i64 800}
-; CHECK: ![[COUNT_IND_CALLER]] = !{!"VP", i32 0, i64 56, i64 111, i64 32, i64 222, i64 16, i64 333, i64 8}