[SampleFDO] Extend profile-sample-accurate option to cover isFunctionColdInCallGraph

[wmi]

Summary

For SampleFDO, when a callsite doesn't appear in the profile, it will not be marked as cold callsite unless the option -profile-sample-accurate is specified.

But profile-sample-accurate doesn't cover function isFunctionColdInCallGraph which is used to decide whether a function should be put into text.unlikely section, so even if the user knows the profile is accurate and specifies profile-sample-accurate, those functions not appearing in the sample profile are still not be put into text.unlikely section right now.

The patch fixes that.

[tejohnson]

lib/Analysis/ProfileSummaryInfo.cpp

268

Don't need to do extra checking unless !Count, so you could simplify like:

if (Count)

return isColdCount(*Count);

if (!hasSampleProfile())

return false;

const Function *F = BB->getParent();
return ProfileSampleAccurate ||

(F && F->hasFnAttribute("profile-sample-accurate"));

[wmi]

lib/Analysis/ProfileSummaryInfo.cpp

268

Thanks, that is better, Fixed.

[wmi]

Address Teresa's comment.

[tejohnson]

lgtm

[wmi]

Thanks to Easwaran who gave me a good suggestion offline to improve the patch. We can initialize function entry count to 0 when ProfileSampleAccurate is specified. That could save the effort to handle ProfileSampleAccurate in multiple places. I will post the updated patch in a separate review thread.

lib/Analysis/ProfileSummaryInfo.cpp

 bool ProfileSummaryInfo::isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI) {
   auto Count = BFI->getBlockProfileCount(BB);
   return Count && isHotCount(*Count);
 }
 
 bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB,
                                   BlockFrequencyInfo *BFI) {
   auto Count = BFI->getBlockProfileCount(BB);
+  if (hasSampleProfile()) {
+    const Function *F = BB->getParent();
+    if (ProfileSampleAccurate ||
+        (F && F->hasFnAttribute("profile-sample-accurate")))
+      return !Count || isColdCount(*Count);
+  }
   return Count && isColdCount(*Count);
 }
 
 bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS,
                                        BlockFrequencyInfo *BFI) {
   auto C = getProfileCount(CS.getInstruction(), BFI);
   return C && isHotCount(*C);
 }

test/Transforms/CodeGenPrepare/section-samplepgo.ll

 ; RUN: opt < %s -codegenprepare -S | FileCheck %s
+; RUN: opt < %s -codegenprepare -profile-sample-accurate -S | FileCheck %s --check-prefix ACCURATE
 
 target triple = "x86_64-pc-linux-gnu"
 
 ; This tests that hot/cold functions get correct section prefix assigned
 
 ; CHECK: hot_func{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
+; ACCURATE: hot_func{{.*}}!section_prefix ![[HOT_ID:[0-9]+]]
 ; The entry is hot
 define void @hot_func() !prof !15 {
   ret void
 }
 
 ; CHECK: hot_call_func{{.*}}!section_prefix ![[HOT_ID]]
+; ACCURATE: hot_call_func{{.*}}!section_prefix ![[HOT_ID]]
 ; The sum of 2 callsites are hot
 define void @hot_call_func() !prof !16 {
   call void @hot_func(), !prof !17
   call void @hot_func(), !prof !17
   ret void
 }
 
 ; CHECK-NOT: normal_func{{.*}}!section_prefix
+; ACCURATE-NOT: normal_func{{.*}}!section_prefix
 ; The sum of all callsites are neither hot or cold
 define void @normal_func() !prof !16 {
   call void @hot_func(), !prof !17
   call void @hot_func(), !prof !18
   call void @hot_func(), !prof !18
   ret void
 }
 
 ; CHECK: cold_func{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; ACCURATE: cold_func{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
 ; The entry and the callsite are both cold
 define void @cold_func() !prof !16 {
   call void @hot_func(), !prof !18
   ret void
 }
 
+
+; CHECK-NOT: foo_not_in_profile{{.*}}!section_prefix
+; The function not appearing in profile is neither hot nor cold
+;
+; ACCURATE: foo_not_in_profile{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; The function not appearing in profile is cold when -profile-sample-accurate
+; is on
+define void @foo_not_in_profile() !prof !19 {
+  call void @hot_func()
+  ret void
+}
+
+; CHECK: bar_not_in_profile{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; ACCURATE: bar_not_in_profile{{.*}}!section_prefix ![[COLD_ID:[0-9]+]]
+; The function not appearing in profile is cold when the func has
+; profile-sample-accurate attribute
+define void @bar_not_in_profile() #0 !prof !19 {
+  call void @hot_func()
+  ret void
+}
+
+attributes #0 = { "profile-sample-accurate" }
+
 ; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
 ; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
 !3 = !{!"ProfileFormat", !"SampleProfile"}
 !4 = !{!"TotalCount", i64 10000}
 !5 = !{!"MaxCount", i64 1000}
 !6 = !{!"MaxInternalCount", i64 1}
 !7 = !{!"MaxFunctionCount", i64 1000}
 !8 = !{!"NumCounts", i64 3}
 !9 = !{!"NumFunctions", i64 3}
 !10 = !{!"DetailedSummary", !11}
 !11 = !{!12, !13, !14}
 !12 = !{i32 10000, i64 100, i32 1}
 !13 = !{i32 999000, i64 100, i32 1}
 !14 = !{i32 999999, i64 1, i32 2}
 !15 = !{!"function_entry_count", i64 1000}
 !16 = !{!"function_entry_count", i64 1}
 !17 = !{!"branch_weights", i32 80}
 !18 = !{!"branch_weights", i32 1}
+!19 = !{!"function_entry_count", i64 -1}