Index: lib/Transforms/IPO/FunctionImport.cpp
===================================================================
--- lib/Transforms/IPO/FunctionImport.cpp
+++ lib/Transforms/IPO/FunctionImport.cpp
@@ -265,7 +265,8 @@
}
}
-using EdgeInfo = std::pair;
+using EdgeInfo = std::tuple;
/// Compute the list of functions to import for a given caller. Mark these
/// imported functions and the symbols they reference in their source module as
@@ -285,9 +286,10 @@
continue;
}
+ const bool IsHotCallsite = Edge.second.Hotness == CalleeInfo::HotnessType::Hot;
// FIXME: Also lower the threshold for cold callsites.
const auto NewThreshold =
- Edge.second.Hotness == CalleeInfo::HotnessType::Hot
+ IsHotCallsite
? Threshold * ImportHotMultiplier
: Threshold;
auto *CalleeSummary = selectCallee(GUID, NewThreshold, Index);
@@ -339,7 +341,7 @@
}
// Insert the newly imported function to the worklist.
- Worklist.push_back(std::make_pair(ResolvedCalleeSummary, Threshold));
+ Worklist.emplace_back(ResolvedCalleeSummary, Threshold, IsHotCallsite);
}
}
@@ -370,15 +372,17 @@
ExportLists);
}
+ // Process the newly imported functions and add callees to the worklist.
while (!Worklist.empty()) {
- auto FuncInfo = Worklist.pop_back_val();
- auto *Summary = FuncInfo.first;
- auto Threshold = FuncInfo.second;
-
- // Process the newly imported functions and add callees to the worklist.
- // Adjust the threshold
- Threshold = Threshold * ImportInstrFactor;
-
+ const FunctionSummary *Summary;
+ unsigned Threshold;
+ bool IsHotCallsite;
+ std::tie(Summary, Threshold, IsHotCallsite) = Worklist.pop_back_val();
+
+ // Adjust the threshold only for non-hot callsites. We don't do it for hot
+ // callsites because we can then inline chain of hot calls.
+ if (!IsHotCallsite)
+ Threshold = Threshold * ImportInstrFactor;
computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries,
Worklist, ImportList, ExportLists);
}
Index: test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
===================================================================
--- test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
+++ test/Transforms/FunctionImport/Inputs/hotness_based_import.ll
@@ -6,38 +6,72 @@
define void @hot1() #1 {
ret void
}
-define void @hot2() #1 {
- call void @externalFunction()
- call void @externalFunction()
+define void @hot2() #1 !prof !20 {
+ call void @calledFromHot()
+ call void @calledFromHot()
ret void
}
-define void @hot3() #1 {
- call void @externalFunction()
- call void @externalFunction()
- call void @externalFunction()
+define void @hot3() #1 !prof !20 {
+ call void @calledFromHot()
+ call void @calledFromHot()
+ call void @calledFromHot()
ret void
}
-define void @cold() #1 {
+define void @cold() #1 !prof !0 {
ret void
}
-define void @cold2() #1 {
- call void @externalFunction()
- call void @externalFunction()
+define void @cold2() #1 !prof !0 {
+ call void @calledFromHot()
+ call void @calledFromHot()
ret void
}
define void @none1() #1 {
ret void
}
+
define void @none2() #1 {
- call void @externalFunction()
+ call void @calledFromHot()
ret void
}
define void @none3() #1 {
- call void @externalFunction()
- call void @externalFunction()
+ call void @calledFromHot()
+ call void @calledFromHot()
+ ret void
+}
+
+define void @calledFromCold() {
+ ret void
+}
+
+define void @calledFromHot() !prof !20 {
+ call void @calledFromHot2()
+ ret void
+}
+
+define void @calledFromHot2() {
+ call void @calledFromHot3()
ret void
}
+declare void @calledFromHot3()
+
+!0 = !{!"function_entry_count", i64 1}
+!20 = !{!"function_entry_count", i64 110}
+
+!llvm.module.flags = !{!1}
-declare void @externalFunction()
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
\ No newline at end of file
Index: test/Transforms/FunctionImport/hotness_based_import.ll
===================================================================
--- test/Transforms/FunctionImport/hotness_based_import.ll
+++ test/Transforms/FunctionImport/hotness_based_import.ll
@@ -6,8 +6,11 @@
; Test import with default hot multiplier (3)
; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-instr-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT
; HOT-DEFAULT-DAG: define available_externally void @hot1()
; HOT-DEFAULT-DAG: define available_externally void @hot2()
+; HOT-DEFAULT-DAG: define available_externally void @calledFromHot()
+; HOT-DEFAULT-DAG: define available_externally void @calledFromHot2()
; HOT-DEFAULT-DAG: define available_externally void @cold()
; HOT-DEFAULT-DAG: define available_externally void @none1()
@@ -19,6 +22,7 @@
; Test import with hot multiplier 1.0 - treat hot callsites as normal.
; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE
+; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE
; HOT-ONE-DAG: define available_externally void @hot1()
; HOT-ONE-DAG: define available_externally void @cold()
; HOT-ONE-DAG: define available_externally void @none1()
@@ -30,6 +34,8 @@
; Test import with hot multiplier 0.0 and high threshold - don't import functions called from hot callsite.
+; for hot callsites the threshold doesn't decay, that's why it doesn't make difference if the import factor
+; is 0 or higher - calledFromHot's will be imported
; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=10 -import-hot-multiplier=0.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ZERO
; HOT-ZERO-DAG: define available_externally void @cold()
; HOT-ZERO-DAG: define available_externally void @none1()
@@ -39,7 +45,8 @@
; HOT-ZERO-NOT: define available_externally void @hot2()
; HOT-ZERO-NOT: define available_externally void @hot1()
; HOT-ZERO-NOT: define available_externally void @hot3()
-
+; HOT-ZERO-NOT: define available_externally void @calledFromHot()
+; HOT-ZERO-NOT: define available_externally void @calledFromHot2()
; ModuleID = 'thinlto-function-summary-callgraph.ll'