Index: lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- lib/Transforms/IPO/FunctionImport.cpp +++ lib/Transforms/IPO/FunctionImport.cpp @@ -11,6 +11,7 @@ // //===----------------------------------------------------------------------===// +#include #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/ADT/SmallVector.h" @@ -265,7 +266,8 @@ } } -using EdgeInfo = std::pair; +using EdgeInfo = std::tuple; /// Compute the list of functions to import for a given caller. Mark these /// imported functions and the symbols they reference in their source module as @@ -285,9 +287,10 @@ continue; } + const bool IsHotCallsite = Edge.second.Hotness == CalleeInfo::HotnessType::Hot; // FIXME: Also lower the threshold for cold callsites. const auto NewThreshold = - Edge.second.Hotness == CalleeInfo::HotnessType::Hot + IsHotCallsite ? Threshold * ImportHotMultiplier : Threshold; auto *CalleeSummary = selectCallee(GUID, NewThreshold, Index); @@ -339,7 +342,7 @@ } // Insert the newly imported function to the worklist. - Worklist.push_back(std::make_pair(ResolvedCalleeSummary, Threshold)); + Worklist.emplace_back(ResolvedCalleeSummary, Threshold, IsHotCallsite); } } @@ -370,15 +373,19 @@ ExportLists); } + // Process the newly imported functions and add callees to the worklist. while (!Worklist.empty()) { - auto FuncInfo = Worklist.pop_back_val(); - auto *Summary = FuncInfo.first; - auto Threshold = FuncInfo.second; - - // Process the newly imported functions and add callees to the worklist. - // Adjust the threshold - Threshold = Threshold * ImportInstrFactor; - + const FunctionSummary *Summary; + unsigned Threshold; + bool IsHotCallsite; + std::tie(Summary, Threshold, IsHotCallsite) = Worklist.pop_back_val(); + + // Adjust the threshold only for non-hot callsites. We don't do it for hot + // callsites because we can then inline chain of hot calls. + if (!IsHotCallsite) + Threshold = Threshold * ImportInstrFactor; + else + std::cerr << Threshold << std::endl; computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries, Worklist, ImportList, ExportLists); } Index: test/Transforms/FunctionImport/Inputs/hotness_based_import.ll =================================================================== --- test/Transforms/FunctionImport/Inputs/hotness_based_import.ll +++ test/Transforms/FunctionImport/Inputs/hotness_based_import.ll @@ -6,38 +6,72 @@ define void @hot1() #1 { ret void } -define void @hot2() #1 { - call void @externalFunction() - call void @externalFunction() +define void @hot2() #1 !prof !20 { + call void @calledFromHot() + call void @calledFromHot() ret void } -define void @hot3() #1 { - call void @externalFunction() - call void @externalFunction() - call void @externalFunction() +define void @hot3() #1 !prof !20 { + call void @calledFromHot() + call void @calledFromHot() + call void @calledFromHot() ret void } -define void @cold() #1 { +define void @cold() #1 !prof !0 { ret void } -define void @cold2() #1 { - call void @externalFunction() - call void @externalFunction() +define void @cold2() #1 !prof !0 { + call void @calledFromHot() + call void @calledFromHot() ret void } define void @none1() #1 { ret void } + define void @none2() #1 { - call void @externalFunction() + call void @calledFromHot() ret void } define void @none3() #1 { - call void @externalFunction() - call void @externalFunction() + call void @calledFromHot() + call void @calledFromHot() + ret void +} + +define void @calledFromCold() { + ret void +} + +define void @calledFromHot() !prof !20 { + call void @calledFromHot2() + ret void +} + +define void @calledFromHot2() { + call void @calledFromHot3() ret void } +declare void @calledFromHot3() + +!0 = !{!"function_entry_count", i64 1} +!20 = !{!"function_entry_count", i64 110} + +!llvm.module.flags = !{!1} -declare void @externalFunction() +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} \ No newline at end of file Index: test/Transforms/FunctionImport/hotness_based_import.ll =================================================================== --- test/Transforms/FunctionImport/hotness_based_import.ll +++ test/Transforms/FunctionImport/hotness_based_import.ll @@ -6,8 +6,11 @@ ; Test import with default hot multiplier (3) ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-instr-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT ; HOT-DEFAULT-DAG: define available_externally void @hot1() ; HOT-DEFAULT-DAG: define available_externally void @hot2() +; HOT-DEFAULT-DAG: define available_externally void @calledFromHot() +; HOT-DEFAULT-DAG: define available_externally void @calledFromHot2() ; HOT-DEFAULT-DAG: define available_externally void @cold() ; HOT-DEFAULT-DAG: define available_externally void @none1() @@ -19,6 +22,7 @@ ; Test import with hot multiplier 1.0 - treat hot callsites as normal. ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE ; HOT-ONE-DAG: define available_externally void @hot1() ; HOT-ONE-DAG: define available_externally void @cold() ; HOT-ONE-DAG: define available_externally void @none1() @@ -30,6 +34,8 @@ ; Test import with hot multiplier 0.0 and high threshold - don't import functions called from hot callsite. +; for hot callsites the threshold doesn't decay, that's why it doesn't make difference if the import factor +; is 0 or higher - calledFromHot's will be imported ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=10 -import-hot-multiplier=0.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ZERO ; HOT-ZERO-DAG: define available_externally void @cold() ; HOT-ZERO-DAG: define available_externally void @none1() @@ -39,7 +45,8 @@ ; HOT-ZERO-NOT: define available_externally void @hot2() ; HOT-ZERO-NOT: define available_externally void @hot1() ; HOT-ZERO-NOT: define available_externally void @hot3() - +; HOT-ZERO-NOT: define available_externally void @calledFromHot() +; HOT-ZERO-NOT: define available_externally void @calledFromHot2() ; ModuleID = 'thinlto-function-summary-callgraph.ll'