Index: llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp =================================================================== --- llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp +++ llvm/trunk/lib/Transforms/IPO/FunctionImport.cpp @@ -49,6 +49,13 @@ "`import-instr-limit` threshold by this factor " "before processing newly imported functions")); +static cl::opt ImportHotInstrFactor( + "import-hot-evolution-factor", cl::init(1.0), cl::Hidden, + cl::value_desc("x"), + cl::desc("As we import functions called from hot callsite, multiply the " + "`import-instr-limit` threshold by this factor " + "before processing newly imported functions")); + static cl::opt ImportHotMultiplier( "import-hot-multiplier", cl::init(3.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for hot callsites")); @@ -300,6 +307,7 @@ const auto NewThreshold = Threshold * GetBonusMultiplier(Edge.second.Hotness); + auto *CalleeSummary = selectCallee(GUID, NewThreshold, Index); if (!CalleeSummary) { DEBUG(dbgs() << "ignored! No qualifying callee with summary found.\n"); @@ -348,8 +356,20 @@ } } + auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) { + // Adjust the threshold for next level of imported functions. + // The threshold is different for hot callsites because we can then + // inline chains of hot calls. + if (IsHotCallsite) + return Threshold * ImportHotInstrFactor; + return Threshold * ImportInstrFactor; + }; + + bool IsHotCallsite = Edge.second.Hotness == CalleeInfo::HotnessType::Hot; + // Insert the newly imported function to the worklist. - Worklist.push_back(std::make_pair(ResolvedCalleeSummary, Threshold)); + Worklist.emplace_back(ResolvedCalleeSummary, + GetAdjustedThreshold(Threshold, IsHotCallsite)); } } @@ -380,15 +400,12 @@ ExportLists); } + // Process the newly imported functions and add callees to the worklist. while (!Worklist.empty()) { auto FuncInfo = Worklist.pop_back_val(); auto *Summary = FuncInfo.first; auto Threshold = FuncInfo.second; - // Process the newly imported functions and add callees to the worklist. - // Adjust the threshold - Threshold = Threshold * ImportInstrFactor; - computeImportForFunction(*Summary, Index, Threshold, DefinedGVSummaries, Worklist, ImportList, ExportLists); } Index: llvm/trunk/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll =================================================================== --- llvm/trunk/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll +++ llvm/trunk/test/Transforms/FunctionImport/Inputs/hotness_based_import.ll @@ -6,38 +6,76 @@ define void @hot1() #1 { ret void } -define void @hot2() #1 { - call void @externalFunction() - call void @externalFunction() +define void @hot2() #1 !prof !20 { + call void @calledFromHot() + call void @calledFromHot() ret void } -define void @hot3() #1 { - call void @externalFunction() - call void @externalFunction() - call void @externalFunction() +define void @hot3() #1 !prof !20 { + call void @calledFromHot() + call void @calledFromHot() + call void @calledFromHot() ret void } -define void @cold() #1 { +define void @cold() #1 !prof !0 { ret void } -define void @cold2() #1 { - call void @externalFunction() - call void @externalFunction() +define void @cold2() #1 !prof !0 { + call void @calledFromCold() + call void @calledFromCold() ret void } define void @none1() #1 { ret void } + define void @none2() #1 { - call void @externalFunction() + call void @calledFromNone() ret void } define void @none3() #1 { - call void @externalFunction() - call void @externalFunction() + call void @calledFromNone() + call void @calledFromNone() + ret void +} + +define void @calledFromCold() { ret void } +define void @calledFromHot() !prof !20 { + call void @calledFromHot2() + ret void +} + +define void @calledFromHot2() !prof !20 { + call void @calledFromHot3() + ret void +} + +define void @calledFromNone() !prof !0 { + ret void +} + +declare void @calledFromHot3() + +!0 = !{!"function_entry_count", i64 1} +!20 = !{!"function_entry_count", i64 110} + +!llvm.module.flags = !{!1} -declare void @externalFunction() +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 10} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} \ No newline at end of file Index: llvm/trunk/test/Transforms/FunctionImport/hotness_based_import.ll =================================================================== --- llvm/trunk/test/Transforms/FunctionImport/hotness_based_import.ll +++ llvm/trunk/test/Transforms/FunctionImport/hotness_based_import.ll @@ -3,20 +3,38 @@ ; RUN: opt -module-summary %p/Inputs/hotness_based_import.ll -o %t2.bc ; RUN: llvm-lto -thinlto -o %t3 %t.bc %t2.bc -; Test import with default hot multiplier (3) +; Test import with default hot multiplier (3) and default hot-evolution-factor (1.0) ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-cold-multiplier=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT - +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-instr-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-hot-evolution-factor=1.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-DEFAULT ; HOT-DEFAULT-DAG: define available_externally void @hot1() ; HOT-DEFAULT-DAG: define available_externally void @hot2() +; HOT-DEFAULT-DAG: define available_externally void @calledFromHot() +; HOT-DEFAULT-DAG: define available_externally void @calledFromHot2() ; HOT-DEFAULT-DAG: define available_externally void @none1() - ; HOT-DEFAULT-NOT: define available_externally void @cold() ; HOT-DEFAULT-NOT: define available_externally void @hot3() ; HOT-DEFAULT-NOT: define available_externally void @none2() ; HOT-DEFAULT-NOT: define available_externally void @none3() ; HOT-DEFAULT-NOT: define available_externally void @cold2() +; HOT-DEFAULT-NOT: define available_externally void @calledFromCold() +; HOT-DEFAULT-NOT: define available_externally void @calledFromNone() +; This one tests if we decay threshold for hot callsites. With hot-evolution-factor of 0 +; we should not import any of calledFromHot functions +; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 --S -import-hot-multiplier=3.0 -import-hot-evolution-factor=0.0 | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-EVOLUTION +; HOT-EVOLUTION-DAG: define available_externally void @hot1() +; HOT-EVOLUTION-DAG: define available_externally void @hot2() +; HOT-EVOLUTION-DAG: define available_externally void @none1() +; HOT-EVOLUTION-NOT: define available_externally void @hot3() +; HOT-EVOLUTION-NOT: define available_externally void @cold() +; HOT-EVOLUTION-NOT: define available_externally void @none2() +; HOT-EVOLUTION-NOT: define available_externally void @none3() +; HOT-EVOLUTION-NOT: define available_externally void @cold2() +; HOT-EVOLUTION-NOT: define available_externally void @calledFromHot() +; HOT-EVOLUTION-NOT: define available_externally void @calledFromHot2() ; Test import with hot multiplier 1.0 - treat hot callsites as normal. ; RUN: opt -function-import -summary-file %t3.thinlto.bc %t.bc -import-instr-limit=1 -import-hot-multiplier=1.0 --S | FileCheck %s --check-prefix=CHECK --check-prefix=HOT-ONE @@ -46,10 +64,13 @@ ; HOT-ZERO-DAG: define available_externally void @none2() ; HOT-ZERO-DAG: define available_externally void @none3() ; HOT-ZERO-DAG: define available_externally void @cold2() +; HOT-ZERO-DAG: define available_externally void @calledFromCold() +; HOT-ZERO-DAG: define available_externally void @calledFromNone() ; HOT-ZERO-NOT: define available_externally void @hot2() ; HOT-ZERO-NOT: define available_externally void @hot1() ; HOT-ZERO-NOT: define available_externally void @hot3() - +; HOT-ZERO-NOT: define available_externally void @calledFromHot() +; HOT-ZERO-NOT: define available_externally void @calledFromHot2() ; ModuleID = 'thinlto-function-summary-callgraph.ll'