diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -759,30 +759,32 @@ CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", "", nullptr, PGOOptions::IRInstr, PGOOptions::NoCSAction, - CodeGenOpts.DebugInfoForProfiling); + PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.hasProfileIRUse()) { // -fprofile-use. auto CSAction = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse : PGOOptions::NoCSAction; - PGOOpt = - PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "", - CodeGenOpts.ProfileRemappingFile, VFS, PGOOptions::IRUse, - CSAction, CodeGenOpts.DebugInfoForProfiling); + PGOOpt = PGOOptions( + CodeGenOpts.ProfileInstrumentUsePath, "", + CodeGenOpts.ProfileRemappingFile, VFS, PGOOptions::IRUse, CSAction, + PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling); } else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use PGOOpt = PGOOptions( CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile, VFS, PGOOptions::SampleUse, PGOOptions::NoCSAction, - CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling); + PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling, + CodeGenOpts.PseudoProbeForProfiling); else if (CodeGenOpts.PseudoProbeForProfiling) // -fpseudo-probe-for-profiling PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction, - PGOOptions::NoCSAction, + PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling, true); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction, - PGOOptions::NoCSAction, true); + PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None, + true); // Check to see if we want to generate a CS profile. if (CodeGenOpts.hasProfileCSIRInstr()) { @@ -799,13 +801,13 @@ : CodeGenOpts.InstrProfileOutput; PGOOpt->CSAction = PGOOptions::CSIRInstr; } else - PGOOpt = - PGOOptions("", - CodeGenOpts.InstrProfileOutput.empty() - ? getDefaultProfileGenName() - : CodeGenOpts.InstrProfileOutput, - "", nullptr, PGOOptions::NoAction, PGOOptions::CSIRInstr, - CodeGenOpts.DebugInfoForProfiling); + PGOOpt = PGOOptions("", + CodeGenOpts.InstrProfileOutput.empty() + ? getDefaultProfileGenName() + : CodeGenOpts.InstrProfileOutput, + "", nullptr, PGOOptions::NoAction, + PGOOptions::CSIRInstr, PGOOptions::ColdFuncAttr::None, + CodeGenOpts.DebugInfoForProfiling); } if (TM) TM->setPGOOption(PGOOpt); diff --git a/llvm/include/llvm/Support/PGOOptions.h b/llvm/include/llvm/Support/PGOOptions.h --- a/llvm/include/llvm/Support/PGOOptions.h +++ b/llvm/include/llvm/Support/PGOOptions.h @@ -27,10 +27,12 @@ struct PGOOptions { enum PGOAction { NoAction, IRInstr, IRUse, SampleUse }; enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse }; + enum class ColdFuncAttr { None, OptSize, MinSize, OptNone }; PGOOptions(std::string ProfileFile, std::string CSProfileGenFile, std::string ProfileRemappingFile, IntrusiveRefCntPtr FS, PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction, + ColdFuncAttr ColdType = ColdFuncAttr::None, bool DebugInfoForProfiling = false, bool PseudoProbeForProfiling = false); PGOOptions(const PGOOptions &); @@ -42,6 +44,7 @@ std::string ProfileRemappingFile; PGOAction Action; CSPGOAction CSAction; + ColdFuncAttr ColdType; bool DebugInfoForProfiling; bool PseudoProbeForProfiling; IntrusiveRefCntPtr FS; diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h --- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -18,6 +18,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/IR/PassManager.h" +#include "llvm/Support/PGOOptions.h" #include #include @@ -62,9 +63,11 @@ /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: - PGOInstrumentationUse(std::string Filename = "", - std::string RemappingFilename = "", bool IsCS = false, - IntrusiveRefCntPtr FS = nullptr); + PGOInstrumentationUse( + std::string Filename = "", std::string RemappingFilename = "", + bool IsCS = false, + PGOOptions::ColdFuncAttr ColdType = PGOOptions::ColdFuncAttr::None, + IntrusiveRefCntPtr FS = nullptr); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); @@ -73,6 +76,7 @@ std::string ProfileRemappingFileName; // If this is a context sensitive instrumentation. bool IsCS; + PGOOptions::ColdFuncAttr ColdType; IntrusiveRefCntPtr FS; }; diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -237,19 +237,23 @@ std::optional PGOOpt; if (!Conf.SampleProfile.empty()) PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping, FS, - PGOOptions::SampleUse, PGOOptions::NoCSAction, true); + PGOOptions::SampleUse, PGOOptions::NoCSAction, + PGOOptions::ColdFuncAttr::None, true); else if (Conf.RunCSIRInstr) { - PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping, FS, - PGOOptions::IRUse, PGOOptions::CSIRInstr, - Conf.AddFSDiscriminator); + PGOOpt = + PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping, FS, + PGOOptions::IRUse, PGOOptions::CSIRInstr, + PGOOptions::ColdFuncAttr::None, Conf.AddFSDiscriminator); } else if (!Conf.CSIRProfile.empty()) { - PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping, FS, - PGOOptions::IRUse, PGOOptions::CSIRUse, - Conf.AddFSDiscriminator); + PGOOpt = + PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping, FS, + PGOOptions::IRUse, PGOOptions::CSIRUse, + PGOOptions::ColdFuncAttr::None, Conf.AddFSDiscriminator); NoPGOWarnMismatch = !Conf.PGOWarnMismatch; } else if (Conf.AddFSDiscriminator) { PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction, - PGOOptions::NoCSAction, true); + PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None, + true); } TM->setPGOOption(PGOOpt); diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -766,8 +766,8 @@ if (!RunProfileGen) { assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); - MPM.addPass( - PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS)); + MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, + PGOOpt->ColdType, FS)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); @@ -803,8 +803,8 @@ IntrusiveRefCntPtr FS) { if (!RunProfileGen) { assert(!ProfileFile.empty() && "Profile use expecting a profile file!"); - MPM.addPass( - PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS)); + MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, + PGOOpt->ColdType, FS)); // Cache ProfileSummaryAnalysis once to avoid the potential need to insert // RequireAnalysisPass for PSI before subsequent non-module passes. MPM.addPass(RequireAnalysisPass()); diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -90,7 +90,7 @@ MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation()) MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion()) MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen()) -MODULE_PASS("pgo-instr-use", PGOInstrumentationUse()) +MODULE_PASS("pgo-instr-use", PGOInstrumentationUse("", "", false, PGOOpt ? PGOOpt->ColdType : PGOOptions::ColdFuncAttr::None)) MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs())) MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs())) MODULE_PASS("print-callgraph-sccs", CallGraphSCCsPrinterPass(dbgs())) diff --git a/llvm/lib/Support/PGOOptions.cpp b/llvm/lib/Support/PGOOptions.cpp --- a/llvm/lib/Support/PGOOptions.cpp +++ b/llvm/lib/Support/PGOOptions.cpp @@ -14,11 +14,11 @@ PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile, std::string ProfileRemappingFile, IntrusiveRefCntPtr FS, PGOAction Action, - CSPGOAction CSAction, bool DebugInfoForProfiling, - bool PseudoProbeForProfiling) + CSPGOAction CSAction, ColdFuncAttr ColdType, + bool DebugInfoForProfiling, bool PseudoProbeForProfiling) : ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile), ProfileRemappingFile(ProfileRemappingFile), Action(Action), - CSAction(CSAction), + CSAction(CSAction), ColdType(ColdType), DebugInfoForProfiling(DebugInfoForProfiling || (Action == SampleUse && !PseudoProbeForProfiling)), PseudoProbeForProfiling(PseudoProbeForProfiling), FS(std::move(FS)) { diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp --- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -2237,7 +2237,7 @@ function_ref LookupTLI, function_ref LookupBPI, function_ref LookupBFI, - ProfileSummaryInfo *PSI, bool IsCS) { + ProfileSummaryInfo *PSI, bool IsCS, PGOOptions::ColdFuncAttr ColdType) { LLVM_DEBUG(dbgs() << "Read in profile counters: "); auto &Ctx = M.getContext(); // Read the counter array from file. @@ -2420,6 +2420,33 @@ continue; } F->addFnAttr(Attribute::Cold); + // Add optsize/minsize/optnone if requested. + switch (ColdType) { + case PGOOptions::ColdFuncAttr::None: + break; + case PGOOptions::ColdFuncAttr::OptSize: + if (!F->hasFnAttribute(Attribute::OptimizeNone) && + !F->hasFnAttribute(Attribute::OptimizeForSize) && + !F->hasFnAttribute(Attribute::MinSize)) { + F->addFnAttr(Attribute::OptimizeForSize); + } + break; + case PGOOptions::ColdFuncAttr::MinSize: + // Change optsize to minsize. + if (!F->hasFnAttribute(Attribute::OptimizeNone) && + !F->hasFnAttribute(Attribute::MinSize)) { + F->removeFnAttr(Attribute::OptimizeForSize); + F->addFnAttr(Attribute::MinSize); + } + break; + case PGOOptions::ColdFuncAttr::OptNone: + // Strip optsize/minsize. + F->removeFnAttr(Attribute::OptimizeForSize); + F->removeFnAttr(Attribute::MinSize); + F->addFnAttr(Attribute::OptimizeNone); + F->addFnAttr(Attribute::NoInline); + break; + } LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n"); } @@ -2428,10 +2455,10 @@ PGOInstrumentationUse::PGOInstrumentationUse( std::string Filename, std::string RemappingFilename, bool IsCS, - IntrusiveRefCntPtr VFS) + PGOOptions::ColdFuncAttr ColdType, IntrusiveRefCntPtr VFS) : ProfileFileName(std::move(Filename)), ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS), - FS(std::move(VFS)) { + ColdType(ColdType), FS(std::move(VFS)) { if (!PGOTestProfileFile.empty()) ProfileFileName = PGOTestProfileFile; if (!PGOTestProfileRemappingFile.empty()) @@ -2457,7 +2484,8 @@ auto *PSI = &AM.getResult(M); if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, *FS, - LookupTLI, LookupBPI, LookupBFI, PSI, IsCS)) + LookupTLI, LookupBPI, LookupBFI, PSI, IsCS, + ColdType)) return PreservedAnalyses::all(); return PreservedAnalyses::none(); diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext @@ -0,0 +1,42 @@ +# IR level Instrumentation Flag +:ir +:entry_first +hot +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +9000 + +cold +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +10 + +cold1 +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +10 + +cold2 +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +10 + +cold3 +# Func Hash: +742261418966908927 +# Num Counters: +1 +# Counter Values: +10 diff --git a/llvm/test/Transforms/PGOProfile/cold-func-attr.ll b/llvm/test/Transforms/PGOProfile/cold-func-attr.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/cold-func-attr.ll @@ -0,0 +1,62 @@ +; RUN: llvm-profdata merge %S/Inputs/cold-func-attr.proftext -o %t.profdata +; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=none | FileCheck %s --check-prefixes=NONE +; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=optsize | FileCheck %s --check-prefixes=OPTSIZE,CHECK +; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=minsize | FileCheck %s --check-prefixes=MINSIZE,CHECK +; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=optnone | FileCheck %s --check-prefixes=OPTNONE,CHECK + +; NONE: Function Attrs: cold{{$}} +; OPTSIZE: Function Attrs: cold optsize{{$}} +; MINSIZE: Function Attrs: cold minsize{{$}} +; OPTNONE: Function Attrs: cold noinline optnone{{$}} +; CHECK-NEXT: define void @cold() + +; NONE: Function Attrs: cold optsize{{$}} +; OPTSIZE: Function Attrs: cold optsize{{$}} +; MINSIZE: Function Attrs: cold minsize{{$}} +; OPTNONE: Function Attrs: cold noinline optnone{{$}} +; CHECK-NEXT: define void @cold1() + +; NONE: Function Attrs: cold minsize{{$}} +; OPTSIZE: Function Attrs: cold minsize{{$}} +; MINSIZE: Function Attrs: cold minsize{{$}} +; OPTNONE: Function Attrs: cold noinline optnone{{$}} +; CHECK-NEXT: define void @cold2() + +; CHECK: Function Attrs: cold noinline optnone{{$}} +; CHECK-NEXT: define void @cold3() + +; CHECK-NOT: Function Attrs: {{.*}}optsize +; CHECK-NOT: Function Attrs: {{.*}}minsize +; CHECK-NOT: Function Attrs: {{.*}}optnone + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@s = common dso_local local_unnamed_addr global i32 0, align 4 + +define void @cold() { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold1() optsize { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold2() minsize { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold3() optnone noinline { + store i32 1, ptr @s, align 4 + ret void +} + +define void @hot() { + %l = load i32, ptr @s, align 4 + %add = add nsw i32 %l, 4 + store i32 %add, ptr @s, align 4 + ret void +} diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp --- a/llvm/tools/opt/NewPMDriver.cpp +++ b/llvm/tools/opt/NewPMDriver.cpp @@ -199,6 +199,18 @@ cl::desc("Path to the profile remapping file."), cl::Hidden); +static cl::opt PGOColdFuncAttr( + "pgo-cold-func-attr", cl::init(PGOOptions::ColdFuncAttr::None), cl::Hidden, + cl::desc( + "Function attribute to apply to cold functions as determined by PGO"), + cl::values(clEnumValN(PGOOptions::ColdFuncAttr::None, "none", "None"), + clEnumValN(PGOOptions::ColdFuncAttr::OptSize, "optsize", + "Mark cold functions with optsize."), + clEnumValN(PGOOptions::ColdFuncAttr::MinSize, "minsize", + "Mark cold functions with minsize."), + clEnumValN(PGOOptions::ColdFuncAttr::OptNone, "optnone", + "Mark cold functions with optnone."))); + static cl::opt DebugInfoForProfiling( "debug-info-for-profiling", cl::init(false), cl::Hidden, cl::desc("Emit special debug info to enable PGO profile generation.")); @@ -338,21 +350,23 @@ std::optional P; switch (PGOKindFlag) { case InstrGen: - P = PGOOptions(ProfileFile, "", "", FS, PGOOptions::IRInstr); + P = PGOOptions(ProfileFile, "", "", FS, PGOOptions::IRInstr, + PGOOptions::NoCSAction, PGOColdFuncAttr); break; case InstrUse: - P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS, - PGOOptions::IRUse); + P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS, PGOOptions::IRUse, + PGOOptions::NoCSAction, PGOColdFuncAttr); break; case SampleUse: P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS, - PGOOptions::SampleUse); + PGOOptions::SampleUse, PGOOptions::NoCSAction, + PGOColdFuncAttr); break; case NoPGO: if (DebugInfoForProfiling || PseudoProbeForProfiling) P = PGOOptions("", "", "", nullptr, PGOOptions::NoAction, - PGOOptions::NoCSAction, DebugInfoForProfiling, - PseudoProbeForProfiling); + PGOOptions::NoCSAction, PGOColdFuncAttr, + DebugInfoForProfiling, PseudoProbeForProfiling); else P = std::nullopt; } @@ -372,7 +386,8 @@ P->CSProfileGenFile = CSProfileGenFile; } else P = PGOOptions("", CSProfileGenFile, ProfileRemappingFile, FS, - PGOOptions::NoAction, PGOOptions::CSIRInstr); + PGOOptions::NoAction, PGOOptions::CSIRInstr, + PGOColdFuncAttr); } else /* CSPGOKindFlag == CSInstrUse */ { if (!P) { errs() << "CSInstrUse needs to be together with InstrUse";