diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -759,30 +759,32 @@
CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName()
: CodeGenOpts.InstrProfileOutput,
"", "", nullptr, PGOOptions::IRInstr, PGOOptions::NoCSAction,
- CodeGenOpts.DebugInfoForProfiling);
+ PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling);
else if (CodeGenOpts.hasProfileIRUse()) {
// -fprofile-use.
auto CSAction = CodeGenOpts.hasProfileCSIRUse() ? PGOOptions::CSIRUse
: PGOOptions::NoCSAction;
- PGOOpt =
- PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "",
- CodeGenOpts.ProfileRemappingFile, VFS, PGOOptions::IRUse,
- CSAction, CodeGenOpts.DebugInfoForProfiling);
+ PGOOpt = PGOOptions(
+ CodeGenOpts.ProfileInstrumentUsePath, "",
+ CodeGenOpts.ProfileRemappingFile, VFS, PGOOptions::IRUse, CSAction,
+ PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling);
} else if (!CodeGenOpts.SampleProfileFile.empty())
// -fprofile-sample-use
PGOOpt = PGOOptions(
CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
VFS, PGOOptions::SampleUse, PGOOptions::NoCSAction,
- CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
+ PGOOptions::ColdFuncAttr::None, CodeGenOpts.DebugInfoForProfiling,
+ CodeGenOpts.PseudoProbeForProfiling);
else if (CodeGenOpts.PseudoProbeForProfiling)
// -fpseudo-probe-for-profiling
PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction,
- PGOOptions::NoCSAction,
+ PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
CodeGenOpts.DebugInfoForProfiling, true);
else if (CodeGenOpts.DebugInfoForProfiling)
// -fdebug-info-for-profiling
PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction,
- PGOOptions::NoCSAction, true);
+ PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
+ true);
// Check to see if we want to generate a CS profile.
if (CodeGenOpts.hasProfileCSIRInstr()) {
@@ -799,13 +801,13 @@
: CodeGenOpts.InstrProfileOutput;
PGOOpt->CSAction = PGOOptions::CSIRInstr;
} else
- PGOOpt =
- PGOOptions("",
- CodeGenOpts.InstrProfileOutput.empty()
- ? getDefaultProfileGenName()
- : CodeGenOpts.InstrProfileOutput,
- "", nullptr, PGOOptions::NoAction, PGOOptions::CSIRInstr,
- CodeGenOpts.DebugInfoForProfiling);
+ PGOOpt = PGOOptions("",
+ CodeGenOpts.InstrProfileOutput.empty()
+ ? getDefaultProfileGenName()
+ : CodeGenOpts.InstrProfileOutput,
+ "", nullptr, PGOOptions::NoAction,
+ PGOOptions::CSIRInstr, PGOOptions::ColdFuncAttr::None,
+ CodeGenOpts.DebugInfoForProfiling);
}
if (TM)
TM->setPGOOption(PGOOpt);
diff --git a/llvm/include/llvm/Support/PGOOptions.h b/llvm/include/llvm/Support/PGOOptions.h
--- a/llvm/include/llvm/Support/PGOOptions.h
+++ b/llvm/include/llvm/Support/PGOOptions.h
@@ -27,10 +27,12 @@
struct PGOOptions {
enum PGOAction { NoAction, IRInstr, IRUse, SampleUse };
enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse };
+ enum class ColdFuncAttr { None, OptSize, MinSize, OptNone };
PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile,
IntrusiveRefCntPtr<vfs::FileSystem> FS,
PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction,
+ ColdFuncAttr ColdType = ColdFuncAttr::None,
bool DebugInfoForProfiling = false,
bool PseudoProbeForProfiling = false);
PGOOptions(const PGOOptions &);
@@ -42,6 +44,7 @@
std::string ProfileRemappingFile;
PGOAction Action;
CSPGOAction CSAction;
+ ColdFuncAttr ColdType;
bool DebugInfoForProfiling;
bool PseudoProbeForProfiling;
IntrusiveRefCntPtr<vfs::FileSystem> FS;
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/Support/PGOOptions.h"
#include <cstdint>
#include <string>
@@ -62,9 +63,11 @@
/// The profile annotation (profile-instr-use) pass for IR based PGO.
class PGOInstrumentationUse : public PassInfoMixin<PGOInstrumentationUse> {
public:
- PGOInstrumentationUse(std::string Filename = "",
- std::string RemappingFilename = "", bool IsCS = false,
- IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr);
+ PGOInstrumentationUse(
+ std::string Filename = "", std::string RemappingFilename = "",
+ bool IsCS = false,
+ PGOOptions::ColdFuncAttr ColdType = PGOOptions::ColdFuncAttr::None,
+ IntrusiveRefCntPtr<vfs::FileSystem> FS = nullptr);
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
@@ -73,6 +76,7 @@
std::string ProfileRemappingFileName;
// If this is a context sensitive instrumentation.
bool IsCS;
+ PGOOptions::ColdFuncAttr ColdType;
IntrusiveRefCntPtr<vfs::FileSystem> FS;
};
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -237,19 +237,23 @@
std::optional<PGOOptions> PGOOpt;
if (!Conf.SampleProfile.empty())
PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping, FS,
- PGOOptions::SampleUse, PGOOptions::NoCSAction, true);
+ PGOOptions::SampleUse, PGOOptions::NoCSAction,
+ PGOOptions::ColdFuncAttr::None, true);
else if (Conf.RunCSIRInstr) {
- PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping, FS,
- PGOOptions::IRUse, PGOOptions::CSIRInstr,
- Conf.AddFSDiscriminator);
+ PGOOpt =
+ PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping, FS,
+ PGOOptions::IRUse, PGOOptions::CSIRInstr,
+ PGOOptions::ColdFuncAttr::None, Conf.AddFSDiscriminator);
} else if (!Conf.CSIRProfile.empty()) {
- PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping, FS,
- PGOOptions::IRUse, PGOOptions::CSIRUse,
- Conf.AddFSDiscriminator);
+ PGOOpt =
+ PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping, FS,
+ PGOOptions::IRUse, PGOOptions::CSIRUse,
+ PGOOptions::ColdFuncAttr::None, Conf.AddFSDiscriminator);
NoPGOWarnMismatch = !Conf.PGOWarnMismatch;
} else if (Conf.AddFSDiscriminator) {
PGOOpt = PGOOptions("", "", "", nullptr, PGOOptions::NoAction,
- PGOOptions::NoCSAction, true);
+ PGOOptions::NoCSAction, PGOOptions::ColdFuncAttr::None,
+ true);
}
TM->setPGOOption(PGOOpt);
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -766,8 +766,8 @@
if (!RunProfileGen) {
assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
- MPM.addPass(
- PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
+ MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS,
+ PGOOpt->ColdType, FS));
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
// RequireAnalysisPass for PSI before subsequent non-module passes.
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
@@ -803,8 +803,8 @@
IntrusiveRefCntPtr<vfs::FileSystem> FS) {
if (!RunProfileGen) {
assert(!ProfileFile.empty() && "Profile use expecting a profile file!");
- MPM.addPass(
- PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS, FS));
+ MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS,
+ PGOOpt->ColdType, FS));
// Cache ProfileSummaryAnalysis once to avoid the potential need to insert
// RequireAnalysisPass for PSI before subsequent non-module passes.
MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -90,7 +90,7 @@
MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation())
MODULE_PASS("pgo-icall-prom", PGOIndirectCallPromotion())
MODULE_PASS("pgo-instr-gen", PGOInstrumentationGen())
-MODULE_PASS("pgo-instr-use", PGOInstrumentationUse())
+MODULE_PASS("pgo-instr-use", PGOInstrumentationUse("", "", false, PGOOpt ? PGOOpt->ColdType : PGOOptions::ColdFuncAttr::None))
MODULE_PASS("print-profile-summary", ProfileSummaryPrinterPass(dbgs()))
MODULE_PASS("print-callgraph", CallGraphPrinterPass(dbgs()))
MODULE_PASS("print-callgraph-sccs", CallGraphSCCsPrinterPass(dbgs()))
diff --git a/llvm/lib/Support/PGOOptions.cpp b/llvm/lib/Support/PGOOptions.cpp
--- a/llvm/lib/Support/PGOOptions.cpp
+++ b/llvm/lib/Support/PGOOptions.cpp
@@ -14,11 +14,11 @@
PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile,
std::string ProfileRemappingFile,
IntrusiveRefCntPtr<vfs::FileSystem> FS, PGOAction Action,
- CSPGOAction CSAction, bool DebugInfoForProfiling,
- bool PseudoProbeForProfiling)
+ CSPGOAction CSAction, ColdFuncAttr ColdType,
+ bool DebugInfoForProfiling, bool PseudoProbeForProfiling)
: ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
ProfileRemappingFile(ProfileRemappingFile), Action(Action),
- CSAction(CSAction),
+ CSAction(CSAction), ColdType(ColdType),
DebugInfoForProfiling(DebugInfoForProfiling ||
(Action == SampleUse && !PseudoProbeForProfiling)),
PseudoProbeForProfiling(PseudoProbeForProfiling), FS(std::move(FS)) {
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -2237,7 +2237,7 @@
function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
- ProfileSummaryInfo *PSI, bool IsCS) {
+ ProfileSummaryInfo *PSI, bool IsCS, PGOOptions::ColdFuncAttr ColdType) {
LLVM_DEBUG(dbgs() << "Read in profile counters: ");
auto &Ctx = M.getContext();
// Read the counter array from file.
@@ -2420,6 +2420,33 @@
continue;
}
F->addFnAttr(Attribute::Cold);
+ // Add optsize/minsize/optnone if requested.
+ switch (ColdType) {
+ case PGOOptions::ColdFuncAttr::None:
+ break;
+ case PGOOptions::ColdFuncAttr::OptSize:
+ if (!F->hasFnAttribute(Attribute::OptimizeNone) &&
+ !F->hasFnAttribute(Attribute::OptimizeForSize) &&
+ !F->hasFnAttribute(Attribute::MinSize)) {
+ F->addFnAttr(Attribute::OptimizeForSize);
+ }
+ break;
+ case PGOOptions::ColdFuncAttr::MinSize:
+ // Change optsize to minsize.
+ if (!F->hasFnAttribute(Attribute::OptimizeNone) &&
+ !F->hasFnAttribute(Attribute::MinSize)) {
+ F->removeFnAttr(Attribute::OptimizeForSize);
+ F->addFnAttr(Attribute::MinSize);
+ }
+ break;
+ case PGOOptions::ColdFuncAttr::OptNone:
+ // Strip optsize/minsize.
+ F->removeFnAttr(Attribute::OptimizeForSize);
+ F->removeFnAttr(Attribute::MinSize);
+ F->addFnAttr(Attribute::OptimizeNone);
+ F->addFnAttr(Attribute::NoInline);
+ break;
+ }
LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()
<< "\n");
}
@@ -2428,10 +2455,10 @@
PGOInstrumentationUse::PGOInstrumentationUse(
std::string Filename, std::string RemappingFilename, bool IsCS,
- IntrusiveRefCntPtr<vfs::FileSystem> VFS)
+ PGOOptions::ColdFuncAttr ColdType, IntrusiveRefCntPtr<vfs::FileSystem> VFS)
: ProfileFileName(std::move(Filename)),
ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS),
- FS(std::move(VFS)) {
+ ColdType(ColdType), FS(std::move(VFS)) {
if (!PGOTestProfileFile.empty())
ProfileFileName = PGOTestProfileFile;
if (!PGOTestProfileRemappingFile.empty())
@@ -2457,7 +2484,8 @@
auto *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName, *FS,
- LookupTLI, LookupBPI, LookupBFI, PSI, IsCS))
+ LookupTLI, LookupBPI, LookupBFI, PSI, IsCS,
+ ColdType))
return PreservedAnalyses::all();
return PreservedAnalyses::none();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext b/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext
new file mode 100644
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/Inputs/cold-func-attr.proftext
@@ -0,0 +1,42 @@
+# IR level Instrumentation Flag
+:ir
+:entry_first
+hot
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+9000
+
+cold
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+10
+
+cold1
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+10
+
+cold2
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+10
+
+cold3
+# Func Hash:
+742261418966908927
+# Num Counters:
+1
+# Counter Values:
+10
diff --git a/llvm/test/Transforms/PGOProfile/cold-func-attr.ll b/llvm/test/Transforms/PGOProfile/cold-func-attr.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/cold-func-attr.ll
@@ -0,0 +1,62 @@
+; RUN: llvm-profdata merge %S/Inputs/cold-func-attr.proftext -o %t.profdata
+; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=none | FileCheck %s --check-prefixes=NONE
+; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=optsize | FileCheck %s --check-prefixes=OPTSIZE,CHECK
+; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=minsize | FileCheck %s --check-prefixes=MINSIZE,CHECK
+; RUN: opt < %s -passes=pgo-instr-use -pgo-kind=pgo-instr-use-pipeline -pgo-test-profile-file=%t.profdata -S -pgo-cold-func-attr=optnone | FileCheck %s --check-prefixes=OPTNONE,CHECK
+
+; NONE: Function Attrs: cold{{$}}
+; OPTSIZE: Function Attrs: cold optsize{{$}}
+; MINSIZE: Function Attrs: cold minsize{{$}}
+; OPTNONE: Function Attrs: cold noinline optnone{{$}}
+; CHECK-NEXT: define void @cold()
+
+; NONE: Function Attrs: cold optsize{{$}}
+; OPTSIZE: Function Attrs: cold optsize{{$}}
+; MINSIZE: Function Attrs: cold minsize{{$}}
+; OPTNONE: Function Attrs: cold noinline optnone{{$}}
+; CHECK-NEXT: define void @cold1()
+
+; NONE: Function Attrs: cold minsize{{$}}
+; OPTSIZE: Function Attrs: cold minsize{{$}}
+; MINSIZE: Function Attrs: cold minsize{{$}}
+; OPTNONE: Function Attrs: cold noinline optnone{{$}}
+; CHECK-NEXT: define void @cold2()
+
+; CHECK: Function Attrs: cold noinline optnone{{$}}
+; CHECK-NEXT: define void @cold3()
+
+; CHECK-NOT: Function Attrs: {{.*}}optsize
+; CHECK-NOT: Function Attrs: {{.*}}minsize
+; CHECK-NOT: Function Attrs: {{.*}}optnone
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@s = common dso_local local_unnamed_addr global i32 0, align 4
+
+define void @cold() {
+ store i32 1, ptr @s, align 4
+ ret void
+}
+
+define void @cold1() optsize {
+ store i32 1, ptr @s, align 4
+ ret void
+}
+
+define void @cold2() minsize {
+ store i32 1, ptr @s, align 4
+ ret void
+}
+
+define void @cold3() optnone noinline {
+ store i32 1, ptr @s, align 4
+ ret void
+}
+
+define void @hot() {
+ %l = load i32, ptr @s, align 4
+ %add = add nsw i32 %l, 4
+ store i32 %add, ptr @s, align 4
+ ret void
+}
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -199,6 +199,18 @@
cl::desc("Path to the profile remapping file."),
cl::Hidden);
+static cl::opt<PGOOptions::ColdFuncAttr> PGOColdFuncAttr(
+ "pgo-cold-func-attr", cl::init(PGOOptions::ColdFuncAttr::None), cl::Hidden,
+ cl::desc(
+ "Function attribute to apply to cold functions as determined by PGO"),
+ cl::values(clEnumValN(PGOOptions::ColdFuncAttr::None, "none", "None"),
+ clEnumValN(PGOOptions::ColdFuncAttr::OptSize, "optsize",
+ "Mark cold functions with optsize."),
+ clEnumValN(PGOOptions::ColdFuncAttr::MinSize, "minsize",
+ "Mark cold functions with minsize."),
+ clEnumValN(PGOOptions::ColdFuncAttr::OptNone, "optnone",
+ "Mark cold functions with optnone.")));
+
static cl::opt<bool> DebugInfoForProfiling(
"debug-info-for-profiling", cl::init(false), cl::Hidden,
cl::desc("Emit special debug info to enable PGO profile generation."));
@@ -338,21 +350,23 @@
std::optional<PGOOptions> P;
switch (PGOKindFlag) {
case InstrGen:
- P = PGOOptions(ProfileFile, "", "", FS, PGOOptions::IRInstr);
+ P = PGOOptions(ProfileFile, "", "", FS, PGOOptions::IRInstr,
+ PGOOptions::NoCSAction, PGOColdFuncAttr);
break;
case InstrUse:
- P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS,
- PGOOptions::IRUse);
+ P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS, PGOOptions::IRUse,
+ PGOOptions::NoCSAction, PGOColdFuncAttr);
break;
case SampleUse:
P = PGOOptions(ProfileFile, "", ProfileRemappingFile, FS,
- PGOOptions::SampleUse);
+ PGOOptions::SampleUse, PGOOptions::NoCSAction,
+ PGOColdFuncAttr);
break;
case NoPGO:
if (DebugInfoForProfiling || PseudoProbeForProfiling)
P = PGOOptions("", "", "", nullptr, PGOOptions::NoAction,
- PGOOptions::NoCSAction, DebugInfoForProfiling,
- PseudoProbeForProfiling);
+ PGOOptions::NoCSAction, PGOColdFuncAttr,
+ DebugInfoForProfiling, PseudoProbeForProfiling);
else
P = std::nullopt;
}
@@ -372,7 +386,8 @@
P->CSProfileGenFile = CSProfileGenFile;
} else
P = PGOOptions("", CSProfileGenFile, ProfileRemappingFile, FS,
- PGOOptions::NoAction, PGOOptions::CSIRInstr);
+ PGOOptions::NoAction, PGOOptions::CSIRInstr,
+ PGOColdFuncAttr);
} else /* CSPGOKindFlag == CSInstrUse */ {
if (!P) {
errs() << "CSInstrUse needs to be together with InstrUse";