diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp --- a/bolt/lib/Profile/DataAggregator.cpp +++ b/bolt/lib/Profile/DataAggregator.cpp @@ -1296,37 +1296,55 @@ uint64_t NumTotalSamples = 0; while (hasData()) { - ErrorOr SampleRes = parseBranchSample(); - if (std::error_code EC = SampleRes.getError()) { - if (EC == errc::no_such_process) - continue; - return EC; - } - - PerfBranchSample &Sample = SampleRes.get(); + if (opts::BasicAggregation) { + ErrorOr SampleRes = parseBasicSample(); + if (std::error_code EC = SampleRes.getError()) { + if (EC == errc::no_such_process) + continue; + return EC; + } + PerfBasicSample &Sample = SampleRes.get(); + HM.registerAddress(Sample.PC); + NumTotalSamples++; + } else { + ErrorOr SampleRes = parseBranchSample(); + if (std::error_code EC = SampleRes.getError()) { + if (EC == errc::no_such_process) + continue; + return EC; + } - // LBRs are stored in reverse execution order. NextLBR refers to the next - // executed branch record. - const LBREntry *NextLBR = nullptr; - for (const LBREntry &LBR : Sample.LBR) { - if (NextLBR) { - // Record fall-through trace. - const uint64_t TraceFrom = LBR.To; - const uint64_t TraceTo = NextLBR->From; - ++FallthroughLBRs[Trace(TraceFrom, TraceTo)].InternCount; + PerfBranchSample &Sample = SampleRes.get(); + + // LBRs are stored in reverse execution order. NextLBR refers to the next + // executed branch record. + const LBREntry *NextLBR = nullptr; + for (const LBREntry &LBR : Sample.LBR) { + if (NextLBR) { + // Record fall-through trace. + const uint64_t TraceFrom = LBR.To; + const uint64_t TraceTo = NextLBR->From; + ++FallthroughLBRs[Trace(TraceFrom, TraceTo)].InternCount; + } + NextLBR = &LBR; } - NextLBR = &LBR; - } - if (!Sample.LBR.empty()) { - HM.registerAddress(Sample.LBR.front().To); - HM.registerAddress(Sample.LBR.back().From); + if (!Sample.LBR.empty()) { + HM.registerAddress(Sample.LBR.front().To); + HM.registerAddress(Sample.LBR.back().From); + } + NumTotalSamples += Sample.LBR.size(); } - NumTotalSamples += Sample.LBR.size(); } if (!NumTotalSamples) { - errs() << "HEATMAP-ERROR: no LBR traces detected in profile. " - "Cannot build heatmap.\n"; + if (!opts::BasicAggregation) { + errs() << "HEATMAP-ERROR: no LBR traces detected in profile. " + "Cannot build heatmap. Use -nl for building heatmap from " + "basic events.\n"; + } else { + errs() << "HEATMAP-ERROR: no samples detected in profile. " + "Cannot build heatmap." + } exit(1); } diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp --- a/lld/ELF/LinkerScript.cpp +++ b/lld/ELF/LinkerScript.cpp @@ -93,8 +93,9 @@ // conservative decisions at the section granularity. if (isSectionPrefix(".text", s->name)) { if (config->zKeepTextSectionPrefix) - for (StringRef v : {".text.hot", ".text.unknown", ".text.unlikely", - ".text.startup", ".text.exit", ".text.split"}) + for (StringRef v : + {".text.hot", ".text.unknown", ".text.unlikely", ".text.startup", + ".text.exit", ".text.split", ".text.propeller"}) if (isSectionPrefix(v.substr(5), s->name.substr(5))) return v; return ".text"; diff --git a/llvm/include/llvm/Analysis/BasicBlockSectionsInfo.h b/llvm/include/llvm/Analysis/BasicBlockSectionsInfo.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/Analysis/BasicBlockSectionsInfo.h @@ -0,0 +1,144 @@ +//===-- BasicBlockSections.cpp ---=========--------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// BasicBlockSections implementation. +// +// The purpose of this pass is to assign sections to basic blocks when +// -fbasic-block-sections= option is used. Further, with profile information +// only the subset of basic blocks with profiles are placed in separate sections +// and the rest are grouped in a cold section. The exception handling blocks are +// treated specially to ensure they are all in one seciton. +// +// Basic Block Sections +// ==================== +// +// With option, -fbasic-block-sections=list, every function may be split into +// clusters of basic blocks. Every cluster will be emitted into a separate +// section with its basic blocks sequenced in the given order. To get the +// optimized performance, the clusters must form an optimal BB layout for the +// function. Every cluster's section is labeled with a symbol to allow the +// linker to reorder the sections in any arbitrary sequence. A global order of +// these sections would encapsulate the function layout. +// +// There are a couple of challenges to be addressed: +// +// 1. The last basic block of every cluster should not have any implicit +// fallthrough to its next basic block, as it can be reordered by the linker. +// The compiler should make these fallthroughs explicit by adding +// unconditional jumps.. +// +// 2. All inter-cluster branch targets would now need to be resolved by the +// linker as they cannot be calculated during compile time. This is done +// using static relocations. Further, the compiler tries to use short branch +// instructions on some ISAs for small branch offsets. This is not possible +// for inter-cluster branches as the offset is not determined at compile +// time, and therefore, long branch instructions have to be used for those. +// +// 3. Debug Information (DebugInfo) and Call Frame Information (CFI) emission +// needs special handling with basic block sections. DebugInfo needs to be +// emitted with more relocations as basic block sections can break a +// function into potentially several disjoint pieces, and CFI needs to be +// emitted per cluster. This also bloats the object file and binary sizes. +// +// Basic Block Labels +// ================== +// +// With -fbasic-block-sections=labels, we emit the offsets of BB addresses of +// every function into the .llvm_bb_addr_map section. Along with the function +// symbols, this allows for mapping of virtual addresses in PMU profiles back to +// the corresponding basic blocks. This logic is implemented in AsmPrinter. This +// pass only assigns the BBSectionType of every function to ``labels``. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" + +using llvm::SmallSet; +using llvm::SmallVector; +using llvm::StringMap; +using llvm::StringRef; +using namespace llvm; + +namespace llvm { + +// This struct represents the cluster information for a machine basic block. +struct BBClusterInfo { + // MachineBasicBlock ID. + unsigned MBBNumber; + // Cluster ID this basic block belongs to. + unsigned ClusterID; + // Position of basic block within the cluster. + unsigned PositionInCluster; +}; + +using ProgramBBClusterInfoMapTy = StringMap>; + +class BasicBlockSectionsInfo : public ImmutablePass { +public: + static char ID; + + // This contains the basic-block-sections profile. + const MemoryBuffer *MBuf = nullptr; + + // This encapsulates the BB cluster information for the whole program. + // + // For every function name, it contains the cluster information for (all or + // some of) its basic blocks. The cluster information for every basic block + // includes its cluster ID along with the position of the basic block in that + // cluster. + ProgramBBClusterInfoMapTy ProgramBBClusterInfo; + + // Some functions have alias names. We use this map to find the main alias + // name for which we have mapping in ProgramBBClusterInfo. + StringMap FuncAliasMap; + + BasicBlockSectionsInfo(const MemoryBuffer *Buf) + : ImmutablePass(ID), MBuf(Buf) { + initializeBasicBlockSectionsInfoPass(*PassRegistry::getPassRegistry()); + }; + + BasicBlockSectionsInfo() : ImmutablePass(ID) { + initializeBasicBlockSectionsInfoPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "Basic Block Sections Info Analysis"; + } + + StringRef getAliasName(StringRef FuncName) const { + auto R = FuncAliasMap.find(FuncName); + return R == FuncAliasMap.end() ? FuncName : R->second; + } + + std::pair> + getBBClusterInfoForFunction(StringRef FuncName) const { + std::pair> cluster_info(false, {}); + auto R = ProgramBBClusterInfo.find(getAliasName(FuncName)); + if (R != ProgramBBClusterInfo.end()) { + cluster_info.second = R->second; + cluster_info.first = true; + } + return cluster_info; + } + + /// Read profiles of basic blocks if available here. + void initializePass() override; +}; + +ImmutablePass *createBasicBlockSectionsInfoPass(const MemoryBuffer *Buf); + +} // namespace llvm diff --git a/llvm/include/llvm/Analysis/Passes.h b/llvm/include/llvm/Analysis/Passes.h --- a/llvm/include/llvm/Analysis/Passes.h +++ b/llvm/include/llvm/Analysis/Passes.h @@ -13,6 +13,7 @@ #ifndef LLVM_ANALYSIS_PASSES_H #define LLVM_ANALYSIS_PASSES_H +#include "llvm/Support/MemoryBuffer.h" namespace llvm { class FunctionPass; diff --git a/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h b/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h --- a/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h +++ b/llvm/include/llvm/CodeGen/BasicBlockSectionUtils.h @@ -15,6 +15,7 @@ namespace llvm { extern cl::opt BBSectionsColdTextPrefix; +extern cl::opt BBSectionsSectionPrefix; class MachineFunction; class MachineBasicBlock; diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -54,7 +54,7 @@ /// basic blocks and is enabled with -fbasic-block-sections. Buf is a memory /// buffer that contains the list of functions and basic block ids to /// selectively enable basic block sections. - MachineFunctionPass *createBasicBlockSectionsPass(const MemoryBuffer *Buf); + MachineFunctionPass *createBasicBlockSectionsPass(); /// createMachineFunctionSplitterPass - This pass splits machine functions /// using profile information. diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -82,6 +82,7 @@ void initializeAtomicExpandPass(PassRegistry&); void initializeAttributorLegacyPassPass(PassRegistry&); void initializeAttributorCGSCCLegacyPassPass(PassRegistry &); +void initializeBasicBlockSectionsInfoPass(PassRegistry &); void initializeBasicBlockSectionsPass(PassRegistry &); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); diff --git a/llvm/lib/Analysis/BasicBlockSectionsInfo.cpp b/llvm/lib/Analysis/BasicBlockSectionsInfo.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Analysis/BasicBlockSectionsInfo.cpp @@ -0,0 +1,177 @@ +//===-- BasicBlockSections.cpp ---=========--------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// BasicBlockSections implementation. +// +// The purpose of this pass is to assign sections to basic blocks when +// -fbasic-block-sections= option is used. Further, with profile information +// only the subset of basic blocks with profiles are placed in separate sections +// and the rest are grouped in a cold section. The exception handling blocks are +// treated specially to ensure they are all in one seciton. +// +// Basic Block Sections +// ==================== +// +// With option, -fbasic-block-sections=list, every function may be split into +// clusters of basic blocks. Every cluster will be emitted into a separate +// section with its basic blocks sequenced in the given order. To get the +// optimized performance, the clusters must form an optimal BB layout for the +// function. Every cluster's section is labeled with a symbol to allow the +// linker to reorder the sections in any arbitrary sequence. A global order of +// these sections would encapsulate the function layout. +// +// There are a couple of challenges to be addressed: +// +// 1. The last basic block of every cluster should not have any implicit +// fallthrough to its next basic block, as it can be reordered by the linker. +// The compiler should make these fallthroughs explicit by adding +// unconditional jumps.. +// +// 2. All inter-cluster branch targets would now need to be resolved by the +// linker as they cannot be calculated during compile time. This is done +// using static relocations. Further, the compiler tries to use short branch +// instructions on some ISAs for small branch offsets. This is not possible +// for inter-cluster branches as the offset is not determined at compile +// time, and therefore, long branch instructions have to be used for those. +// +// 3. Debug Information (DebugInfo) and Call Frame Information (CFI) emission +// needs special handling with basic block sections. DebugInfo needs to be +// emitted with more relocations as basic block sections can break a +// function into potentially several disjoint pieces, and CFI needs to be +// emitted per cluster. This also bloats the object file and binary sizes. +// +// Basic Block Labels +// ================== +// +// With -fbasic-block-sections=labels, we emit the offsets of BB addresses of +// every function into the .llvm_bb_addr_map section. Along with the function +// symbols, this allows for mapping of virtual addresses in PMU profiles back to +// the corresponding basic blocks. This logic is implemented in AsmPrinter. This +// pass only assigns the BBSectionType of every function to ``labels``. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/BasicBlockSectionsInfo.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" + +using llvm::SmallSet; +using llvm::SmallVector; +using llvm::StringMap; +using llvm::StringRef; +using namespace llvm; + +char BasicBlockSectionsInfo::ID = 0; +INITIALIZE_PASS(BasicBlockSectionsInfo, "bbsections-info-prepare", + "Prepares for basic block sections, by splitting functions " + "into clusters of basic blocks.", + false, false) + +// Basic Block Sections can be enabled for a subset of machine basic blocks. +// This is done by passing a file containing names of functions for which basic +// block sections are desired. Additionally, machine basic block ids of the +// functions can also be specified for a finer granularity. Moreover, a cluster +// of basic blocks could be assigned to the same section. +// A file with basic block sections for all of function main and three blocks +// for function foo (of which 1 and 2 are placed in a cluster) looks like this: +// ---------------------------- +// list.txt: +// !main +// !foo +// !!1 2 +// !!4 +static Error getBBClusterInfo(const MemoryBuffer *MBuf, + ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, + StringMap &FuncAliasMap) { + assert(MBuf); + line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); + + auto invalidProfileError = [&](auto Message) { + return make_error( + Twine("Invalid profile " + MBuf->getBufferIdentifier() + " at line " + + Twine(LineIt.line_number()) + ": " + Message), + inconvertibleErrorCode()); + }; + + auto FI = ProgramBBClusterInfo.end(); + + // Current cluster ID corresponding to this function. + unsigned CurrentCluster = 0; + // Current position in the current cluster. + unsigned CurrentPosition = 0; + + // Temporary set to ensure every basic block ID appears once in the clusters + // of a function. + SmallSet FuncBBIDs; + + for (; !LineIt.is_at_eof(); ++LineIt) { + StringRef S(*LineIt); + if (S[0] == '@') + continue; + // Check for the leading "!" + if (!S.consume_front("!") || S.empty()) + break; + // Check for second "!" which indicates a cluster of basic blocks. + if (S.consume_front("!")) { + if (FI == ProgramBBClusterInfo.end()) + return invalidProfileError( + "Cluster list does not follow a function name specifier."); + SmallVector BBIndexes; + S.split(BBIndexes, ' '); + // Reset current cluster position. + CurrentPosition = 0; + for (auto BBIndexStr : BBIndexes) { + unsigned long long BBIndex; + if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex)) + return invalidProfileError(Twine("Unsigned integer expected: '") + + BBIndexStr + "'."); + if (!FuncBBIDs.insert(BBIndex).second) + return invalidProfileError(Twine("Duplicate basic block id found '") + + BBIndexStr + "'."); + if (!BBIndex && CurrentPosition) + return invalidProfileError("Entry BB (0) does not begin a cluster."); + + FI->second.emplace_back(BBClusterInfo{ + ((unsigned)BBIndex), CurrentCluster, CurrentPosition++}); + } + CurrentCluster++; + } else { // This is a function name specifier. + // Function aliases are separated using '/'. We use the first function + // name for the cluster info mapping and delegate all other aliases to + // this one. + SmallVector Aliases; + S.split(Aliases, '/'); + for (size_t i = 1; i < Aliases.size(); ++i) + FuncAliasMap.try_emplace(Aliases[i], Aliases.front()); + + // Prepare for parsing clusters of this function name. + // Start a new cluster map for this function name. + FI = ProgramBBClusterInfo.try_emplace(Aliases.front()).first; + CurrentCluster = 0; + FuncBBIDs.clear(); + } + } + return Error::success(); +} + +void BasicBlockSectionsInfo::initializePass() { + if (!MBuf) + return; + if (auto Err = getBBClusterInfo(MBuf, ProgramBBClusterInfo, FuncAliasMap)) + report_fatal_error(std::move(Err)); +} + +ImmutablePass *llvm::createBasicBlockSectionsInfoPass(const MemoryBuffer *Buf) { + return new BasicBlockSectionsInfo(Buf); +} diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt --- a/llvm/lib/Analysis/CMakeLists.txt +++ b/llvm/lib/Analysis/CMakeLists.txt @@ -34,6 +34,7 @@ AssumeBundleQueries.cpp AssumptionCache.cpp BasicAliasAnalysis.cpp + BasicBlockSectionsInfo.cpp BlockFrequencyInfo.cpp BlockFrequencyInfoImpl.cpp BranchProbabilityInfo.cpp diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -73,16 +73,15 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Analysis/BasicBlockSectionsInfo.h" #include "llvm/CodeGen/BasicBlockSectionUtils.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" #include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" -#include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetMachine.h" using llvm::SmallSet; @@ -108,41 +107,11 @@ namespace { -// This struct represents the cluster information for a machine basic block. -struct BBClusterInfo { - // MachineBasicBlock ID. - unsigned MBBNumber; - // Cluster ID this basic block belongs to. - unsigned ClusterID; - // Position of basic block within the cluster. - unsigned PositionInCluster; -}; - -using ProgramBBClusterInfoMapTy = StringMap>; - class BasicBlockSections : public MachineFunctionPass { public: static char ID; - // This contains the basic-block-sections profile. - const MemoryBuffer *MBuf = nullptr; - - // This encapsulates the BB cluster information for the whole program. - // - // For every function name, it contains the cluster information for (all or - // some of) its basic blocks. The cluster information for every basic block - // includes its cluster ID along with the position of the basic block in that - // cluster. - ProgramBBClusterInfoMapTy ProgramBBClusterInfo; - - // Some functions have alias names. We use this map to find the main alias - // name for which we have mapping in ProgramBBClusterInfo. - StringMap FuncAliasMap; - - BasicBlockSections(const MemoryBuffer *Buf) - : MachineFunctionPass(ID), MBuf(Buf) { - initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry()); - }; + BasicBlockSectionsInfo *BBSectionsInfo = nullptr; BasicBlockSections() : MachineFunctionPass(ID) { initializeBasicBlockSectionsPass(*PassRegistry::getPassRegistry()); @@ -154,9 +123,6 @@ void getAnalysisUsage(AnalysisUsage &AU) const override; - /// Read profiles of basic blocks if available here. - bool doInitialization(Module &M) override; - /// Identify basic blocks that need separate sections and prepare to emit them /// accordingly. bool runOnMachineFunction(MachineFunction &MF) override; @@ -204,39 +170,6 @@ } } -// This function provides the BBCluster information associated with a function. -// Returns true if a valid association exists and false otherwise. -static bool getBBClusterInfoForFunction( - const MachineFunction &MF, const StringMap FuncAliasMap, - const ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, - std::vector> &V) { - // Get the main alias name for the function. - auto FuncName = MF.getName(); - auto R = FuncAliasMap.find(FuncName); - StringRef AliasName = R == FuncAliasMap.end() ? FuncName : R->second; - - // Find the assoicated cluster information. - auto P = ProgramBBClusterInfo.find(AliasName); - if (P == ProgramBBClusterInfo.end()) - return false; - - if (P->second.empty()) { - // This indicates that sections are desired for all basic blocks of this - // function. We clear the BBClusterInfo vector to denote this. - V.clear(); - return true; - } - - V.resize(MF.getNumBlockIDs()); - for (auto bbClusterInfo : P->second) { - // Bail out if the cluster information contains invalid MBB numbers. - if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs()) - return false; - V[bbClusterInfo.MBBNumber] = bbClusterInfo; - } - return true; -} - // This function sorts basic blocks according to the cluster's information. // All explicitly specified clusters of basic blocks will be ordered // accordingly. All non-specified BBs go into a separate "Cold" section. @@ -351,6 +284,34 @@ return false; } +// This function provides the BBCluster information associated with a function. +// Returns true if a valid association exists and false otherwise. +bool getBBClusterInfoForFunction(const MachineFunction &MF, + BasicBlockSectionsInfo *BBSectionsInfo, + std::vector> &V) { + + // Find the assoicated cluster information. + auto P = BBSectionsInfo->getBBClusterInfoForFunction(MF.getName()); + if (!P.first) + return false; + + if (P.second.empty()) { + // This indicates that sections are desired for all basic blocks of this + // function. We clear the BBClusterInfo vector to denote this. + V.clear(); + return true; + } + + V.resize(MF.getNumBlockIDs()); + for (auto bbClusterInfo : P.second) { + // Bail out if the cluster information contains invalid MBB numbers. + if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs()) + return false; + V[bbClusterInfo.MBBNumber] = bbClusterInfo; + } + return true; +} + bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) { auto BBSectionsType = MF.getTarget().getBBSectionsType(); assert(BBSectionsType != BasicBlockSection::None && @@ -377,10 +338,11 @@ return true; } + BBSectionsInfo = &getAnalysis(); + std::vector> FuncBBClusterInfo; if (BBSectionsType == BasicBlockSection::List && - !getBBClusterInfoForFunction(MF, FuncAliasMap, ProgramBBClusterInfo, - FuncBBClusterInfo)) + !getBBClusterInfoForFunction(MF, BBSectionsInfo, FuncBBClusterInfo)) return true; MF.setBBSectionsType(BBSectionsType); assignSections(MF, FuncBBClusterInfo); @@ -427,107 +389,12 @@ return true; } -// Basic Block Sections can be enabled for a subset of machine basic blocks. -// This is done by passing a file containing names of functions for which basic -// block sections are desired. Additionally, machine basic block ids of the -// functions can also be specified for a finer granularity. Moreover, a cluster -// of basic blocks could be assigned to the same section. -// A file with basic block sections for all of function main and three blocks -// for function foo (of which 1 and 2 are placed in a cluster) looks like this: -// ---------------------------- -// list.txt: -// !main -// !foo -// !!1 2 -// !!4 -static Error getBBClusterInfo(const MemoryBuffer *MBuf, - ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, - StringMap &FuncAliasMap) { - assert(MBuf); - line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); - - auto invalidProfileError = [&](auto Message) { - return make_error( - Twine("Invalid profile " + MBuf->getBufferIdentifier() + " at line " + - Twine(LineIt.line_number()) + ": " + Message), - inconvertibleErrorCode()); - }; - - auto FI = ProgramBBClusterInfo.end(); - - // Current cluster ID corresponding to this function. - unsigned CurrentCluster = 0; - // Current position in the current cluster. - unsigned CurrentPosition = 0; - - // Temporary set to ensure every basic block ID appears once in the clusters - // of a function. - SmallSet FuncBBIDs; - - for (; !LineIt.is_at_eof(); ++LineIt) { - StringRef S(*LineIt); - if (S[0] == '@') - continue; - // Check for the leading "!" - if (!S.consume_front("!") || S.empty()) - break; - // Check for second "!" which indicates a cluster of basic blocks. - if (S.consume_front("!")) { - if (FI == ProgramBBClusterInfo.end()) - return invalidProfileError( - "Cluster list does not follow a function name specifier."); - SmallVector BBIndexes; - S.split(BBIndexes, ' '); - // Reset current cluster position. - CurrentPosition = 0; - for (auto BBIndexStr : BBIndexes) { - unsigned long long BBIndex; - if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex)) - return invalidProfileError(Twine("Unsigned integer expected: '") + - BBIndexStr + "'."); - if (!FuncBBIDs.insert(BBIndex).second) - return invalidProfileError(Twine("Duplicate basic block id found '") + - BBIndexStr + "'."); - if (!BBIndex && CurrentPosition) - return invalidProfileError("Entry BB (0) does not begin a cluster."); - - FI->second.emplace_back(BBClusterInfo{ - ((unsigned)BBIndex), CurrentCluster, CurrentPosition++}); - } - CurrentCluster++; - } else { // This is a function name specifier. - // Function aliases are separated using '/'. We use the first function - // name for the cluster info mapping and delegate all other aliases to - // this one. - SmallVector Aliases; - S.split(Aliases, '/'); - for (size_t i = 1; i < Aliases.size(); ++i) - FuncAliasMap.try_emplace(Aliases[i], Aliases.front()); - - // Prepare for parsing clusters of this function name. - // Start a new cluster map for this function name. - FI = ProgramBBClusterInfo.try_emplace(Aliases.front()).first; - CurrentCluster = 0; - FuncBBIDs.clear(); - } - } - return Error::success(); -} - -bool BasicBlockSections::doInitialization(Module &M) { - if (!MBuf) - return false; - if (auto Err = getBBClusterInfo(MBuf, ProgramBBClusterInfo, FuncAliasMap)) - report_fatal_error(std::move(Err)); - return false; -} - void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); + AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } -MachineFunctionPass * -llvm::createBasicBlockSectionsPass(const MemoryBuffer *Buf) { - return new BasicBlockSections(Buf); +MachineFunctionPass *llvm::createBasicBlockSectionsPass() { + return new BasicBlockSections(); } diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -21,6 +21,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/BasicBlockSectionsInfo.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/ConstantFolding.h" @@ -177,6 +178,11 @@ "profile-guided-section-prefix", cl::Hidden, cl::init(true), cl::ZeroOrMore, cl::desc("Use profile info to add section prefix for hot/cold functions")); +cl::opt BBSectionsSectionPrefix( + "bb-sections-section-prefix", cl::Hidden, cl::init("propeller"), + cl::ZeroOrMore, + cl::desc("Section prefix to use for hot propeller functions.")); + static cl::opt ProfileUnknownInSpecialSection( "profile-unknown-in-special-section", cl::Hidden, cl::init(false), cl::ZeroOrMore, @@ -274,6 +280,7 @@ const TargetLowering *TLI = nullptr; const TargetRegisterInfo *TRI; const TargetTransformInfo *TTI = nullptr; + const BasicBlockSectionsInfo *BBSectionsInfo = nullptr; const TargetLibraryInfo *TLInfo; const LoopInfo *LI; std::unique_ptr BFI; @@ -349,6 +356,7 @@ AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); } private: @@ -447,6 +455,7 @@ INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsInfo) INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE, "Optimize for code generation", false, false) @@ -473,13 +482,20 @@ BPI.reset(new BranchProbabilityInfo(F, *LI)); BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); PSI = &getAnalysis().getPSI(); + BBSectionsInfo = getAnalysisIfAvailable(); OptSize = F.hasOptSize(); if (ProfileGuidedSectionPrefix) { // The hot attribute overwrites profile count based hotness while profile // counts based hotness overwrite the cold attribute. // This is a conservative behabvior. - if (F.hasFnAttribute(Attribute::Hot) || - PSI->isFunctionHotInCallGraph(&F, *BFI)) + if (BBSectionsInfo->MBuf != nullptr && + BBSectionsInfo->getBBClusterInfoForFunction(F.getName()).first && + BBSectionsInfo->getBBClusterInfoForFunction(F.getName()) + .second.front() + .MBBNumber == 0) + F.setSectionPrefix(BBSectionsSectionPrefix); + else if (F.hasFnAttribute(Attribute::Hot) || + PSI->isFunctionHotInCallGraph(&F, *BFI)) F.setSectionPrefix("hot"); // If PSI shows this function is not hot, we will placed the function // into unlikely section if (1) PSI shows this is a cold function, or diff --git a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp --- a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp +++ b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp @@ -25,6 +25,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/BasicBlockSectionsInfo.h" #include "llvm/Analysis/ProfileSummaryInfo.h" #include "llvm/CodeGen/BasicBlockSectionUtils.h" #include "llvm/CodeGen/MachineBasicBlock.h" @@ -64,6 +65,7 @@ class MachineFunctionSplitter : public MachineFunctionPass { public: static char ID; + BasicBlockSectionsInfo *BBSectionsInfo = nullptr; MachineFunctionSplitter() : MachineFunctionPass(ID) { initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry()); } @@ -114,6 +116,15 @@ return false; } + BBSectionsInfo = getAnalysisIfAvailable(); + + // With -basic-block-sections we only proceed if the function is not in the + // propeller profile. + if (BBSectionsInfo->MBuf != nullptr && + BBSectionsInfo->getBBClusterInfoForFunction(MF.getFunction().getName()) + .first) + return false; + // Renumbering blocks here preserves the order of the blocks as // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort // blocks. Preserving the order of blocks is essential to retaining decisions @@ -154,6 +165,7 @@ } void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addUsedIfAvailable(); AU.addRequired(); AU.addRequired(); AU.addRequired(); diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -1012,7 +1012,7 @@ Name += ".text.eh."; Name += MBB.getParent()->getName(); } else { - Name += MBB.getParent()->getSection()->getName(); + Name += ".text.propeller"; if (TM.getUniqueBasicBlockSectionNames()) { if (!Name.endswith(".")) Name += "."; diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -16,6 +16,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/Analysis/BasicBlockSectionsInfo.h" #include "llvm/Analysis/CFLAndersAliasAnalysis.h" #include "llvm/Analysis/CFLSteensAliasAnalysis.h" #include "llvm/Analysis/CallGraphSCCPass.h" @@ -1260,12 +1261,16 @@ // cannot be enabled at the same time. Basic block sections takes precedence. // FIXME: In principle, BasicBlockSection::Labels and splitting can used // together. Update this check once we have addressed any issues. - if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) { - addPass(llvm::createBasicBlockSectionsPass(TM->getBBSectionsFuncListBuf())); - } else if (TM->Options.EnableMachineFunctionSplitter || - EnableMachineFunctionSplitter) { + if (TM->getBBSectionsType() != llvm::BasicBlockSection::Labels && + (TM->Options.EnableMachineFunctionSplitter || + EnableMachineFunctionSplitter)) { addPass(createMachineFunctionSplitterPass()); } + if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) { + addPass( + llvm::createBasicBlockSectionsInfoPass(TM->getBBSectionsFuncListBuf())); + addPass(llvm::createBasicBlockSectionsPass()); + } // Add passes that directly emit MI after all other MI passes. addPreEmitPass2();