Index: llvm/include/llvm/CodeGen/AsmPrinter.h =================================================================== --- llvm/include/llvm/CodeGen/AsmPrinter.h +++ llvm/include/llvm/CodeGen/AsmPrinter.h @@ -132,6 +132,10 @@ /// default, this is equal to CurrentFnSym. MCSymbol *CurrentFnSymForSize = nullptr; + // The symbol used to represent the start of the current BB section of the + // function. This is used to calculate the size of the BB section. + MCSymbol *CurrentSectionBeginSym = nullptr; + /// Map global GOT equivalent MCSymbols to GlobalVariables and keep track of /// its number of uses by other globals. using GOTEquivUsePair = std::pair; Index: llvm/include/llvm/CodeGen/MachineBasicBlock.h =================================================================== --- llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -46,19 +46,6 @@ class TargetRegisterClass; class TargetRegisterInfo; -enum MachineBasicBlockSection : unsigned { - /// This is also the order of sections in a function. Basic blocks that are - /// part of the original function section (entry block) come first, followed - /// by exception handling basic blocks, cold basic blocks and finally basic - // blocks that need unique sections. - MBBS_Entry, - MBBS_Exception, - MBBS_Cold, - MBBS_Unique, - /// None implies no sections for any basic block, the default. - MBBS_None, -}; - template <> struct ilist_traits { private: friend class MachineBasicBlock; // Set by the owning MachineBasicBlock. @@ -91,6 +78,10 @@ : PhysReg(PhysReg), LaneMask(LaneMask) {} }; + // Special section IDs for the cold and exception section + const static unsigned ColdSectionID = UINT_MAX; + const static unsigned ExceptionSectionID = UINT_MAX - 1; + private: using Instructions = ilist>; @@ -143,8 +134,8 @@ /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; - /// Stores the Section type of the basic block with basic block sections. - MachineBasicBlockSection SectionType = MBBS_None; + /// With basic block sections, this stores the Section ID of the basic block. + Optional SectionID; /// Default target of the callbr of a basic block. bool InlineAsmBrDefaultTarget = false; @@ -440,11 +431,11 @@ /// Returns true if this block ends any section. bool isEndSection() const; - /// Returns the type of section this basic block belongs to. - MachineBasicBlockSection getSectionType() const { return SectionType; } + /// Returns the section ID of this basic block. + Optional getSectionID() const { return SectionID; } /// Indicate that the basic block belongs to a Section Type. - void setSectionType(MachineBasicBlockSection V) { SectionType = V; } + void setSectionID(unsigned V) { SectionID = V; } /// Returns true if this is the indirect dest of an INLINEASM_BR. bool isInlineAsmBrIndirectTarget(const MachineBasicBlock *Tgt) const { Index: llvm/include/llvm/CodeGen/MachineFunction.h =================================================================== --- llvm/include/llvm/CodeGen/MachineFunction.h +++ llvm/include/llvm/CodeGen/MachineFunction.h @@ -346,11 +346,6 @@ /// Section Type for basic blocks, only relevant with basic block sections. BasicBlockSection BBSectionsType = BasicBlockSection::None; - /// With Basic Block Sections, this stores the bb ranges of cold and - /// exception sections. - std::pair ColdSectionRange = {-1, -1}; - std::pair ExceptionSectionRange = {-1, -1}; - /// List of C++ TypeInfo used. std::vector TypeInfos; @@ -505,19 +500,6 @@ void setBBSectionsType(BasicBlockSection V) { BBSectionsType = V; } - void setSectionRange(); - - /// Returns true if this basic block number starts a cold or exception - /// section. - bool isSectionStartMBB(int N) const { - return (N == ColdSectionRange.first || N == ExceptionSectionRange.first); - } - - /// Returns true if this basic block ends a cold or exception section. - bool isSectionEndMBB(int N) const { - return (N == ColdSectionRange.second || N == ExceptionSectionRange.second); - } - /// Creates basic block Labels for this function. void createBBLabels(); Index: llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1097,14 +1097,6 @@ // Print out code for the function. bool HasAnyRealCode = false; int NumInstsInFunction = 0; - bool emitBBSections = MF->hasBBSections(); - MachineBasicBlock *EndOfRegularSectionMBB = nullptr; - if (emitBBSections) { - EndOfRegularSectionMBB = - const_cast(MF->front().getSectionEndMBB()); - assert(EndOfRegularSectionMBB->isEndSection() && - "The MBB at the end of the regular section must end a section"); - } for (auto &MBB : *MF) { // Print a label for the basic block. @@ -1185,17 +1177,22 @@ } } } - if (&MBB != EndOfRegularSectionMBB && - (MF->hasBBLabels() || MBB.isEndSection())) { - // Emit size directive for the size of this basic block. Create a symbol - // for the end of the basic block. + + if (MF->hasBBLabels() || + (MBB.isEndSection() && !MBB.sameSection(&MF->front()))) { + // Emit size directive for the size of this basic block (or basic block + // section). Create a symbol for the end of the basic block. MCSymbol *CurrentBBEnd = OutContext.createTempSymbol(); + OutStreamer->emitLabel(CurrentBBEnd); + MCSymbol *SymForSize = + MBB.isEndSection() ? CurrentSectionBeginSym : MBB.getSymbol(); const MCExpr *SizeExp = MCBinaryExpr::createSub( MCSymbolRefExpr::create(CurrentBBEnd, OutContext), - MCSymbolRefExpr::create(MBB.getSymbol(), OutContext), OutContext); - OutStreamer->emitLabel(CurrentBBEnd); - MBB.setEndMCSymbol(CurrentBBEnd); - OutStreamer->emitELFSize(MBB.getSymbol(), SizeExp); + MCSymbolRefExpr::create(SymForSize, OutContext), OutContext); + OutStreamer->emitELFSize(SymForSize, SizeExp); + + if (MBB.isEndSection()) + MBB.setEndMCSymbol(CurrentBBEnd); } emitBasicBlockEnd(MBB); } @@ -1231,7 +1228,7 @@ } // Switch to the original section if basic block sections was used. - if (emitBBSections) + if (MF->hasBBSections()) OutStreamer->SwitchSection(MF->getSection()); const Function &F = MF->getFunction(); @@ -1249,7 +1246,7 @@ emitFunctionBodyEnd(); if (needFuncLabelsForEHOrDebugInfo(*MF, MMI) || - MAI->hasDotTypeDotSizeDirective() || emitBBSections) { + MAI->hasDotTypeDotSizeDirective() || MF->hasBBSections()) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->emitLabel(CurrentFnEnd); @@ -1272,8 +1269,9 @@ HI.Handler->markFunctionEnd(); } - if (emitBBSections) - EndOfRegularSectionMBB->setEndMCSymbol(CurrentFnEnd); + if (MF->hasBBSections()) + (const_cast(MF->front().getSectionEndMBB())) + ->setEndMCSymbol(CurrentFnEnd); // Print out jump tables referenced by the function. emitJumpTableInfo(); @@ -1758,6 +1756,7 @@ CurrentFnSymForSize = CurrentFnSym; CurrentFnBegin = nullptr; + CurrentSectionBeginSym = nullptr; CurExceptionSym = nullptr; bool NeedsLocalForSize = MAI->needsLocalForSize(); if (F.hasFnAttribute("patchable-function-entry") || @@ -2995,7 +2994,6 @@ /// MachineBasicBlock, an alignment (if present) and a comment describing /// it if appropriate. void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { - bool BBSections = MF->hasBBSections(); // End the previous funclet and start a new one. if (MBB.isEHFuncletEntry()) { for (const HandlerInfo &HI : Handlers) { @@ -3005,7 +3003,7 @@ } // Emit an alignment directive for this block, if needed. - if (MBB.pred_empty() || !BBSections) { + if (MBB.pred_empty() || !MF->hasBBSections()) { const Align Alignment = MBB.getAlignment(); if (Alignment != Align(1)) emitAlignment(Alignment); @@ -3041,7 +3039,8 @@ emitBasicBlockLoopComments(MBB, MLI, *this); } - bool emitBBLabels = BBSections || MF->hasBBLabels(); + bool emitBBLabels = MF->hasBBSections() || MF->hasBBLabels(); + if (MBB.pred_empty() || (!emitBBLabels && isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry() && !MBB.hasLabelMustBeEmitted())) { @@ -3054,24 +3053,25 @@ if (isVerbose() && MBB.hasLabelMustBeEmitted()) { OutStreamer->AddComment("Label of block must be emitted"); } - // With -fbasicblock-sections, a basic block can start a new section. - if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Exception) { - // Create the exception section for this function. - OutStreamer->SwitchSection( - getObjFileLowering().getNamedSectionForMachineBasicBlock( - MF->getFunction(), MBB, TM, ".eh")); - } else if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Cold) { - // Create the cold section here. - OutStreamer->SwitchSection( - getObjFileLowering().getNamedSectionForMachineBasicBlock( - MF->getFunction(), MBB, TM, ".unlikely")); - } else if (MBB.isBeginSection() && MBB.isEndSection()) { - OutStreamer->SwitchSection( - getObjFileLowering().getSectionForMachineBasicBlock(MF->getFunction(), - MBB, TM)); - } else if (BBSections) { - OutStreamer->SwitchSection(MF->getSection()); + if (MBB.isBeginSection()) { + // With -fbasicblock-sections, a basic block can start a new section. + if (MBB.getSectionID() == MachineBasicBlock::ExceptionSectionID) { + // Create the exception section for this function. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".eh")); + } else if (MBB.getSectionID() == MachineBasicBlock::ColdSectionID) { + // Create the cold section here. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".unlikely")); + } else + OutStreamer->SwitchSection( + getObjFileLowering().getSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM)); + CurrentSectionBeginSym = MBB.getSymbol(); } + OutStreamer->emitLabel(MBB.getSymbol()); } } Index: llvm/lib/CodeGen/BBSectionsPrepare.cpp =================================================================== --- llvm/lib/CodeGen/BBSectionsPrepare.cpp +++ llvm/lib/CodeGen/BBSectionsPrepare.cpp @@ -9,47 +9,41 @@ // BBSectionsPrepare implementation. // // The purpose of this pass is to assign sections to basic blocks when -// -fbasicblock-sections= option is used. Exception landing pad blocks are -// specially handled by grouping them in a single section. Further, with -// profile information only the subset of basic blocks with profiles are placed -// in a separate section and the rest are grouped in a cold section. +// -fbasicblock-sections= option is used. Further, with profile information only +// the subset of basic blocks with profiles are placed in separate sections and +// the rest are grouped in a cold section. The exception handling blocks are +// treated specially to ensure they are all in one seciton. // // Basic Block Sections // ==================== // -// With option, -fbasicblock-sections=, each basic block could be placed in a -// unique ELF text section in the object file along with a symbol labelling the -// basic block. The linker can then order the basic block sections in any -// arbitrary sequence which when done correctly can encapsulate block layout, -// function layout and function splitting optimizations. However, there are a -// couple of challenges to be addressed for this to be feasible: +// With option, -fbasicblock-sections=list, every function may be split into +// clusters of basic blocks. Every cluster will be emitted into a separate +// section with its basic blocks sequenced in the given order. To get the +// optimized performance, the clusters must form an optimal BB layout for the +// function. Every cluster's section is labeled with a symbol to allow the +// linker to reorder the sections in any arbitrary sequence. A global order of +// these sections would encapsulate the function layout. // -// 1. The compiler must not allow any implicit fall-through between any two -// adjacent basic blocks as they could be reordered at link time to be -// non-adjacent. In other words, the compiler must make a fall-through -// between adjacent basic blocks explicit by retaining the direct jump -// instruction that jumps to the next basic block. +// There are a couple of challenges to be addressed: // -// 2. All inter-basic block branch targets would now need to be resolved by the +// 1. The last basic block of every cluster should not have any implicit +// fallthrough to its next basic block, as it can be reordered by the linker. +// The compiler should make these fallthroughs explicit by adding +// unconditional jumps.. +// +// 2. All inter-cluster branch targets would now need to be resolved by the // linker as they cannot be calculated during compile time. This is done // using static relocations. Further, the compiler tries to use short branch // instructions on some ISAs for small branch offsets. This is not possible -// with basic block sections as the offset is not determined at compile time, -// and long branch instructions have to be used everywhere. -// -// 3. Each additional section bloats object file sizes by tens of bytes. The -// number of basic blocks can be potentially very large compared to the size -// of functions and can bloat object sizes significantly. Option -// fbasicblock-sections= also takes a file path which can be used to specify -// a subset of basic blocks that needs unique sections to keep the bloats -// small. +// for inter-cluster branches as the offset is not determined at compile +// time, and therefore, long branch instructions have to be used for those. // -// 4. Debug Information (DebugInfo) and Call Frame Information (CFI) emission +// 3. Debug Information (DebugInfo) and Call Frame Information (CFI) emission // needs special handling with basic block sections. DebugInfo needs to be // emitted with more relocations as basic block sections can break a // function into potentially several disjoint pieces, and CFI needs to be -// emitted per basic block. This also bloats the object file and binary -// sizes. +// emitted per cluster. This also bloats the object file and binary sizes. // // Basic Block Labels // ================== @@ -70,7 +64,9 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineFunction.h" @@ -79,34 +75,63 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" +#include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetMachine.h" +#include +#include #include using llvm::SmallSet; +using llvm::SmallVector; using llvm::StringMap; using llvm::StringRef; using namespace llvm; namespace { +// This struct represents the cluster information for a machine basic block. +struct BBClusterInfo { + // MachineBasicBlock ID. + unsigned MBBNumber; + // Cluster ID this basic block belongs to. + unsigned ClusterID; + // Position of basic block within the cluster. + unsigned PositionInCluster; +}; + +using ProgramBBClusterInfoMapTy = StringMap>; + class BBSectionsPrepare : public MachineFunctionPass { public: static char ID; - StringMap> BBSectionsList; + + // This contains the basic-block-sections profile. const MemoryBuffer *MBuf = nullptr; - BBSectionsPrepare() : MachineFunctionPass(ID) { - initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); - } + // This encapsulates the BB cluster information for the whole program. + // + // For every function name, it contains the cluster information for (all or + // some of) its basic blocks. The cluster information for every basic block + // includes its cluster ID along with the position of the basic block in that + // cluster. + ProgramBBClusterInfoMapTy ProgramBBClusterInfo; + + // Some functions have alias names. We use this map to find the main alias + // name for which we have mapping in ProgramBBClusterInfo. + StringMap FuncAliasMap; BBSectionsPrepare(const MemoryBuffer *Buf) : MachineFunctionPass(ID), MBuf(Buf) { initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); }; + BBSectionsPrepare() : MachineFunctionPass(ID) { + initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); + } + StringRef getPassName() const override { return "Basic Block Sections Analysis"; } @@ -125,104 +150,159 @@ char BBSectionsPrepare::ID = 0; INITIALIZE_PASS(BBSectionsPrepare, "bbsections-prepare", - "Determine if a basic block needs a special section", false, - false) - -// This inserts an unconditional branch at the end of MBB to the next basic -// block S if and only if the control-flow implicitly falls through from MBB to -// S and S and MBB belong to different sections. This is necessary with basic -// block sections as MBB and S could be potentially reordered. -static void insertUnconditionalFallthroughBranch(MachineBasicBlock &MBB) { - MachineBasicBlock *Fallthrough = MBB.getFallThrough(); + "Prepares for basic block sections, by splitting functions " + "into clusters of basic blocks.", + false, false) - if (Fallthrough == nullptr) - return; +// This function updates and optimizes the branching instructions of every basic +// block in a given function to accout for changes in the layout. +static void updateBranches( + MachineFunction &MF, + const SmallVector &PreLayoutFallThroughs) { + const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + SmallVector Cond; + for (auto &MBB : MF) { + auto NextMBBI = std::next(MBB.getIterator()); + auto *FTMBB = PreLayoutFallThroughs[MBB.getNumber()]; + // If this block had a fallthrough before we need an explicit unconditional + // branch to that block if either + // 1- the block ends a section, which means its next block may be + // reorderd by the linker, or + // 2- the fallthrough block is not adjacent to the block in the new + // order. + if (FTMBB && (MBB.isEndSection() || &*NextMBBI != FTMBB)) + TII->insertUnconditionalBranch(MBB, FTMBB, MBB.findBranchDebugLoc()); + + // We do not optimize branches for machine basic blocks ending sections, as + // their adjacent block might be reordered by the linker. + if (MBB.isEndSection()) + continue; - // If this basic block and the Fallthrough basic block are in the same - // section then do not insert the jump. - if (MBB.sameSection(Fallthrough)) - return; + // It might be possible to optimize branches by flipping the branch + // condition. + Cond.clear(); + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; // For analyzeBranch. + if (!TII->analyzeBranch(MBB, TBB, FBB, Cond)) + MBB.updateTerminator(); + } +} - const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); - SmallVector Cond; - MachineBasicBlock *TBB = nullptr, *FBB = nullptr; +// This function provides the BBCluster information associated with a function. +// Returns true if a valid association exists and false otherwise. +static bool getBBClusterInfoForFunction( + const MachineFunction &MF, const StringMap FuncAliasMap, + const ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, + std::vector> &V) { + // Get the main alias name for the function. + auto FuncName = MF.getName(); + auto R = FuncAliasMap.find(FuncName); + StringRef AliasName = R == FuncAliasMap.end() ? FuncName : R->second; + + // Find the assoicated cluster information. + auto P = ProgramBBClusterInfo.find(AliasName); + if (P == ProgramBBClusterInfo.end()) + return false; - // If a branch to the fall through block already exists, return. - if (!TII->analyzeBranch(MBB, TBB, FBB, Cond) && - (TBB == Fallthrough || FBB == Fallthrough)) { - return; + if (P->second.empty()) { + // This indicates that sections are desired for all basic blocks of this + // function. We clear the BBClusterInfo vector to denote this. + V.clear(); + return true; } - Cond.clear(); - DebugLoc DL = MBB.findBranchDebugLoc(); - TII->insertBranch(MBB, Fallthrough, nullptr, Cond, DL); + V.resize(MF.getNumBlockIDs()); + for (auto bbClusterInfo : P->second) { + // Bail out if the cluster information contains invalid MBB numbers. + if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs()) + return false; + V[bbClusterInfo.MBBNumber] = bbClusterInfo; + } + return true; } -/// This function sorts basic blocks according to the sections in which they are -/// emitted. Basic block sections automatically turn on function sections so -/// the entry block is in the function section. The other sections that are -/// created are: -/// 1) Exception section - basic blocks that are landing pads -/// 2) Cold section - basic blocks that will not have unique sections. -/// 3) Unique section - one per basic block that is emitted in a unique section. +// This function sorts basic blocks according to the cluster's information. +// All explicitly specified clusters of basic blocks will be ordered +// accordingly. All non-specified BBs go into a separate "Cold" section. +// Additionally, if exception handling landing pads end up in more than one +// clusters, they are moved into a single "Exception" section. Eventually, +// clusters are ordered in increasing order of their IDs, with the "Exception" +// and "Cold" succeeding all other clusters. +// FuncBBClusterInfo represent the cluster information for basic blocks. If this +// is empty, it means unique sections for all basic blocks in the function. static bool assignSectionsAndSortBasicBlocks( MachineFunction &MF, - const StringMap> &BBSectionsList) { - SmallSet S = BBSectionsList.lookup(MF.getName()); - - bool HasHotEHPads = false; + const std::vector> &FuncBBClusterInfo) { + assert(MF.hasBBSections() && "BB Sections is not set for function."); + // This is the set of sections which have EHPads in them. + SmallSet EHPadsSections; for (auto &MBB : MF) { - // Entry basic block cannot start another section because the function - // starts one already. - if (MBB.getNumber() == MF.front().getNumber()) { - MBB.setSectionType(MachineBasicBlockSection::MBBS_Entry); - continue; - } - // Check if this BB is a cold basic block. With the list option, all cold - // basic blocks can be clustered in a single cold section. - // All Exception landing pads must be in a single section. If all the - // landing pads are cold, it can be kept in the cold section. Otherwise, we - // create a separate exception section. - bool isColdBB = ((MF.getTarget().getBBSectionsType() == - llvm::BasicBlockSection::List) && - !S.empty() && !S.count(MBB.getNumber())); - if (isColdBB) { - MBB.setSectionType(MachineBasicBlockSection::MBBS_Cold); - } else if (MBB.isEHPad()) { - // We handle non-cold basic eh blocks later. - HasHotEHPads = true; - } else { - // Place this MBB in a unique section. A unique section begins and ends - // that section by definition. - MBB.setSectionType(MachineBasicBlockSection::MBBS_Unique); + // With the 'all' option, every basic block is placed in a unique section. + // With the 'list' option, every basic block is placed in a section + // associated with its cluster, unless we want individual unique sections + // for every basic block in this function (if FuncBBClusterInfo is empty). + if (MF.getTarget().getBBSectionsType() == llvm::BasicBlockSection::All || + FuncBBClusterInfo.empty()) { + // If unique sections are desired for all basic blocks of the function, we + // set every basic block's section ID equal to its number (basic block + // id). This further ensures that basic blocks are ordered canonically. + MBB.setSectionID(MBB.getNumber()); + } else if (FuncBBClusterInfo[MBB.getNumber()].hasValue()) + MBB.setSectionID(FuncBBClusterInfo[MBB.getNumber()]->ClusterID); + else { + // BB goes into the special cold section if it is not specified in the + // cluster info map. + MBB.setSectionID(MachineBasicBlock::ColdSectionID); } + + if (MBB.isEHPad()) + EHPadsSections.insert(MBB.getSectionID().getValue()); } - // If some EH Pads are not cold then we move all EH Pads to the exception - // section as we require that all EH Pads be in a single section. - if (HasHotEHPads) { + // If EHPads are in more than one section, we move all of them to a specific + // exception section, as we need all EH Pads to be in a single section. + if (EHPadsSections.size() > 1) { std::for_each(MF.begin(), MF.end(), [&](MachineBasicBlock &MBB) { if (MBB.isEHPad()) - MBB.setSectionType(MachineBasicBlockSection::MBBS_Exception); + MBB.setSectionID(MachineBasicBlock::ExceptionSectionID); }); } - for (auto &MBB : MF) { - // With -fbasicblock-sections, fall through blocks must be made - // explicitly reachable. Do this after sections is set as - // unnecessary fallthroughs can be avoided. - insertUnconditionalFallthroughBranch(MBB); - } - - MF.sort(([&](MachineBasicBlock &X, MachineBasicBlock &Y) { - unsigned TypeX = X.getSectionType(); - unsigned TypeY = Y.getSectionType(); + SmallVector PreLayoutFallThroughs( + MF.getNumBlockIDs()); + for (auto &MBB : MF) + PreLayoutFallThroughs[MBB.getNumber()] = MBB.getFallThrough(); + + // We make sure that the cluster including the entry basic block precedes all + // other clusters. + auto EntryBBSectionID = MF.front().getSectionID().getValue(); + + // We sort all basic blocks to make sure the basic blocks of every cluster are + // contiguous and ordered accordingly. Furthermore, clusters are ordered in + // increasing order of their section IDs, with the exception and the + // cold section placed at the end of the function. + MF.sort([&](MachineBasicBlock &X, MachineBasicBlock &Y) { + auto XSectionID = X.getSectionID().getValue(); + auto YSectionID = Y.getSectionID().getValue(); + // If the two basic block are in the same section, the order is decided by + // their position within the section. + if (XSectionID == YSectionID) + return XSectionID < MachineBasicBlock::ExceptionSectionID + ? FuncBBClusterInfo[X.getNumber()]->PositionInCluster < + FuncBBClusterInfo[Y.getNumber()]->PositionInCluster + : X.getNumber() < Y.getNumber(); + // We make sure that the section containing the entry block precedes all the + // other sections. + if (XSectionID == EntryBBSectionID || YSectionID == EntryBBSectionID) + return XSectionID == EntryBBSectionID; + return XSectionID < YSectionID; + }); + + // After reordering basic blocks, we must update basic block branches to + // insert explicit fallthrough branches when required and optimize branches + // when possible. + updateBranches(MF, PreLayoutFallThroughs); - return (TypeX != TypeY) ? TypeX < TypeY : X.getNumber() < Y.getNumber(); - })); - - MF.setSectionRange(); return true; } @@ -243,65 +323,117 @@ return true; } + std::vector> FuncBBClusterInfo; if (BBSectionsType == BasicBlockSection::List && - BBSectionsList.find(MF.getName()) == BBSectionsList.end()) + !getBBClusterInfoForFunction(MF, FuncAliasMap, ProgramBBClusterInfo, + FuncBBClusterInfo)) return true; - MF.setBBSectionsType(BBSectionsType); MF.createBBLabels(); - assignSectionsAndSortBasicBlocks(MF, BBSectionsList); - + assignSectionsAndSortBasicBlocks(MF, FuncBBClusterInfo); return true; } // Basic Block Sections can be enabled for a subset of machine basic blocks. // This is done by passing a file containing names of functions for which basic // block sections are desired. Additionally, machine basic block ids of the -// functions can also be specified for a finer granularity. -// A file with basic block sections for all of function main and two blocks for -// function foo looks like this: +// functions can also be specified for a finer granularity. Moreover, a cluster +// of basic blocks could be assigned to the same section. +// A file with basic block sections for all of function main and three blocks +// for function foo (of which 1 and 2 are placed in a cluster) looks like this: // ---------------------------- // list.txt: // !main // !foo -// !!2 +// !!1 2 // !!4 -static bool getBBSectionsList(const MemoryBuffer *MBuf, - StringMap> &bbMap) { - if (!MBuf) - return false; - +static Error getBBClusterInfo(const MemoryBuffer *MBuf, + ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, + StringMap &FuncAliasMap) { + assert(MBuf); line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); - StringMap>::iterator fi = bbMap.end(); + auto invalidProfileError = [&](auto Message) { + SmallString<128> ErrorMessage("Invalid profile "); + ErrorMessage += MBuf->getBufferIdentifier(); + ErrorMessage += " at line "; + ErrorMessage += Twine(LineIt.line_number()).str(); + ErrorMessage += ": "; + ErrorMessage += Message; + return make_error(ErrorMessage, inconvertibleErrorCode()); + }; + + auto FI = ProgramBBClusterInfo.end(); + + // Current cluster ID corresponding to this function. + unsigned CurrentCluster = 0; + // Current position in the current cluster. + unsigned CurrentPosition = 0; + + // Temporary set to ensure every basic block ID appears once in the clusters + // of a function. + SmallSet FuncBBIDs; for (; !LineIt.is_at_eof(); ++LineIt) { - StringRef s(*LineIt); - if (s[0] == '@') + StringRef S(*LineIt); + if (S[0] == '@') continue; // Check for the leading "!" - if (!s.consume_front("!") || s.empty()) + if (!S.consume_front("!") || S.empty()) break; - // Check for second "!" which encodes basic block ids. - if (s.consume_front("!")) { - if (fi != bbMap.end()) - fi->second.insert(std::stoi(s.str())); - else - return false; - } else { + // Check for second "!" which indicates a cluster of basic blocks. + if (S.consume_front("!")) { + if (FI == ProgramBBClusterInfo.end()) + return invalidProfileError( + "Cluster list does not follow a function name specifier."); + std::istringstream ISS(S.str()); + // Reset current cluster position. + CurrentPosition = 0; + std::string BBIndexStr; + while (getline(ISS, BBIndexStr, ' ')) { + unsigned long long BBIndex; + if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex)) + return invalidProfileError( + std::string("Unsigned integer expected: '") + BBIndexStr + "'."); + if (!FuncBBIDs.insert(BBIndex).second) + return invalidProfileError( + std::string("Duplicate basic block id found '") + BBIndexStr + + "'."); + if (!BBIndex && CurrentPosition) + return invalidProfileError("Entry BB (0) does not begin a cluster."); + + FI->second.emplace_back(BBClusterInfo{ + ((unsigned)BBIndex), CurrentCluster, CurrentPosition++}); + } + CurrentCluster++; + } else { // This is a function name specifier. + // Function aliases are separated using '/'. We use the first function + // name for the cluster info mapping and delegate all other aliases to + // this one. + auto P = S.split('/'); // Start a new function. - auto R = bbMap.try_emplace(s.split('/').first); - fi = R.first; - assert(R.second); + FI = ProgramBBClusterInfo.try_emplace(P.first).first; + + auto AliasStr = P.second; + while (AliasStr != "") { + auto Q = AliasStr.split('/'); + FuncAliasMap.try_emplace(Q.first, P.first); + AliasStr = Q.second; + } + // Prepare for parsing clusters of this function name. + CurrentCluster = 0; + FuncBBIDs.clear(); } } - return true; + return Error::success(); } bool BBSectionsPrepare::doInitialization(Module &M) { - if (MBuf) - getBBSectionsList(MBuf, BBSectionsList); - return true; + if (!MBuf) + return false; + if (auto Err = getBBClusterInfo(MBuf, ProgramBBClusterInfo, FuncAliasMap)) + report_fatal_error(std::move(Err)); + return false; } void BBSectionsPrepare::getAnalysisUsage(AnalysisUsage &AU) const { Index: llvm/lib/CodeGen/MIRParser/MIParser.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -495,7 +495,7 @@ bool parseOffset(int64_t &Offset); bool parseAlignment(unsigned &Alignment); bool parseAddrspace(unsigned &Addrspace); - bool parseMBBS(MachineBasicBlockSection &T); + bool parseSectionID(Optional &SID); bool parseOperandsOffset(MachineOperand &Op); bool parseIRValue(const Value *&V); bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags); @@ -620,21 +620,24 @@ return true; } -// Parse Machine Basic Block Section Type. -bool MIParser::parseMBBS(MachineBasicBlockSection &T) { +// Parse Machine Basic Block Section ID. +bool MIParser::parseSectionID(Optional &SID) { assert(Token.is(MIToken::kw_bbsections)); lex(); - const StringRef &S = Token.stringValue(); - if (S == "Entry") - T = MBBS_Entry; - else if (S == "Exception") - T = MBBS_Exception; - else if (S == "Cold") - T = MBBS_Cold; - else if (S == "Unique") - T = MBBS_Unique; - else - return error("Unknown Section Type"); + if (Token.is(MIToken::IntegerLiteral)) { + unsigned Value = 0; + if (getUnsigned(Value)) + return error("Uknown Section ID"); + SID = Value; + } else { + const StringRef &S = Token.stringValue(); + if (S == "Exception") + SID = MachineBasicBlock::ExceptionSectionID; + else if (S == "Cold") + SID = MachineBasicBlock::ColdSectionID; + else + return error("Unknown Section ID"); + } lex(); return false; } @@ -651,7 +654,7 @@ bool HasAddressTaken = false; bool IsLandingPad = false; bool IsEHFuncletEntry = false; - MachineBasicBlockSection SectionType = MBBS_None; + Optional SectionID; unsigned Alignment = 0; BasicBlock *BB = nullptr; if (consumeIfPresent(MIToken::lparen)) { @@ -681,7 +684,7 @@ lex(); break; case MIToken::kw_bbsections: - if (parseMBBS(SectionType)) + if (parseSectionID(SectionID)) return true; break; default: @@ -714,8 +717,8 @@ MBB->setHasAddressTaken(); MBB->setIsEHPad(IsLandingPad); MBB->setIsEHFuncletEntry(IsEHFuncletEntry); - if (SectionType != MBBS_None) { - MBB->setSectionType(SectionType); + if (SectionID.hasValue()) { + MBB->setSectionID(SectionID.getValue()); MF.setBBSectionsType(BasicBlockSection::List); } return false; Index: llvm/lib/CodeGen/MIRParser/MIRParser.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -441,10 +441,8 @@ if (MF.getTarget().getBBSectionsType() == BasicBlockSection::Labels) { MF.createBBLabels(); MF.setBBSectionsType(BasicBlockSection::Labels); - } else if (MF.hasBBSections()) { - MF.setSectionRange(); + } else if (MF.hasBBSections()) MF.createBBLabels(); - } PFS.SM = &SM; // Initialize the frame information after creating all the MBBs so that the Index: llvm/lib/CodeGen/MIRPrinter.cpp =================================================================== --- llvm/lib/CodeGen/MIRPrinter.cpp +++ llvm/lib/CodeGen/MIRPrinter.cpp @@ -640,24 +640,18 @@ OS << "align " << MBB.getAlignment().value(); HasAttributes = true; } - if (MBB.getSectionType() != MBBS_None) { + if (MBB.getSectionID().hasValue()) { OS << (HasAttributes ? ", " : " ("); OS << "bbsections "; - switch (MBB.getSectionType()) { - case MBBS_Entry: - OS << "Entry"; + switch (MBB.getSectionID().getValue()) { + case MachineBasicBlock::ExceptionSectionID: + OS << "Excetion"; break; - case MBBS_Exception: - OS << "Exception"; - break; - case MBBS_Cold: + case MachineBasicBlock::ColdSectionID: OS << "Cold"; break; - case MBBS_Unique: - OS << "Unique"; - break; default: - llvm_unreachable("No such section type"); + OS << MBB.getSectionID(); } HasAttributes = true; } Index: llvm/lib/CodeGen/MachineBasicBlock.cpp =================================================================== --- llvm/lib/CodeGen/MachineBasicBlock.cpp +++ llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -552,42 +552,46 @@ bool MachineBasicBlock::sameSection(const MachineBasicBlock *Other) const { if (this == Other) return true; - - if (this->getSectionType() != Other->getSectionType()) - return false; - - // If either is in a unique section, return false. - if (this->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique || - Other->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique) + if (this->getSectionID() != Other->getSectionID()) return false; - return true; } +// Returns the basic block ending the section containing this basic block. +// Returns null if basic block sections is not enabled for this function. +// This function has a linear time complexity. const MachineBasicBlock *MachineBasicBlock::getSectionEndMBB() const { - if (this->isEndSection()) - return this; - auto I = std::next(this->getIterator()); const MachineFunction *MF = getParent(); - while (I != MF->end()) { - const MachineBasicBlock &MBB = *I; - if (MBB.isEndSection()) - return &MBB; - I = std::next(I); - } - llvm_unreachable("No End Basic Block for this section."); + if (!MF->hasBBSections()) + return nullptr; + // Iterate over basic blocks looking for a basic block with a different + // section ID. + auto I = getIterator(); + while (I != MF->end() && I->getSectionID() == getSectionID()) + ++I; + return I == MF->end() ? &MF->back() : &*std::prev(I); } -// Returns true if this block begins any section. +// Returns true if this block begins any section (If function has BB sections). +// Checks only the previous basic block to see if it has a different section ID. +// The first basic block always begins a section. bool MachineBasicBlock::isBeginSection() const { - return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || - getParent()->isSectionStartMBB(getNumber())); + const MachineFunction *MF = getParent(); + if (!MF->hasBBSections()) + return false; + auto Before = std::next(getReverseIterator()); + return (Before == MF->rend()) || (Before->getSectionID() != getSectionID()); } -// Returns true if this block begins any section. +// Returns true if this block ends any section (If function has BB sections). +// Checks only the next basic block to see if it has a different section ID. +// The last basic block always ends a section. bool MachineBasicBlock::isEndSection() const { - return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || - getParent()->isSectionEndMBB(getNumber())); + const MachineFunction *MF = getParent(); + if (!MF->hasBBSections()) + return false; + auto After = std::next(getIterator()); + return (After == MF->end()) || (After->getSectionID() != getSectionID()); } void MachineBasicBlock::updateTerminator() { @@ -1572,3 +1576,6 @@ "Liveness information is accurate"); return LiveIns.begin(); } + +const unsigned MachineBasicBlock::ColdSectionID; +const unsigned MachineBasicBlock::ExceptionSectionID; Index: llvm/lib/CodeGen/MachineFunction.cpp =================================================================== --- llvm/lib/CodeGen/MachineFunction.cpp +++ llvm/lib/CodeGen/MachineFunction.cpp @@ -341,32 +341,6 @@ MBBNumbering.resize(BlockNo); } -/// This sets the section ranges of cold or exception section with basic block -/// sections. -void MachineFunction::setSectionRange() { - // Compute the Section Range of cold and exception basic blocks. Find the - // first and last block of each range. - auto SectionRange = - ([&](llvm::MachineBasicBlockSection S) -> std::pair { - auto MBBP = - std::find_if(begin(), end(), [&](MachineBasicBlock &MBB) -> bool { - return MBB.getSectionType() == S; - }); - if (MBBP == end()) - return std::make_pair(-1, -1); - - auto MBBQ = - std::find_if(rbegin(), rend(), [&](MachineBasicBlock &MBB) -> bool { - return MBB.getSectionType() == S; - }); - assert(MBBQ != rend() && "Section end not found!"); - return std::make_pair(MBBP->getNumber(), MBBQ->getNumber()); - }); - - ExceptionSectionRange = SectionRange(MBBS_Exception); - ColdSectionRange = SectionRange(llvm::MBBS_Cold); -} - /// This is used with -fbasicblock-sections or -fbasicblock-labels option. /// A unary encoding of basic block labels is done to keep ".strtab" sizes /// small. Index: llvm/test/CodeGen/X86/basicblock-sections-clusters-branches.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-clusters-branches.ll @@ -0,0 +1,65 @@ +; BB cluster sections test for optimizing basic block branches. +; +; Test1: Basic blocks #0 (entry) and #2 will be placed in the same section. +; There should be a jne from #0 to #1 and a fallthrough to #2. +; BB #1 will be in a unique section. Therefore, it should retain its jmp to #3. +; #2 must also have an explicit jump to #3. +; RUN: echo '!foo' > %t1 +; RUN: echo '!!0 2' >> %t1 +; RUN: echo '!!1' >> %t1 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1 +; +; Test2: Basic blocks #1 and #3 will be placed in the same section. +; The rest (#0 and #2) go into the function's section. +; This means #1 must fall through to #3, and #0 must fall through to #2. +; #2 must have an explicit jump to #3. +; RUN: echo '!foo' > %t2 +; RUN: echo '!!1 3' >> %t2 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2 + +define void @foo(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @bar() + br label %10 + +8: ; preds = %1 + %9 = call i32 @baz() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @bar() #1 + +declare i32 @baz() #1 + +; LINUX-SECTIONS1: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS1-LABEL: foo: +; LINUX-SECTIONS1: jne a.BB.foo +; LINUX-SECTIONS1-NOT: {{jne|je|jmp}} +; LINUX-SECTIONS1-LABEL: aa.BB.foo: +; LINUX-SECTIONS1: jmp raa.BB.foo +; LINUX-SECTIONS1: .section .text.foo,"ax",@progbits,unique,1 +; LINUX-SECTIONS1-LABEL: a.BB.foo: +; LINUX-SECTIONS1: jmp raa.BB.foo +; LINUX-SECTIONS1: .section .text.foo.unlikely,"ax",@progbits +; LINUX-SECTIONS1-LABEL: raa.BB.foo: + +; LINUX-SECTIONS2: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS2-LABEL: foo: +; LINUX-SECTIONS2: jne a.BB.foo +; LINUX-SECTIONS2-NOT: {{jne|je|jmp}} +; LINUX-SECTIONS2-LABEL: aa.BB.foo: +; LINUX-SECTIONS2: jmp raa.BB.foo +; LINUX-SECTIONS2: .section .text.foo,"ax",@progbits,unique,1 +; LINUX-SECTIONS2-NEXT: a.BB.foo: +; LINUX-SECTIONS2-NOT: {{jne|je|jmp}} +; LINUX-SECTIONS2-LABEL: raa.BB.foo: Index: llvm/test/CodeGen/X86/basicblock-sections-clusters-eh.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-clusters-eh.ll @@ -0,0 +1,89 @@ +; BB cluster section test for exception handling. +; +; Test1: Basic blocks #1 and #3 are landing pads and must be in the same section. +; Basic block 2 will be placed in a unique section, but #1 and #3 are placed in the special exception section. +; The rest will be placed in a section along with the entry basic block. +; RUN: echo '!main' > %t1 +; RUN: echo '!!1 2' >> %t1 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1 +; +; Test2: Basic blocks #1, #2, and #3 go into a separate section. +; No separate exception section will be created as #1 and #3 are already in one section. +; The rest will be placed in a section along with the entry basic block. +; RUN: echo '!main' > %t2 +; RUN: echo '!!1 2 3' >> %t2 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2 +; + +@_ZTIi = external constant i8* + +define i32 @main() uwtable optsize ssp personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @_Z1fv() optsize + to label %try.cont unwind label %lpad1 + +lpad1: + %0 = landingpad { i8*, i32 } + cleanup + catch i8* bitcast (i8** @_ZTIi to i8*) + br label %eh.resume1 + +try.cont: + invoke void @_Z2fv() optsize + to label %try.cont unwind label %lpad2 + ret i32 0 + +lpad2: + %2 = landingpad { i8*, i32 } + cleanup + catch i8* bitcast (i8** @_ZTIi to i8*) + br label %eh.resume2 + +eh.resume1: + resume { i8*, i32 } %0 + +eh.resume2: + resume { i8*, i32 } %2 +} + +declare void @_Z1fv() optsize + +declare void @_Z2fv() optsize + +declare i32 @__gxx_personality_v0(...) + +; LINUX-SECTIONS1: .section .text.main,"ax",@progbits +; LINUX-SECTIONS1-LABEL: main: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: alal.BB.main: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: aalal.BB.main: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: aaalal.BB.main: +; LINUX-SECTIONS1: .section .text.main,"ax",@progbits,unique,1 +; LINUX-SECTIONS1-LABEL: al.BB.main: +; LINUX-SECTIONS1: .section .text.main.eh,"ax",@progbits +; LINUX-SECTIONS1-LABEL: l.BB.main: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: lal.BB.main: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1: .section .text.main,"ax",@progbits +; LINUX-SECTIONS1-LABEL: .Lfunc_end0 + + +; LINUX-SECTIONS2: .section .text.main,"ax",@progbits +; LINUX-SECTIONS2-LABEL: main: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: alal.BB.main: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: aalal.BB.main: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: aaalal.BB.main: +; LINUX-SECTIONS2: .section .text.main,"ax",@progbits,unique,1 +; LINUX-SECTIONS2-LABEL: l.BB.main: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: al.BB.main: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: lal.BB.main: +; LINUX-SECTIONS2: .section .text.main,"ax",@progbits +; LINUX-SECTIONS2-LABEL: .Lfunc_end0 Index: llvm/test/CodeGen/X86/basicblock-sections-clusters-error.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-clusters-error.ll @@ -0,0 +1,25 @@ +; BB cluster sections error handling +; RUN: echo '!f' > %t1 +; RUN: echo '!!1 4' >> %t1 +; RUN: echo '!!1' >> %t1 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR1 +; CHECK-ERROR1: LLVM ERROR: Invalid profile {{.*}} at line 3: Duplicate basic block id found '1'. +; RUN: echo '!f' > %t2 +; RUN: echo '!!4 0' >> %t2 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR2 +; CHECK-ERROR2: LLVM ERROR: Invalid profile {{.*}} at line 2: Entry BB (0) does not begin a cluster. +; RUN: echo '!f' > %t3 +; RUN: echo '!!-1' >> %t3 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t3 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR3 +; CHECK-ERROR3: LLVM ERROR: Invalid profile {{.*}} at line 2: Unsigned integer expected: '-1'. +; RUN: echo '!!1' > %t4 +; RUN: echo '!f' >> %t4 +; RUN: not --crash llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t4 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR4 +; CHECK-ERROR4: LLVM ERROR: Invalid profile {{.*}} at line 1: Cluster list does not follow a function name specifier. + +define i32 @dummy(i32 %x, i32 %y, i32 %z) { + entry: + %tmp = mul i32 %x, %y + %tmp2 = add i32 %tmp, %z + ret i32 %tmp2 +} Index: llvm/test/CodeGen/X86/basicblock-sections-clusters.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-clusters.ll @@ -0,0 +1,72 @@ +; BB cluster section tests. +; +; Test1: Basic blocks #0 (entry) and #2 will be placed in the same section. +; Basic block 1 will be placed in a unique section. +; The rest will be placed in the cold section. +; RUN: echo '!foo' > %t1 +; RUN: echo '!!0 2' >> %t1 +; RUN: echo '!!1' >> %t1 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t1 | FileCheck %s -check-prefix=LINUX-SECTIONS1 +; +; Test2: Basic blocks #1 and #3 will be placed in the same section. +; All other BBs (including the entry block) go into the function's section. +; RUN: echo '!foo' > %t2 +; RUN: echo '!!1 3' >> %t2 +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t2 | FileCheck %s -check-prefix=LINUX-SECTIONS2 + +define void @foo(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @bar() + br label %10 + +8: ; preds = %1 + %9 = call i32 @baz() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @bar() #1 + +declare i32 @baz() #1 + +; LINUX-SECTIONS1: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: foo: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1-LABEL: aa.BB.foo: +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1: .section .text.foo,"ax",@progbits,unique,1 +; LINUX-SECTIONS1-LABEL: a.BB.foo: +; LINUX-SECTIONS1-LABEL: .Ltmp0: +; LINUX-SECTIONS1-NEXT: .size a.BB.foo, .Ltmp0-a.BB.foo +; LINUX-SECTIONS1-NOT: .section +; LINUX-SECTIONS1: .section .text.foo.unlikely,"ax",@progbits +; LINUX-SECTIONS1-LABEL: raa.BB.foo: +; LINUX-SECTIONS1: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS1-LABEL: .Lfunc_end0: +; LINUX-SECTIONS1-NEXT: .size foo, .Lfunc_end0-foo + +; LINUX-SECTIONS2: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: foo: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: aa.BB.foo: +; LINUX-SECTIONS2: .section .text.foo,"ax",@progbits,unique,1 +; LINUX-SECTIONS2-NEXT: a.BB.foo: +; LINUX-SECTIONS2-NOT: .section +; LINUX-SECTIONS2-LABEL: raa.BB.foo: +; LINUX-SECTIONS2-LABEL: .Ltmp0: +; LINUX-SECTIONS2-NEXT: .size a.BB.foo, .Ltmp0-a.BB.foo +; LINUX-SECTIONS2: .section .text.foo,"ax",@progbits +; LINUX-SECTIONS2-LABEL: .Lfunc_end0: +; LINUX-SECTIONS2-NEXT: .size foo, .Lfunc_end0-foo + Index: llvm/test/CodeGen/X86/basicblock-sections-eh.ll =================================================================== --- llvm/test/CodeGen/X86/basicblock-sections-eh.ll +++ llvm/test/CodeGen/X86/basicblock-sections-eh.ll @@ -80,5 +80,5 @@ ;LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits ;LINUX-SECTIONS: _Z3foob: -;LINUX-SECTIONS: .section .text._Z3foob.eh,"ax",@progbits +;LINUX-SECTIONS: .section .text._Z3foob.laara.BB._Z3foob,"ax",@progbits ;LINUX-SECTIONS: l{{[a|r]*}}.BB._Z3foob: Index: llvm/test/CodeGen/X86/basicblock-sections-listbb.ll =================================================================== --- llvm/test/CodeGen/X86/basicblock-sections-listbb.ll +++ llvm/test/CodeGen/X86/basicblock-sections-listbb.ll @@ -28,11 +28,12 @@ declare i32 @_Z3foov() #1 -; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits -; LINUX-SECTIONS: _Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: ; Check that the basic block with id 1 doesn't get a section. -; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique -; LINUX-SECTIONS: r.BB._Z3bazb: -; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique -; LINUX-SECTIONS: rr.BB._Z3bazb: -; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .Ltmp0: +; LINUX-SECTIONS-NEXT: .size rr.BB._Z3bazb, .Ltmp0-rr.BB._Z3bazb Index: llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir =================================================================== --- llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir +++ llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir @@ -87,7 +87,7 @@ constants: [] machineFunctionInfo: {} body: | - bb.0 (%ir-block.1, bbsections Entry): + bb.0 (%ir-block.1, align 4, bbsections 0): successors: %bb.2(0x40000000), %bb.1(0x40000000) liveins: $edi @@ -102,19 +102,19 @@ JCC_1 %bb.2, 4, implicit killed $eflags JMP_1 %bb.1 - bb.1 (%ir-block.7, bbsections Unique): + bb.1 (%ir-block.7, bbsections 1): successors: %bb.3(0x80000000) MOV32mi $rbp, 1, $noreg, -8, $noreg, 1 :: (store 4 into %ir.2) JMP_1 %bb.3 - bb.2 (%ir-block.8, bbsections Unique): + bb.2 (%ir-block.8, bbsections 2): successors: %bb.3(0x80000000) MOV32mi $rbp, 1, $noreg, -8, $noreg, 0 :: (store 4 into %ir.2) JMP_1 %bb.3 - bb.3 (%ir-block.9, bbsections Unique): + bb.3 (%ir-block.9, bbsections 3): renamable $eax = MOV32rm $rbp, 1, $noreg, -8, $noreg :: (load 4 from %ir.2) $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp CFI_INSTRUCTION def_cfa $rsp, 8 Index: llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll =================================================================== --- llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll +++ llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll @@ -26,7 +26,7 @@ ret i32 %10 } -; CHECK: bbsections Entry -; CHECK: bbsections Cold -; CHECK: bbsections Cold -; CHECK: bbsections Unique +; CHECK: bb.0 (%ir-block.1, bbsections Cold): +; CHECK: bb.2 (%ir-block.8, bbsections Cold): +; CHECK: bb.3 (%ir-block.9, bbsections Cold): +; CHECK: bb.1 (%ir-block.7, bbsections 0) Index: llvm/test/CodeGen/X86/basicblock-sections.ll =================================================================== --- llvm/test/CodeGen/X86/basicblock-sections.ll +++ llvm/test/CodeGen/X86/basicblock-sections.ll @@ -26,11 +26,13 @@ declare i32 @_Z3foov() #1 -; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits -; LINUX-SECTIONS: _Z3bazb: -; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique,1 -; LINUX-SECTIONS: r.BB._Z3bazb: -; LINUX-SECTIONS: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb -; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique,2 -; LINUX-SECTIONS: rr.BB._Z3bazb: -; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique,1 +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .Ltmp0: +; LINUX-SECTIONS-NEXT: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique,2 +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .Ltmp1: +; LINUX-SECTIONS-NEXT: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb