Index: llvm/include/llvm/CodeGen/MachineBasicBlock.h =================================================================== --- llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -46,6 +46,19 @@ class TargetRegisterClass; class TargetRegisterInfo; +enum MachineBasicBlockSection : unsigned { + /// This is also the order of sections in a function. Basic blocks that are + /// part of the original function section (entry block) come first, followed + /// by exception handling basic blocks, cold basic blocks and finally basic + // blocks that need unique sections. + MBBS_Entry, + MBBS_Exception, + MBBS_Cold, + MBBS_Unique, + /// None implies no sections for any basic block, the default. + MBBS_None, +}; + template <> struct ilist_traits { private: friend class MachineBasicBlock; // Set by the owning MachineBasicBlock. @@ -130,6 +143,9 @@ /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; + /// Stores the Section type of the basic block with basic block sections. + MachineBasicBlockSection SectionType = MBBS_None; + /// Default target of the callbr of a basic block. bool InlineAsmBrDefaultTarget = false; @@ -140,6 +156,9 @@ /// is only computed once and is cached. mutable MCSymbol *CachedMCSymbol = nullptr; + /// Used during basic block sections to mark the end of a basic block. + MCSymbol *EndMCSymbol = nullptr; + // Intrusive list support MachineBasicBlock() = default; @@ -415,6 +434,18 @@ /// Indicates if this is the entry block of a cleanup funclet. void setIsCleanupFuncletEntry(bool V = true) { IsCleanupFuncletEntry = V; } + /// Returns true if this block begins any section. + bool isBeginSection() const; + + /// Returns true if this block ends any section. + bool isEndSection() const; + + /// Returns the type of section this basic block belongs to. + MachineBasicBlockSection getSectionType() const { return SectionType; } + + /// Indicate that the basic block belongs to a Section Type. + void setSectionType(MachineBasicBlockSection V) { SectionType = V; } + /// Returns true if this is the indirect dest of an INLINEASM_BR. bool isInlineAsmBrIndirectTarget(const MachineBasicBlock *Tgt) const { return InlineAsmBrIndirectTargets.count(Tgt); @@ -453,6 +484,12 @@ void moveBefore(MachineBasicBlock *NewAfter); void moveAfter(MachineBasicBlock *NewBefore); + /// Returns true if this and MBB belong to the same section. + bool sameSection(const MachineBasicBlock *MBB) const; + + /// Returns the basic block that ends the section which contains this one. + const MachineBasicBlock *getSectionEndMBB() const; + /// Update the terminator instructions in block to account for changes to the /// layout. If the block previously used a fallthrough, it may now need a /// branch, and if it previously used branching it may now be able to use a @@ -839,6 +876,12 @@ /// Return the MCSymbol for this basic block. MCSymbol *getSymbol() const; + /// Sets the MCSymbol corresponding to the end of this basic block. + void setEndMCSymbol(MCSymbol *Sym) { EndMCSymbol = Sym; } + + /// Returns the MCSymbol corresponding to the end of this basic block. + MCSymbol *getEndMCSymbol() const { return EndMCSymbol; } + Optional getIrrLoopHeaderWeight() const { return IrrLoopHeaderWeight; } Index: llvm/include/llvm/CodeGen/MachineFunction.h =================================================================== --- llvm/include/llvm/CodeGen/MachineFunction.h +++ llvm/include/llvm/CodeGen/MachineFunction.h @@ -37,6 +37,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Recycler.h" +#include "llvm/Target/TargetOptions.h" #include #include #include @@ -64,6 +65,7 @@ class MCContext; class MCInstrDesc; class MCSymbol; +class MCSection; class Pass; class PseudoSourceValueManager; class raw_ostream; @@ -244,6 +246,9 @@ // Keep track of jump tables for switch instructions MachineJumpTableInfo *JumpTableInfo; + // Keep track of the function section. + MCSection *Section = nullptr; + // Keeps track of Wasm exception handling related data. This will be null for // functions that aren't using a wasm EH personality. WasmEHFuncInfo *WasmEHInfo = nullptr; @@ -257,6 +262,12 @@ // numbered and this vector keeps track of the mapping from ID's to MBB's. std::vector MBBNumbering; + // Unary encoding of basic block symbols is used to reduce size of ".strtab". + // Basic block number 'i' gets a prefix of length 'i'. The ith character also + // denotes the type of basic block number 'i'. Return blocks are marked with + // 'r', landing pads with 'l' and regular blocks with 'a'. + std::vector BBSectionsSymbolPrefix; + // Pool-allocate MachineFunction-lifetime and IR objects. BumpPtrAllocator Allocator; @@ -332,6 +343,14 @@ bool HasEHScopes = false; bool HasEHFunclets = false; + /// Section Type for basic blocks, only relevant with basic block sections. + BasicBlockSection::SectionMode BBSectionsType = BasicBlockSection::None; + + /// With Basic Block Sections, this stores the bb ranges of cold and + /// exception sections. + std::pair ColdSectionRange = {-1, -1}; + std::pair ExceptionSectionRange = {-1, -1}; + /// List of C++ TypeInfo used. std::vector TypeInfos; @@ -453,6 +472,12 @@ MachineModuleInfo &getMMI() const { return MMI; } MCContext &getContext() const { return Ctx; } + /// Returns the Section this function belongs to. + MCSection *getSection() const { return Section; } + + /// Indicates the Section this function belongs to. + void setSection(MCSection *S) { Section = S; } + PseudoSourceValueManager &getPSVManager() const { return *PSVManager; } /// Return the DataLayout attached to the Module associated to this MF. @@ -467,6 +492,37 @@ /// getFunctionNumber - Return a unique ID for the current function. unsigned getFunctionNumber() const { return FunctionNumber; } + /// Returns true if this function has basic block sections enabled. + bool hasBBSections() const { + return (BBSectionsType == BasicBlockSection::All || + BBSectionsType == BasicBlockSection::List); + } + + /// Returns true if basic block labels are to be generated for this function. + bool hasBBLabels() const { + return BBSectionsType == BasicBlockSection::Labels; + } + + void setBBSectionsType(BasicBlockSection::SectionMode V) { + BBSectionsType = V; + } + + void setSectionRange(); + + /// Returns true if this basic block number starts a cold or exception + /// section. + bool isSectionStartMBB(int N) const { + return (N == ColdSectionRange.first || N == ExceptionSectionRange.first); + } + + /// Returns true if this basic block ends a cold or exception section. + bool isSectionEndMBB(int N) const { + return (N == ColdSectionRange.second || N == ExceptionSectionRange.second); + } + + /// Creates basic block Labels for this function. + void createBBLabels(); + /// getTarget - Return the target machine this machine code is compiled with const LLVMTargetMachine &getTarget() const { return Target; } @@ -1014,6 +1070,11 @@ /// of the instruction stream. void copyCallSiteInfo(const MachineInstr *Old, const MachineInstr *New); + + const std::vector &getBBSectionsSymbolPrefix() const { + return BBSectionsSymbolPrefix; + } + /// Move the call site info from \p Old to \New call site info. This function /// is used when we are replacing one call instruction with another one to /// the same callee. Index: llvm/include/llvm/CodeGen/Passes.h =================================================================== --- llvm/include/llvm/CodeGen/Passes.h +++ llvm/include/llvm/CodeGen/Passes.h @@ -42,6 +42,11 @@ /// the entry block. FunctionPass *createUnreachableBlockEliminationPass(); + /// createBBSectionsPrepare Pass - This pass assigns sections to machine basic + /// blocks and is enabled with -fbasicblock-sections. + MachineFunctionPass * + createBBSectionsPreparePass(const std::string &ProfileFileName); + /// MachineFunctionPrinter pass - This pass prints out the machine function to /// the given stream as a debugging tool. MachineFunctionPass * Index: llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h =================================================================== --- llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -65,6 +65,15 @@ MCSection *getSectionForJumpTable(const Function &F, const TargetMachine &TM) const override; + MCSection * + getSectionForMachineBasicBlock(const Function &F, + const MachineBasicBlock &MBB, + const TargetMachine &TM) const override; + + MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const override; + bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference, const Function &F) const override; Index: llvm/include/llvm/InitializePasses.h =================================================================== --- llvm/include/llvm/InitializePasses.h +++ llvm/include/llvm/InitializePasses.h @@ -77,6 +77,7 @@ void initializeAtomicExpandPass(PassRegistry&); void initializeAttributorLegacyPassPass(PassRegistry&); void initializeAttributorCGSCCLegacyPassPass(PassRegistry &); +void initializeBBSectionsPreparePass(PassRegistry &); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); Index: llvm/include/llvm/Target/TargetLoweringObjectFile.h =================================================================== --- llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -24,6 +24,7 @@ namespace llvm { class GlobalValue; +class MachineBasicBlock; class MachineModuleInfo; class Mangler; class MCContext; @@ -90,6 +91,15 @@ const Constant *C, unsigned &Align) const; + virtual MCSection * + getSectionForMachineBasicBlock(const Function &F, + const MachineBasicBlock &MBB, + const TargetMachine &TM) const; + + virtual MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const; + /// Classify the specified global variable into a set of target independent /// categories embodied in SectionKind. static SectionKind getKindForGlobal(const GlobalObject *GO, Index: llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -682,7 +682,9 @@ emitConstantPool(); // Print the 'header' of function. - OutStreamer->SwitchSection(getObjFileLowering().SectionForGlobal(&F, TM)); + MF->setSection(getObjFileLowering().SectionForGlobal(&F, TM)); + OutStreamer->SwitchSection(MF->getSection()); + emitVisibility(CurrentFnSym, F.getVisibility()); if (MAI->needsFunctionDescriptors() && @@ -1095,6 +1097,15 @@ // Print out code for the function. bool HasAnyRealCode = false; int NumInstsInFunction = 0; + bool emitBBSections = MF->hasBBSections(); + MachineBasicBlock *EndOfRegularSectionMBB = nullptr; + if (emitBBSections) { + EndOfRegularSectionMBB = + const_cast(MF->front().getSectionEndMBB()); + assert(EndOfRegularSectionMBB->isEndSection() && + "The MBB at the end of the regular section must end a section"); + } + for (auto &MBB : *MF) { // Print a label for the basic block. emitBasicBlockStart(MBB); @@ -1174,7 +1185,18 @@ } } } - + if (&MBB != EndOfRegularSectionMBB && + (MF->hasBBLabels() || MBB.isEndSection())) { + // Emit size directive for the size of this basic block. Create a symbol + // for the end of the basic block. + MCSymbol *CurrentBBEnd = OutContext.createTempSymbol(); + const MCExpr *SizeExp = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(CurrentBBEnd, OutContext), + MCSymbolRefExpr::create(MBB.getSymbol(), OutContext), OutContext); + OutStreamer->emitLabel(CurrentBBEnd); + MBB.setEndMCSymbol(CurrentBBEnd); + OutStreamer->emitELFSize(MBB.getSymbol(), SizeExp); + } emitBasicBlockEnd(MBB); } @@ -1208,6 +1230,10 @@ } } + // Switch to the original section if basic block sections was used. + if (emitBBSections) + OutStreamer->SwitchSection(MF->getSection()); + const Function &F = MF->getFunction(); for (const auto &BB : F) { if (!BB.hasAddressTaken()) @@ -1223,7 +1249,7 @@ emitFunctionBodyEnd(); if (needFuncLabelsForEHOrDebugInfo(*MF, MMI) || - MAI->hasDotTypeDotSizeDirective()) { + MAI->hasDotTypeDotSizeDirective() || emitBBSections) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->emitLabel(CurrentFnEnd); @@ -1246,6 +1272,9 @@ HI.Handler->markFunctionEnd(); } + if (emitBBSections) + EndOfRegularSectionMBB->setEndMCSymbol(CurrentFnEnd); + // Print out jump tables referenced by the function. emitJumpTableInfo(); @@ -2980,10 +3009,11 @@ PrintChildLoopComment(OS, Loop, AP.getFunctionNumber()); } -/// EmitBasicBlockStart - This method prints the label for the specified +/// emitBasicBlockStart - This method prints the label for the specified /// MachineBasicBlock, an alignment (if present) and a comment describing /// it if appropriate. void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { + bool BBSections = MF->hasBBSections(); // End the previous funclet and start a new one. if (MBB.isEHFuncletEntry()) { for (const HandlerInfo &HI : Handlers) { @@ -2993,9 +3023,11 @@ } // Emit an alignment directive for this block, if needed. - const Align Alignment = MBB.getAlignment(); - if (Alignment != Align(1)) - emitAlignment(Alignment); + if (MBB.pred_empty() || !BBSections) { + const Align Alignment = MBB.getAlignment(); + if (Alignment != Align(1)) + emitAlignment(Alignment); + } // If the block has its address taken, emit any labels that were used to // reference the block. It is possible that there is more than one label @@ -3027,18 +3059,37 @@ emitBasicBlockLoopComments(MBB, MLI, *this); } - // Print the main label for the block. + bool emitBBLabels = BBSections || MF->hasBBLabels(); if (MBB.pred_empty() || - (isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry() && - !MBB.hasLabelMustBeEmitted())) { + (!emitBBLabels && isBlockOnlyReachableByFallthrough(&MBB) && + !MBB.isEHFuncletEntry() && !MBB.hasLabelMustBeEmitted())) { if (isVerbose()) { // NOTE: Want this comment at start of line, don't emit with AddComment. OutStreamer->emitRawComment(" %bb." + Twine(MBB.getNumber()) + ":", false); } } else { - if (isVerbose() && MBB.hasLabelMustBeEmitted()) + if (isVerbose() && MBB.hasLabelMustBeEmitted()) { OutStreamer->AddComment("Label of block must be emitted"); + } + // With -fbasicblock-sections, a basic block can start a new section. + if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Exception) { + // Create the exception section for this function. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".eh")); + } else if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Cold) { + // Create the cold section here. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".unlikely")); + } else if (MBB.isBeginSection() && MBB.isEndSection()) { + OutStreamer->SwitchSection( + getObjFileLowering().getSectionForMachineBasicBlock(MF->getFunction(), + MBB, TM)); + } else if (BBSections) { + OutStreamer->SwitchSection(MF->getSection()); + } OutStreamer->emitLabel(MBB.getSymbol()); } } @@ -3071,6 +3122,10 @@ /// the predecessor and this block is a fall-through. bool AsmPrinter:: isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { + // With BasicBlock Sections, no block is a fall through. + if (MBB->isBeginSection()) + return false; + // If this is a landing pad, it isn't a fall through. If it has no preds, // then nothing falls through to it. if (MBB->isEHPad() || MBB->pred_empty()) Index: llvm/lib/CodeGen/BBSectionsPrepare.cpp =================================================================== --- /dev/null +++ llvm/lib/CodeGen/BBSectionsPrepare.cpp @@ -0,0 +1,266 @@ +//===-- BBSectionsPrepare.cpp ---=========---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// BBSectionsPrepare implementation. +// +// The purpose of this pass is to assign sections to basic blocks when +// -fbasicblock-sections= option is used. Exception landing pad blocks are +// specially handled by grouping them in a single section. Further, with +// profile information only the subset of basic blocks with profiles are placed +// in a separate section and the rest are grouped in a cold section. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineModuleInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/InitializePasses.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include + +using llvm::SmallSet; +using llvm::StringMap; +using llvm::StringRef; +using namespace llvm; + +namespace { + +class BBSectionsPrepare : public MachineFunctionPass { +public: + static char ID; + StringMap> BBSectionsList; + std::string ProfileFileName; + + BBSectionsPrepare() : MachineFunctionPass(ID) { + initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); + } + + BBSectionsPrepare(const std::string &ProfileFile) + : MachineFunctionPass(ID), ProfileFileName(ProfileFile) { + initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); + }; + + StringRef getPassName() const override { + return "Basic Block Sections Analysis"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + /// Read profiles of basic blocks if available here. + bool doInitialization(Module &M) override; + + /// Identify basic blocks that need separate sections and prepare to emit them + /// accordingly. + bool runOnMachineFunction(MachineFunction &MF) override; +}; + +} // end anonymous namespace + +char BBSectionsPrepare::ID = 0; +INITIALIZE_PASS(BBSectionsPrepare, "bbsections-prepare", + "Determine if a basic block needs a special section", false, + false); + +// This inserts an unconditional branch at the end of MBB to the next basic +// block S if and only if the control-flow implicitly falls through from MBB to +// S and S and MBB belong to different sections. This is necessary with basic +// block sections as MBB and S could be potentially reordered. +static void insertUnconditionalFallthroughBranch(MachineBasicBlock &MBB) { + MachineBasicBlock *Fallthrough = MBB.getFallThrough(); + + if (Fallthrough == nullptr) + return; + + // If this basic block and the Fallthrough basic block are in the same + // section then do not insert the jump. + if (MBB.sameSection(Fallthrough)) + return; + + const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); + SmallVector Cond; + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; + + // If a branch to the fall through block already exists, return. + if (!TII->analyzeBranch(MBB, TBB, FBB, Cond) && + (TBB == Fallthrough || FBB == Fallthrough)) { + return; + } + + Cond.clear(); + DebugLoc DL = MBB.findBranchDebugLoc(); + TII->insertBranch(MBB, Fallthrough, nullptr, Cond, DL); +} + +/// This function sorts basic blocks according to the sections in which they are +/// emitted. Basic block sections automatically turn on function sections so +/// the entry block is in the function section. The other sections that are +/// created are: +/// 1) Exception section - basic blocks that are landing pads +/// 2) Cold section - basic blocks that will not have unique sections. +/// 3) Unique section - one per basic block that is emitted in a unique section. +static bool assignSectionsAndSortBasicBlocks( + MachineFunction &MF, + const StringMap> &BBSectionsList) { + SmallSet S = BBSectionsList.lookup(MF.getName()); + + bool HasHotEHPads = false; + + for (auto &MBB : MF) { + // Entry basic block cannot start another section because the function + // starts one already. + if (MBB.getNumber() == MF.front().getNumber()) { + MBB.setSectionType(MachineBasicBlockSection::MBBS_Entry); + continue; + } + // Check if this BB is a cold basic block. With the list option, all cold + // basic blocks can be clustered in a single cold section. + // All Exception landing pads must be in a single section. If all the + // landing pads are cold, it can be kept in the cold section. Otherwise, we + // create a separate exception section. + bool isColdBB = ((MF.getTarget().getBBSectionsType() == + llvm::BasicBlockSection::List) && + !S.empty() && !S.count(MBB.getNumber())); + if (isColdBB) { + MBB.setSectionType(MachineBasicBlockSection::MBBS_Cold); + } else if (MBB.isEHPad()) { + // We handle non-cold basic eh blocks later. + HasHotEHPads = true; + } else { + // Place this MBB in a unique section. A unique section begins and ends + // that section by definition. + MBB.setSectionType(MachineBasicBlockSection::MBBS_Unique); + } + } + + // If some EH Pads are not cold then we move all EH Pads to the exception + // section as we require that all EH Pads be in a single section. + if (HasHotEHPads) { + std::for_each(MF.begin(), MF.end(), [&](MachineBasicBlock &MBB) { + if (MBB.isEHPad()) + MBB.setSectionType(MachineBasicBlockSection::MBBS_Exception); + }); + } + + for (auto &MBB : MF) { + // With -fbasicblock-sections, fall through blocks must be made + // explicitly reachable. Do this after sections is set as + // unnecessary fallthroughs can be avoided. + insertUnconditionalFallthroughBranch(MBB); + } + + MF.sort(([&](MachineBasicBlock &X, MachineBasicBlock &Y) { + unsigned TypeX = X.getSectionType(); + unsigned TypeY = Y.getSectionType(); + + return (TypeX != TypeY) ? TypeX < TypeY : X.getNumber() < Y.getNumber(); + })); + + MF.setSectionRange(); + return true; +} + +bool BBSectionsPrepare::runOnMachineFunction(MachineFunction &MF) { + auto BBSectionsType = MF.getTarget().getBBSectionsType(); + assert(BBSectionsType != BasicBlockSection::None && + "BB Sections not enabled!"); + + // Renumber blocks before sorting them for basic block sections. This is + // useful during sorting, basic blocks in the same section will retain the + // default order. This renumbering should also be done for basic block + // labels to match the profiles with the correct blocks. + MF.RenumberBlocks(); + + if (BBSectionsType == BasicBlockSection::Labels) { + MF.setBBSectionsType(BBSectionsType); + MF.createBBLabels(); + return true; + } + + if (BBSectionsType == BasicBlockSection::List && + BBSectionsList.find(MF.getName()) == BBSectionsList.end()) + return true; + + MF.setBBSectionsType(BBSectionsType); + MF.createBBLabels(); + assignSectionsAndSortBasicBlocks(MF, BBSectionsList); + + return true; +} + +// Basic Block Sections can be enabled for a subset of machine basic blocks. +// This is done by passing a file containing names of functions for which basic +// block sections are desired. Additionally, machine basic block ids of the +// functions can also be specified for a finer granularity. +// A file with basic block sections for all of function main and two blocks for +// function foo looks like this: +// ---------------------------- +// list.txt: +// !main +// !foo +// !!2 +// !!4 +static bool getBBSectionsList(StringRef profFileName, + StringMap> &bbMap) { + if (profFileName.empty()) + return false; + + auto MbOrErr = MemoryBuffer::getFile(profFileName); + if (MbOrErr.getError()) + return false; + + MemoryBuffer &Buffer = *MbOrErr.get(); + line_iterator LineIt(Buffer, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); + + StringMap>::iterator fi = bbMap.end(); + + for (; !LineIt.is_at_eof(); ++LineIt) { + StringRef s(*LineIt); + if (s[0] == '@') + continue; + // Check for the leading "!" + if (!s.consume_front("!") || s.empty()) + break; + // Check for second "!" which encodes basic block ids. + if (s.consume_front("!")) { + if (fi != bbMap.end()) + fi->second.insert(std::stoi(s.str())); + else + return false; + } else { + // Start a new function. + auto R = bbMap.try_emplace(s.split('/').first); + fi = R.first; + assert(R.second); + } + } + return true; +} + +bool BBSectionsPrepare::doInitialization(Module &M) { + if (!ProfileFileName.empty()) + getBBSectionsList(ProfileFileName, BBSectionsList); + return true; +} + +void BBSectionsPrepare::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + AU.addRequired(); +} + +MachineFunctionPass * +llvm::createBBSectionsPreparePass(const std::string &ProfileFile) { + return new BBSectionsPrepare(ProfileFile); +} Index: llvm/lib/CodeGen/CMakeLists.txt =================================================================== --- llvm/lib/CodeGen/CMakeLists.txt +++ llvm/lib/CodeGen/CMakeLists.txt @@ -8,6 +8,7 @@ BranchRelaxation.cpp BreakFalseDeps.cpp BuiltinGCs.cpp + BBSectionsPrepare.cpp CalcSpillWeights.cpp CallingConvLower.cpp CFGuardLongjmp.cpp Index: llvm/lib/CodeGen/CodeGen.cpp =================================================================== --- llvm/lib/CodeGen/CodeGen.cpp +++ llvm/lib/CodeGen/CodeGen.cpp @@ -20,6 +20,7 @@ /// initializeCodeGen - Initialize all passes linked into the CodeGen library. void llvm::initializeCodeGen(PassRegistry &Registry) { initializeAtomicExpandPass(Registry); + initializeBBSectionsPreparePass(Registry); initializeBranchFolderPassPass(Registry); initializeBranchRelaxationPass(Registry); initializeCFGuardLongjmpPass(Registry); Index: llvm/lib/CodeGen/MIRParser/MILexer.h =================================================================== --- llvm/lib/CodeGen/MIRParser/MILexer.h +++ llvm/lib/CodeGen/MIRParser/MILexer.h @@ -122,6 +122,7 @@ kw_pre_instr_symbol, kw_post_instr_symbol, kw_heap_alloc_marker, + kw_bbsections, kw_unknown_size, // Named metadata keywords Index: llvm/lib/CodeGen/MIRParser/MILexer.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -268,6 +268,7 @@ .Case("pre-instr-symbol", MIToken::kw_pre_instr_symbol) .Case("post-instr-symbol", MIToken::kw_post_instr_symbol) .Case("heap-alloc-marker", MIToken::kw_heap_alloc_marker) + .Case("bbsections", MIToken::kw_bbsections) .Case("unknown-size", MIToken::kw_unknown_size) .Default(MIToken::Identifier); } Index: llvm/lib/CodeGen/MIRParser/MIParser.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -495,6 +495,7 @@ bool parseOffset(int64_t &Offset); bool parseAlignment(unsigned &Alignment); bool parseAddrspace(unsigned &Addrspace); + MachineBasicBlockSection parseMBBS(); bool parseOperandsOffset(MachineOperand &Op); bool parseIRValue(const Value *&V); bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags); @@ -618,6 +619,27 @@ lex(); return true; } + +// Parse Machine Basic Block Section Type. +MachineBasicBlockSection MIParser::parseMBBS() { + assert(Token.is(MIToken::kw_bbsections)); + lex(); + const StringRef &S = Token.stringValue(); + MachineBasicBlockSection T = MBBS_None; + if (S == "Entry") + T = MBBS_Entry; + else if (S == "Exception") + T = MBBS_Exception; + else if (S == "Cold") + T = MBBS_Cold; + else if (S == "Unique") + T = MBBS_Unique; + else + llvm_unreachable("Invalid Section Type!"); + lex(); + return T; +} + bool MIParser::parseBasicBlockDefinition( DenseMap &MBBSlots) { assert(Token.is(MIToken::MachineBasicBlockLabel)); @@ -629,6 +651,7 @@ lex(); bool HasAddressTaken = false; bool IsLandingPad = false; + MachineBasicBlockSection SectionType = MBBS_None; unsigned Alignment = 0; BasicBlock *BB = nullptr; if (consumeIfPresent(MIToken::lparen)) { @@ -653,6 +676,9 @@ return true; lex(); break; + case MIToken::kw_bbsections: + SectionType = parseMBBS(); + break; default: break; } @@ -682,6 +708,10 @@ if (HasAddressTaken) MBB->setHasAddressTaken(); MBB->setIsEHPad(IsLandingPad); + if (SectionType != MBBS_None) { + MBB->setSectionType(SectionType); + MF.setBBSectionsType(BasicBlockSection::List); + } return false; } Index: llvm/lib/CodeGen/MIRParser/MIRParser.cpp =================================================================== --- llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -438,6 +438,14 @@ diagFromBlockStringDiag(Error, YamlMF.Body.Value.SourceRange)); return true; } + // Check Basic Block Section Flags. + if (MF.getTarget().getBBSectionsType() == BasicBlockSection::Labels) { + MF.createBBLabels(); + MF.setBBSectionsType(BasicBlockSection::Labels); + } else if (MF.hasBBSections()) { + MF.setSectionRange(); + MF.createBBLabels(); + } PFS.SM = &SM; // Initialize the frame information after creating all the MBBs so that the Index: llvm/lib/CodeGen/MIRPrinter.cpp =================================================================== --- llvm/lib/CodeGen/MIRPrinter.cpp +++ llvm/lib/CodeGen/MIRPrinter.cpp @@ -635,6 +635,27 @@ OS << "align " << MBB.getAlignment().value(); HasAttributes = true; } + if (MBB.getSectionType() != MBBS_None) { + OS << (HasAttributes ? ", " : " ("); + OS << "bbsections "; + switch (MBB.getSectionType()) { + case MBBS_Entry: + OS << "Entry"; + break; + case MBBS_Exception: + OS << "Exception"; + break; + case MBBS_Cold: + OS << "Cold"; + break; + case MBBS_Unique: + OS << "Unique"; + break; + default: + llvm_unreachable("No such section type"); + } + HasAttributes = true; + } if (HasAttributes) OS << ")"; OS << ":\n"; Index: llvm/lib/CodeGen/MachineBasicBlock.cpp =================================================================== --- llvm/lib/CodeGen/MachineBasicBlock.cpp +++ llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -61,12 +61,31 @@ const MachineFunction *MF = getParent(); MCContext &Ctx = MF->getContext(); auto Prefix = Ctx.getAsmInfo()->getPrivateLabelPrefix(); + + bool BasicBlockSymbols = MF->hasBBSections() || MF->hasBBLabels(); + auto Delimiter = BasicBlockSymbols ? "." : "_"; assert(getNumber() >= 0 && "cannot get label for unreachable MBB"); - CachedMCSymbol = Ctx.getOrCreateSymbol(Twine(Prefix) + "BB" + - Twine(MF->getFunctionNumber()) + - "_" + Twine(getNumber())); - } + // With Basic Block Sections, we emit a symbol for every basic block. To + // keep the size of strtab small, we choose a unary encoding which can + // compress the symbol names significantly. The basic blocks for function + // foo are named a.BB.foo, aa.BB.foo, and so on. + if (BasicBlockSymbols) { + auto Iter = MF->getBBSectionsSymbolPrefix().begin(); + if (getNumber() < 0 || + getNumber() >= (int)MF->getBBSectionsSymbolPrefix().size()) + report_fatal_error("Unreachable MBB: " + Twine(getNumber())); + std::string Prefix(Iter + 1, Iter + getNumber() + 1); + std::reverse(Prefix.begin(), Prefix.end()); + CachedMCSymbol = + Ctx.getOrCreateSymbol(Prefix + Twine(Delimiter) + "BB" + + Twine(Delimiter) + Twine(MF->getName())); + } else { + CachedMCSymbol = Ctx.getOrCreateSymbol( + Twine(Prefix) + "BB" + Twine(MF->getFunctionNumber()) + + Twine(Delimiter) + Twine(getNumber())); + } + } return CachedMCSymbol; } @@ -529,6 +548,48 @@ getParent()->splice(++NewBefore->getIterator(), getIterator()); } +// Returns true if this basic block and the Other are in the same section. +bool MachineBasicBlock::sameSection(const MachineBasicBlock *Other) const { + if (this == Other) + return true; + + if (this->getSectionType() != Other->getSectionType()) + return false; + + // If either is in a unique section, return false. + if (this->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique || + Other->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique) + return false; + + return true; +} + +const MachineBasicBlock *MachineBasicBlock::getSectionEndMBB() const { + if (this->isEndSection()) + return this; + auto I = std::next(this->getIterator()); + const MachineFunction *MF = getParent(); + while (I != MF->end()) { + const MachineBasicBlock &MBB = *I; + if (MBB.isEndSection()) + return &MBB; + I = std::next(I); + } + llvm_unreachable("No End Basic Block for this section."); +} + +// Returns true if this block begins any section. +bool MachineBasicBlock::isBeginSection() const { + return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || + getParent()->isSectionStartMBB(getNumber())); +} + +// Returns true if this block begins any section. +bool MachineBasicBlock::isEndSection() const { + return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || + getParent()->isSectionEndMBB(getNumber())); +} + void MachineBasicBlock::updateTerminator() { const TargetInstrInfo *TII = getParent()->getSubtarget().getInstrInfo(); // A block with no successors has no concerns with fall-through edges. Index: llvm/lib/CodeGen/MachineFunction.cpp =================================================================== --- llvm/lib/CodeGen/MachineFunction.cpp +++ llvm/lib/CodeGen/MachineFunction.cpp @@ -33,6 +33,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -71,6 +72,7 @@ #include #include #include +#include #include #include @@ -347,6 +349,59 @@ MBBNumbering.resize(BlockNo); } +/// This sets the section ranges of cold or exception section with basic block +/// sections. +void MachineFunction::setSectionRange() { + // Compute the Section Range of cold and exception basic blocks. Find the + // first and last block of each range. + auto SectionRange = + ([&](llvm::MachineBasicBlockSection S) -> std::pair { + auto MBBP = + std::find_if(begin(), end(), [&](MachineBasicBlock &MBB) -> bool { + return MBB.getSectionType() == S; + }); + if (MBBP == end()) + return std::make_pair(-1, -1); + + auto MBBQ = + std::find_if(rbegin(), rend(), [&](MachineBasicBlock &MBB) -> bool { + return MBB.getSectionType() == S; + }); + assert(MBBQ != rend() && "Section end not found!"); + return std::make_pair(MBBP->getNumber(), MBBQ->getNumber()); + }); + + ExceptionSectionRange = SectionRange(MBBS_Exception); + ColdSectionRange = SectionRange(llvm::MBBS_Cold); +} + +/// This is used with -fbasicblock-sections or -fbasicblock-labels option. +/// A unary encoding of basic block labels is done to keep ".strtab" sizes +/// small. +void MachineFunction::createBBLabels() { + const TargetInstrInfo *TII = getSubtarget().getInstrInfo(); + this->BBSectionsSymbolPrefix.resize(getNumBlockIDs(), 'a'); + for (auto MBBI = begin(), E = end(); MBBI != E; ++MBBI) { + assert( + (MBBI->getNumber() >= 0 && MBBI->getNumber() < (int)getNumBlockIDs()) && + "BasicBlock number was out of range!"); + // 'a' - Normal block. + // 'r' - Return block. + // 'l' - Landing Pad. + // 'L' - Return and landing pad. + bool isEHPad = MBBI->isEHPad(); + bool isRetBlock = MBBI->isReturnBlock() && !TII->isTailCall(MBBI->back()); + char type = 'a'; + if (isEHPad && isRetBlock) + type = 'L'; + else if (isEHPad) + type = 'l'; + else if (isRetBlock) + type = 'r'; + BBSectionsSymbolPrefix[MBBI->getNumber()] = type; + } +} + /// Allocate a new MachineInstr. Use this instead of `new MachineInstr'. MachineInstr *MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, Index: llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -21,6 +21,8 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/IR/Comdat.h" @@ -52,8 +54,8 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" -#include "llvm/Support/Format.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include @@ -751,6 +753,56 @@ return DataRelROSection; } +/// Returns a unique section for the given machine basic block. +MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, + const TargetMachine &TM) const { + SmallString<128> Name; + Name = (static_cast(MBB.getParent()->getSection())) + ->getSectionName(); + if (TM.getUniqueBBSectionNames()) { + Name += "."; + Name += MBB.getSymbol()->getName(); + } + unsigned UniqueID = NextUniqueID++; + unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; + std::string GroupName = ""; + if (F.hasComdat()) { + Flags |= ELF::SHF_GROUP; + GroupName = F.getComdat()->getName().str(); + } + return getContext().getELFSection(Name, ELF::SHT_PROGBITS, Flags, + 0 /* Entry Size */, GroupName, UniqueID, + nullptr); +} + +MCSection *TargetLoweringObjectFileELF::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + SmallString<128> Name; + Name = (static_cast(MBB.getParent()->getSection())) + ->getSectionName(); + + // If unique section names is off, explicity add the function name to the + // section name to make sure named sections for functions are unique + // across the module. + if (!TM.getUniqueSectionNames()) { + Name += "."; + Name += MBB.getParent()->getName(); + } + + Name += Suffix; + + unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; + std::string GroupName = ""; + if (F.hasComdat()) { + Flags |= ELF::SHF_GROUP; + GroupName = F.getComdat()->getName().str(); + } + return getContext().getELFSection(Name, ELF::SHT_PROGBITS, Flags, + 0 /* Entry Size */, GroupName); +} + static MCSectionELF *getStaticStructorSection(MCContext &Ctx, bool UseInitArray, bool IsCtor, unsigned Priority, const MCSymbol *KeySym) { Index: llvm/lib/CodeGen/TargetPassConfig.cpp =================================================================== --- llvm/lib/CodeGen/TargetPassConfig.cpp +++ llvm/lib/CodeGen/TargetPassConfig.cpp @@ -983,6 +983,9 @@ addPass(createMachineOutlinerPass(RunOnAllFunctions)); } + if (TM->getBBSectionsType() != llvm::BasicBlockSection::None) + addPass(llvm::createBBSectionsPreparePass(TM->getBBSectionsFuncList())); + // Add passes that directly emit MI after all other MI passes. addPreEmitPass2(); Index: llvm/lib/Target/TargetLoweringObjectFile.cpp =================================================================== --- llvm/lib/Target/TargetLoweringObjectFile.cpp +++ llvm/lib/Target/TargetLoweringObjectFile.cpp @@ -149,6 +149,10 @@ if (isa(GO)) return SectionKind::getText(); + // Basic blocks are classified as text sections. + if (isa(GO)) + return SectionKind::getText(); + // Global variables require more detailed analysis. const auto *GVar = cast(GO); @@ -302,6 +306,18 @@ return DataSection; } +MCSection *TargetLoweringObjectFile::getSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, + const TargetMachine &TM) const { + return nullptr; +} + +MCSection *TargetLoweringObjectFile::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + return nullptr; +} + /// getTTypeGlobalReference - Return an MCExpr to use for a /// reference to the specified global variable from exception /// handling information. Index: llvm/test/CodeGen/X86/basicblock-sections-cold.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-cold.ll @@ -0,0 +1,41 @@ +; Check if basic blocks that don't get unique sections are placed in cold sections. +; Basic block with id 1 and 2 must be in the cold section. +; RUN: echo '!_Z3bazb' > %t +; RUN: echo '!!0' >> %t +; RUN: cat %t +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; Check that the basic block with id 1 doesn't get a section. +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique +; Check that a single cold section is started here and id 1 and 2 blocks are placed here. +; LINUX-SECTIONS: .section .text._Z3bazb.unlikely,"ax",@progbits +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .size _Z3bazb, .Lfunc_end{{[0-9]}}-_Z3bazb Index: llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll @@ -0,0 +1,38 @@ +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %9 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + %8 = trunc i32 %7 to i1 + br i1 %8, label %11, label %9 + +9: ; preds = %1 + %10 = call i32 @_Z3foov() + br label %11 + +11: ; preds = %9, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; LINUX-SECTIONS: jmp a.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.a.BB._Z3bazb,"ax",@progbits,unique,1 +; LINUX-SECTIONS: a.BB._Z3bazb: +; LINUX-SECTIONS: jmp aa.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.aa.BB._Z3bazb,"ax",@progbits,unique,2 +; LINUX-SECTIONS: aa.BB._Z3bazb: +; LINUX-SECTIONS: jmp raa.BB._Z3bazb Index: llvm/test/CodeGen/X86/basicblock-sections-eh.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-eh.ll @@ -0,0 +1,84 @@ +; Check if landing pads are kept in a separate eh section +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +@_ZTIb = external constant i8* +define i32 @_Z3foob(i1 zeroext %0) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = alloca i8* + %5 = alloca i32 + %6 = alloca i8, align 1 + %7 = zext i1 %0 to i8 + store i8 %7, i8* %3, align 1 + %8 = load i8, i8* %3, align 1 + %9 = trunc i8 %8 to i1 + br i1 %9, label %10, label %11 + +10: ; preds = %1 + store i32 1, i32* %2, align 4 + br label %31 + +11: ; preds = %1 + %12 = call i8* @__cxa_allocate_exception(i64 1) #2 + %13 = load i8, i8* %3, align 1 + %14 = trunc i8 %13 to i1 + %15 = zext i1 %14 to i8 + store i8 %15, i8* %12, align 16 + invoke void @__cxa_throw(i8* %12, i8* bitcast (i8** @_ZTIb to i8*), i8* null) #3 + to label %38 unwind label %16 + +16: ; preds = %11 + %17 = landingpad { i8*, i32 } + catch i8* bitcast (i8** @_ZTIb to i8*) + %18 = extractvalue { i8*, i32 } %17, 0 + store i8* %18, i8** %4, align 8 + %19 = extractvalue { i8*, i32 } %17, 1 + store i32 %19, i32* %5, align 4 + br label %20 + +20: ; preds = %16 + %21 = load i32, i32* %5, align 4 + %22 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIb to i8*)) #2 + %23 = icmp eq i32 %21, %22 + br i1 %23, label %24, label %33 + +24: ; preds = %20 + %25 = load i8*, i8** %4, align 8 + %26 = call i8* @__cxa_begin_catch(i8* %25) #2 + %27 = load i8, i8* %26, align 1 + %28 = trunc i8 %27 to i1 + %29 = zext i1 %28 to i8 + store i8 %29, i8* %6, align 1 + call void @__cxa_end_catch() #2 + br label %30 + +30: ; preds = %24 + store i32 0, i32* %2, align 4 + br label %31 + +31: ; preds = %30, %10 + %32 = load i32, i32* %2, align 4 + ret i32 %32 + +33: ; preds = %20 + %34 = load i8*, i8** %4, align 8 + %35 = load i32, i32* %5, align 4 + %36 = insertvalue { i8*, i32 } undef, i8* %34, 0 + %37 = insertvalue { i8*, i32 } %36, i32 %35, 1 + resume { i8*, i32 } %37 + +38: ; preds = %11 + unreachable +} +declare i8* @__cxa_allocate_exception(i64) +declare void @__cxa_throw(i8*, i8*, i8*) +declare i32 @__gxx_personality_v0(...) +; Function Attrs: nounwind readnone +declare i32 @llvm.eh.typeid.for(i8*) #1 +declare i8* @__cxa_begin_catch(i8*) +declare void @__cxa_end_catch() + +;LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits +;LINUX-SECTIONS: _Z3foob: +;LINUX-SECTIONS: .section .text._Z3foob.eh,"ax",@progbits +;LINUX-SECTIONS: l{{[a|r]*}}.BB._Z3foob: Index: llvm/test/CodeGen/X86/basicblock-sections-labels.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-labels.ll @@ -0,0 +1,33 @@ +; Check the basic block sections labels option +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=labels | FileCheck %s -check-prefix=LINUX-LABELS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +; LINUX-LABELS: .section +; LINUX-LABELS: _Z3bazb: +; LINUX-LABELS-NOT: .section +; LINUX-LABELS: r.BB._Z3bazb: +; LINUX-LABELS-NOT: .section +; LINUX-LABELS: rr.BB._Z3bazb: Index: llvm/test/CodeGen/X86/basicblock-sections-list.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-list.ll @@ -0,0 +1,76 @@ +; Check the basic block sections list option. +; RUN: echo '!_Z3foob' > %t +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define i32 @_Z3foob(i1 zeroext %0) #0 { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + %7 = zext i1 %6 to i32 + %8 = icmp sgt i32 %7, 0 + br i1 %8, label %9, label %11 + +9: ; preds = %1 + %10 = call i32 @_Z3barv() + store i32 %10, i32* %2, align 4 + br label %13 + +11: ; preds = %1 + %12 = call i32 @_Z3bazv() + store i32 %12, i32* %2, align 4 + br label %13 + +13: ; preds = %11, %9 + %14 = load i32, i32* %2, align 4 + ret i32 %14 +} + +declare i32 @_Z3barv() #1 +declare i32 @_Z3bazv() #1 + +define i32 @_Z3zipb(i1 zeroext %0) #0 { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + %7 = zext i1 %6 to i32 + %8 = icmp sgt i32 %7, 0 + br i1 %8, label %9, label %11 + +9: ; preds = %1 + %10 = call i32 @_Z3barv() + store i32 %10, i32* %2, align 4 + br label %13 + +11: ; preds = %1 + %12 = call i32 @_Z3bazv() + store i32 %12, i32* %2, align 4 + br label %13 + +13: ; preds = %11, %9 + %14 = load i32, i32* %2, align 4 + ret i32 %14 +} + +; LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits +; LINUX-SECTIONS: _Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.a.BB._Z3foob,"ax",@progbits,unique,1 +; LINUX-SECTIONS: a.BB._Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.aa.BB._Z3foob,"ax",@progbits,unique,2 +; LINUX-SECTIONS: aa.BB._Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.raa.BB._Z3foob,"ax",@progbits,unique,3 +; LINUX-SECTIONS: raa.BB._Z3foob: + +; LINUX-SECTIONS: .section .text._Z3zipb,"ax",@progbits +; LINUX-SECTIONS: _Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.a.BB._Z3zipb,"ax",@progbits,unique,1 +; LINUX-SECTIONS-NOT: a.BB._Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.aa.BB._Z3zipb,"ax",@progbits,unique,2 +; LINUX-SECTIONS-NOT: aa.BB._Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.raa.BB._Z3zipb,"ax",@progbits,unique,3 +; LINUX-SECTIONS-NOT: raa.BB._Z3zipb: Index: llvm/test/CodeGen/X86/basicblock-sections-listbb.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-listbb.ll @@ -0,0 +1,38 @@ +; Fine-grained basic block sections, subset of basic blocks in a function. +; Only basic block with id 2 must get a section. +; RUN: echo '!_Z3bazb' > %t +; RUN: echo '!!2' >> %t +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; Check that the basic block with id 1 doesn't get a section. +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb Index: llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir @@ -0,0 +1,131 @@ +# Start after bbsections0-prepare and check if the right code is generated. +# RUN: llc -start-after=bbsections-prepare %s -o - | FileCheck %s -check-prefix=CHECK + + +# How to generate the input: +# foo.cc +# int foo(bool k) { +# if (k) return 1; +# return 0; +# } +# +# clang -O0 -S -emit-llvm foo.cc +# llc < foo.ll -stop-after=bbsections-prepare -basicblock-sections=all + +--- | + ; Function Attrs: noinline nounwind optnone uwtable + define dso_local i32 @_Z3foob(i1 zeroext %0) #0 { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + br i1 %6, label %7, label %8 + + 7: ; preds = %1 + store i32 1, i32* %2, align 4 + br label %9 + + 8: ; preds = %1 + store i32 0, i32* %2, align 4 + br label %9 + + 9: ; preds = %8, %7 + %10 = load i32, i32* %2, align 4 + ret i32 %10 + } + + attributes #0 = { "frame-pointer"="all" "target-cpu"="x86-64" } + + +... +--- +name: _Z3foob +alignment: 16 +exposesReturnsTwice: false +legalized: false +regBankSelected: false +selected: false +failedISel: false +tracksRegLiveness: true +hasWinCFI: false +registers: [] +liveins: + - { reg: '$edi', virtual-reg: '' } +frameInfo: + isFrameAddressTaken: false + isReturnAddressTaken: false + hasStackMap: false + hasPatchPoint: false + stackSize: 8 + offsetAdjustment: -8 + maxAlignment: 4 + adjustsStack: false + hasCalls: false + stackProtector: '' + maxCallFrameSize: 0 + cvBytesOfCalleeSavedRegisters: 0 + hasOpaqueSPAdjustment: false + hasVAStart: false + hasMustTailInVarArgFunc: false + localFrameSize: 0 + savePoint: '' + restorePoint: '' +fixedStack: + - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default, + callee-saved-register: '', callee-saved-restored: true, debug-info-variable: '', + debug-info-expression: '', debug-info-location: '' } +stack: + - { id: 0, type: default, offset: -24, size: 4, + alignment: 4, stack-id: default, callee-saved-register: '', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } + - { id: 1, type: default, offset: -17, size: 1, + alignment: 1, stack-id: default, callee-saved-register: '', callee-saved-restored: true, + debug-info-variable: '', debug-info-expression: '', debug-info-location: '' } +callSites: [] +constants: [] +machineFunctionInfo: {} +body: | + bb.0 (%ir-block.1, bbsections Entry): + successors: %bb.2(0x40000000), %bb.1(0x40000000) + liveins: $edi + + frame-setup PUSH64r killed $rbp, implicit-def $rsp, implicit $rsp + CFI_INSTRUCTION def_cfa_offset 16 + CFI_INSTRUCTION offset $rbp, -16 + $rbp = frame-setup MOV64rr $rsp + CFI_INSTRUCTION def_cfa_register $rbp + renamable $dil = AND8ri renamable $dil, 1, implicit-def dead $eflags, implicit killed $edi, implicit-def $edi + MOV8mr $rbp, 1, $noreg, -1, $noreg, renamable $dil, implicit killed $edi :: (store 1 into %ir.3) + TEST8mi $rbp, 1, $noreg, -1, $noreg, 1, implicit-def $eflags :: (load 1 from %ir.3) + JCC_1 %bb.2, 4, implicit killed $eflags + JMP_1 %bb.1 + + bb.1 (%ir-block.7, bbsections Unique): + successors: %bb.3(0x80000000) + + MOV32mi $rbp, 1, $noreg, -8, $noreg, 1 :: (store 4 into %ir.2) + JMP_1 %bb.3 + + bb.2 (%ir-block.8, bbsections Unique): + successors: %bb.3(0x80000000) + + MOV32mi $rbp, 1, $noreg, -8, $noreg, 0 :: (store 4 into %ir.2) + JMP_1 %bb.3 + + bb.3 (%ir-block.9, bbsections Unique): + renamable $eax = MOV32rm $rbp, 1, $noreg, -8, $noreg :: (load 4 from %ir.2) + $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp + CFI_INSTRUCTION def_cfa $rsp, 8 + RETQ implicit $eax + +... + +# CHECK: _Z3foob: +# CHECK: .section .text,"ax",@progbits,unique +# CHECK: a.BB._Z3foob: +# CHECK: .section .text,"ax",@progbits,unique +# CHECK: aa.BB._Z3foob: +# CHECK: .section .text,"ax",@progbits,unique +# CHECK: aaa.BB._Z3foob: Index: llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll @@ -0,0 +1,32 @@ +; Stop after bbsections-prepare and check MIR output for section type. +; RUN: echo '!_Z3foob' > %t +; RUN: echo '!!1' >> %t +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -stop-after=bbsections-prepare | FileCheck %s -check-prefix=CHECK + +@_ZTIb = external constant i8* +define dso_local i32 @_Z3foob(i1 zeroext %0) { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + br i1 %6, label %7, label %8 + +7: ; preds = %1 + store i32 1, i32* %2, align 4 + br label %9 + +8: ; preds = %1 + store i32 0, i32* %2, align 4 + br label %9 + +9: ; preds = %8, %7 + %10 = load i32, i32* %2, align 4 + ret i32 %10 +} + +; CHECK: bbsections Entry +; CHECK: bbsections Cold +; CHECK: bbsections Cold +; CHECK: bbsections Unique Index: llvm/test/CodeGen/X86/basicblock-sections.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique,1 +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique,2 +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb