Index: llvm/include/llvm/CodeGen/MachineBasicBlock.h =================================================================== --- llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -129,10 +129,25 @@ /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; + /// Indicate that this basic block begins a new section. + bool IsBeginSection = false; + + /// Indicate that this basic block ends the current section. + bool IsEndSection = false; + + /// Indicate that this basic block belongs to the cold section. + bool IsColdSection = false; + + /// Indicate that this basic block belong to the exception section. + bool IsExceptionSection = false; + /// since getSymbol is a relatively heavy-weight operation, the symbol /// is only computed once and is cached. mutable MCSymbol *CachedMCSymbol = nullptr; + /// Used during basic block sections to mark the end of a basic block. + MCSymbol *EndMCSymbol = nullptr; + // Intrusive list support MachineBasicBlock() = default; @@ -408,6 +423,30 @@ /// Indicates if this is the entry block of a cleanup funclet. void setIsCleanupFuncletEntry(bool V = true) { IsCleanupFuncletEntry = V; } + /// Returns true if this block begins a section. + bool isBeginSection() const { return IsBeginSection; } + + /// Indicate that this basic block begins a new section. + void setBeginSection(bool V = true) { IsBeginSection = V; } + + /// Returns true if this block ends a section. + bool isEndSection() const { return IsEndSection; } + + /// Indicate that this basic block ends a section. + void setEndSection(bool V = true) { IsEndSection = V; } + + /// Returns true if this basic block belongs to the cold section. + bool isColdSection() const { return IsColdSection; } + + /// Indicate that this basic block belongs to the cold section. + void setColdSection(bool V = true) { IsColdSection = V; } + + /// Returns true if this basic block belongs to the exception section. + bool isExceptionSection() const { return IsExceptionSection; } + + /// Indicate that this basic block belongs to the exception section. + void setExceptionSection(bool V = true) { IsExceptionSection = V; } + /// Returns true if it is legal to hoist instructions into this block. bool isLegalToHoistInto() const; @@ -419,6 +458,15 @@ void moveBefore(MachineBasicBlock *NewAfter); void moveAfter(MachineBasicBlock *NewBefore); + /// Insert an unconditional jump to a fallthrough block if any. + void insertUnconditionalFallthroughBranch(); + + /// Returns true if this and MBB belong to the same section. + bool sameSection(const MachineBasicBlock *MBB) const; + + /// Returns the basic block that ends the section which contains this one. + const MachineBasicBlock *getSectionEndMBB() const; + /// Update the terminator instructions in block to account for changes to the /// layout. If the block previously used a fallthrough, it may now need a /// branch, and if it previously used branching it may now be able to use a @@ -803,6 +851,12 @@ /// Return the MCSymbol for this basic block. MCSymbol *getSymbol() const; + /// Sets the MCSymbol corresponding to the end of this basic block. + void setEndMCSymbol(MCSymbol *Sym) { EndMCSymbol = Sym; } + + /// Returns the MCSymbol corresponding to the end of this basic block. + MCSymbol *getEndMCSymbol() const { return EndMCSymbol; } + Optional getIrrLoopHeaderWeight() const { return IrrLoopHeaderWeight; } Index: llvm/include/llvm/CodeGen/MachineFunction.h =================================================================== --- llvm/include/llvm/CodeGen/MachineFunction.h +++ llvm/include/llvm/CodeGen/MachineFunction.h @@ -64,6 +64,7 @@ class MCContext; class MCInstrDesc; class MCSymbol; +class MCSection; class Pass; class PseudoSourceValueManager; class raw_ostream; @@ -243,6 +244,9 @@ // Keep track of jump tables for switch instructions MachineJumpTableInfo *JumpTableInfo; + // Keep track of the function section. + MCSection *Section = nullptr; + // Keeps track of Wasm exception handling related data. This will be null for // functions that aren't using a wasm EH personality. WasmEHFuncInfo *WasmEHInfo = nullptr; @@ -256,6 +260,12 @@ // numbered and this vector keeps track of the mapping from ID's to MBB's. std::vector MBBNumbering; + // Unary encoding of basic block symbols is used to reduce size of ".strtab". + // Basic block number 'i' gets a prefix of length 'i'. The ith character also + // denotes the type of basic block number 'i'. Return blocks are marked with + // 'r', landing pads with 'l' and regular blocks with 'a'. + std::vector MBBSymbolPrefix; + // Pool-allocate MachineFunction-lifetime and IR objects. BumpPtrAllocator Allocator; @@ -331,6 +341,13 @@ bool HasEHScopes = false; bool HasEHFunclets = false; + // True if basic block sections that are generated have been sorted. + bool BBSectionsSorted = false; + // True if sections must be generated for all basic blocks. + bool BBSections = false; + // True if labels must be generated for all basic blocks. + bool BasicBlockLabels = false; + /// List of C++ TypeInfo used. std::vector TypeInfos; @@ -447,6 +464,12 @@ MachineModuleInfo &getMMI() const { return MMI; } MCContext &getContext() const { return Ctx; } + /// Returns the Section this function belongs to. + MCSection *getSection() const { return Section; } + + /// Indicates the Section this function belongs to. + void setSection(MCSection *S) { Section = S; } + PseudoSourceValueManager &getPSVManager() const { return *PSVManager; } /// Return the DataLayout attached to the Module associated to this MF. @@ -461,6 +484,18 @@ /// getFunctionNumber - Return a unique ID for the current function. unsigned getFunctionNumber() const { return FunctionNumber; } + /// Returns true if this function has basic block sections enabled. + bool getBBSections() const { return BBSections; } + + /// Sort the basic blocks according to the sections they belong to. + bool sortBBSections(); + + /// Indicates that basic block Labels are to be generated for this function. + void setBasicBlockLabels(); + + /// Returns true if basic block labels are to be generated for this function. + bool getBasicBlockLabels() const { return BasicBlockLabels; } + /// getTarget - Return the target machine this machine code is compiled with const LLVMTargetMachine &getTarget() const { return Target; } @@ -1011,6 +1046,10 @@ /// of the instruction stream. void copyCallSiteInfo(const MachineInstr *Old, const MachineInstr *New); + + const std::vector &getMBBSymbolPrefix() const { + return MBBSymbolPrefix; + } }; //===--------------------------------------------------------------------===// Index: llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h =================================================================== --- llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -65,6 +65,15 @@ MCSection *getSectionForJumpTable(const Function &F, const TargetMachine &TM) const override; + MCSection * + getSectionForMachineBasicBlock(const Function &F, + const MachineBasicBlock &MBB, + const TargetMachine &TM) const override; + + MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const override; + bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference, const Function &F) const override; Index: llvm/include/llvm/Target/TargetLoweringObjectFile.h =================================================================== --- llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -24,6 +24,7 @@ namespace llvm { class GlobalValue; +class MachineBasicBlock; class MachineModuleInfo; class Mangler; class MCContext; @@ -90,6 +91,15 @@ const Constant *C, unsigned &Align) const; + virtual MCSection * + getSectionForMachineBasicBlock(const Function &F, + const MachineBasicBlock &MBB, + const TargetMachine &TM) const; + + virtual MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const; + /// Classify the specified global variable into a set of target independent /// categories embodied in SectionKind. static SectionKind getKindForGlobal(const GlobalObject *GO, Index: llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -664,7 +664,9 @@ EmitConstantPool(); // Print the 'header' of function. - OutStreamer->SwitchSection(getObjFileLowering().SectionForGlobal(&F, TM)); + MF->setSection(getObjFileLowering().SectionForGlobal(&F, TM)); + OutStreamer->SwitchSection(MF->getSection()); + EmitVisibility(CurrentFnSym, F.getVisibility()); if (MAI->needsFunctionDescriptors() && @@ -1058,6 +1060,19 @@ // Print out code for the function. bool HasAnyRealCode = false; int NumInstsInFunction = 0; + bool emitBBSections = MF->getBBSections(); + MachineBasicBlock *EndOfRegularSectionMBB = nullptr; + bool emitBBLabels = MF->getBBSections() || MF->getBasicBlockLabels(); + if (emitBBLabels) + MF->setBasicBlockLabels(); + if (emitBBSections) { + MF->sortBBSections(); + EndOfRegularSectionMBB = + const_cast(MF->front().getSectionEndMBB()); + assert(EndOfRegularSectionMBB->isEndSection() && + "The MBB at the end of the regular section must end a section"); + } + for (auto &MBB : *MF) { // Print a label for the basic block. EmitBasicBlockStart(MBB); @@ -1137,7 +1152,18 @@ } } } - + if (&MBB != EndOfRegularSectionMBB && + (MF->getBasicBlockLabels() || MBB.isEndSection())) { + // Emit size directive for the size of this basic block. Create a symbol + // for the end of the basic block. + MCSymbol *CurrentBBEnd = OutContext.createTempSymbol(); + const MCExpr *SizeExp = MCBinaryExpr::createSub( + MCSymbolRefExpr::create(CurrentBBEnd, OutContext), + MCSymbolRefExpr::create(MBB.getSymbol(), OutContext), OutContext); + OutStreamer->EmitLabel(CurrentBBEnd); + MBB.setEndMCSymbol(CurrentBBEnd); + OutStreamer->emitELFSize(MBB.getSymbol(), SizeExp); + } EmitBasicBlockEnd(MBB); } @@ -1171,6 +1197,10 @@ } } + // Switch to the original section if basic block sections was used. + if (emitBBSections) + OutStreamer->SwitchSection(MF->getSection()); + const Function &F = MF->getFunction(); for (const auto &BB : F) { if (!BB.hasAddressTaken()) @@ -1186,7 +1216,7 @@ EmitFunctionBodyEnd(); if (needFuncLabelsForEHOrDebugInfo(*MF, MMI) || - MAI->hasDotTypeDotSizeDirective()) { + MAI->hasDotTypeDotSizeDirective() || emitBBSections) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->EmitLabel(CurrentFnEnd); @@ -1209,6 +1239,9 @@ HI.Handler->markFunctionEnd(); } + if (emitBBSections) + EndOfRegularSectionMBB->setEndMCSymbol(CurrentFnEnd); + // Print out jump tables referenced by the function. EmitJumpTableInfo(); @@ -2933,6 +2966,7 @@ /// MachineBasicBlock, an alignment (if present) and a comment describing /// it if appropriate. void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) { + bool BBSections = MF->getBBSections(); // End the previous funclet and start a new one. if (MBB.isEHFuncletEntry()) { for (const HandlerInfo &HI : Handlers) { @@ -2942,9 +2976,11 @@ } // Emit an alignment directive for this block, if needed. - const Align Alignment = MBB.getAlignment(); - if (Alignment != Align::None()) - EmitAlignment(Alignment); + if (MBB.pred_empty() || !BBSections) { + const Align Alignment = MBB.getAlignment(); + if (Alignment != Align::None()) + EmitAlignment(Alignment); + } // If the block has its address taken, emit any labels that were used to // reference the block. It is possible that there is more than one label @@ -2976,18 +3012,41 @@ emitBasicBlockLoopComments(MBB, MLI, *this); } - // Print the main label for the block. + bool emitBBLabels = BBSections || MF->getBasicBlockLabels(); if (MBB.pred_empty() || - (isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry() && - !MBB.hasLabelMustBeEmitted())) { + (!emitBBLabels && isBlockOnlyReachableByFallthrough(&MBB) && + !MBB.isEHFuncletEntry() && !MBB.hasLabelMustBeEmitted())) { if (isVerbose()) { // NOTE: Want this comment at start of line, don't emit with AddComment. OutStreamer->emitRawComment(" %bb." + Twine(MBB.getNumber()) + ":", false); } } else { - if (isVerbose() && MBB.hasLabelMustBeEmitted()) + if (isVerbose() && MBB.hasLabelMustBeEmitted()) { OutStreamer->AddComment("Label of block must be emitted"); + } + // With -fbasicblock-sections, a basic block can start a new section. + if (MBB.isExceptionSection()) { + if (MF->front().isExceptionSection()) { + OutStreamer->SwitchSection(MF->getSection()); + } else { + // Create the exception section for this function. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".eh")); + } + } else if (MBB.isColdSection()) { + // Create the cold section here. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".cold")); + } else if (MBB.isBeginSection() && MBB.isEndSection()) { + OutStreamer->SwitchSection( + getObjFileLowering().getSectionForMachineBasicBlock(MF->getFunction(), + MBB, TM)); + } else if (BBSections) { + OutStreamer->SwitchSection(MF->getSection()); + } OutStreamer->EmitLabel(MBB.getSymbol()); } } @@ -3020,6 +3079,10 @@ /// the predecessor and this block is a fall-through. bool AsmPrinter:: isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { + // With BasicBlock Sections, no block is a fall through. + if (MBB->isBeginSection()) + return false; + // If this is a landing pad, it isn't a fall through. If it has no preds, // then nothing falls through to it. if (MBB->isEHPad() || MBB->pred_empty()) Index: llvm/lib/CodeGen/MachineBasicBlock.cpp =================================================================== --- llvm/lib/CodeGen/MachineBasicBlock.cpp +++ llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -35,6 +35,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include +#include using namespace llvm; #define DEBUG_TYPE "codegen" @@ -61,12 +62,31 @@ const MachineFunction *MF = getParent(); MCContext &Ctx = MF->getContext(); auto Prefix = Ctx.getAsmInfo()->getPrivateLabelPrefix(); + + bool BasicBlockSymbols = MF->getBBSections() || MF->getBasicBlockLabels(); + auto Delimiter = BasicBlockSymbols ? "." : "_"; assert(getNumber() >= 0 && "cannot get label for unreachable MBB"); - CachedMCSymbol = Ctx.getOrCreateSymbol(Twine(Prefix) + "BB" + - Twine(MF->getFunctionNumber()) + - "_" + Twine(getNumber())); - } + // With Basic Block Sections, we emit a symbol for every basic block. To + // keep the size of strtab small, we choose a unary encoding which can + // compress the symbol names significantly. The basic blocks for function + // foo are named a.BB.foo, aa.BB.foo, and so on. + if (BasicBlockSymbols) { + auto Iter = MF->getMBBSymbolPrefix().begin(); + if (getNumber() < 0 || + getNumber() >= (int)MF->getMBBSymbolPrefix().size()) + report_fatal_error("Unreachable MBB: " + Twine(getNumber())); + std::string Prefix(Iter + 1, Iter + getNumber() + 1); + std::reverse(Prefix.begin(), Prefix.end()); + CachedMCSymbol = + Ctx.getOrCreateSymbol(Prefix + Twine(Delimiter) + "BB" + + Twine(Delimiter) + Twine(MF->getName())); + } else { + CachedMCSymbol = Ctx.getOrCreateSymbol( + Twine(Prefix) + "BB" + Twine(MF->getFunctionNumber()) + + Twine(Delimiter) + Twine(getNumber())); + } + } return CachedMCSymbol; } @@ -529,6 +549,64 @@ getParent()->splice(++NewBefore->getIterator(), getIterator()); } +// Returns true if this basic block and the Other are in the same section. +bool MachineBasicBlock::sameSection(const MachineBasicBlock *Other) const { + if (this == Other) + return true; + + if ((this->isColdSection() != Other->isColdSection()) || + (this->isExceptionSection() != Other->isExceptionSection())) + return false; + + if ((this->isBeginSection() && this->isEndSection()) || + (Other->isBeginSection() && Other->isEndSection())) + return false; + + return true; +} + +const MachineBasicBlock *MachineBasicBlock::getSectionEndMBB() const { + if (this->isEndSection()) + return this; + auto I = std::next(this->getIterator()); + const MachineFunction *MF = getParent(); + while (I != MF->end()) { + const MachineBasicBlock &MBB = *I; + if (MBB.isEndSection()) + return &MBB; + I = std::next(I); + } + llvm_unreachable("No End Basic Block for this section."); +} + +// Insert unconditional jumps to the basic block to which there is +// a fall through. +void MachineBasicBlock::insertUnconditionalFallthroughBranch() { + MachineBasicBlock *Fallthrough = getFallThrough(); + + if (Fallthrough == nullptr) + return; + + // If this basic block and the Fallthrough basic block are in the same + // section then do not insert the jump. + if (this->sameSection(Fallthrough)) + return; + + const TargetInstrInfo *TII = getParent()->getSubtarget().getInstrInfo(); + SmallVector Cond; + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; + + // If a branch to the fall through block already exists, return. + if (!TII->analyzeBranch(*this, TBB, FBB, Cond) && + (TBB == Fallthrough || FBB == Fallthrough)) { + return; + } + + Cond.clear(); + DebugLoc DL = findBranchDebugLoc(); + TII->insertBranch(*this, Fallthrough, nullptr, Cond, DL); +} + void MachineBasicBlock::updateTerminator() { const TargetInstrInfo *TII = getParent()->getSubtarget().getInstrInfo(); // A block with no successors has no concerns with fall-through edges. Index: llvm/lib/CodeGen/MachineFunction.cpp =================================================================== --- llvm/lib/CodeGen/MachineFunction.cpp +++ llvm/lib/CodeGen/MachineFunction.cpp @@ -33,6 +33,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" #include "llvm/CodeGen/TargetFrameLowering.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -71,6 +72,7 @@ #include #include #include +#include #include #include @@ -181,6 +183,18 @@ Alignment = std::max(Alignment, STI->getTargetLowering()->getPrefFunctionAlignment()); + // Check if basic block sections are required for this function. + if (Target.getBBSections() == llvm::BasicBlockSection::All || + F.getBBSections() || + (Target.getBBSections() == llvm::BasicBlockSection::List && + Target.isFunctionInBBSectionsList(F.getName()))) + BBSections = true; + + // Check if basic block labels are required for this function. + if (Target.getBBSections() == llvm::BasicBlockSection::Labels || + F.getBasicBlockLabels()) + BasicBlockLabels = true; + if (AlignAllFunctions) Alignment = Align(1ULL << AlignAllFunctions); @@ -337,6 +351,129 @@ MBBNumbering.resize(BlockNo); } +/// This function sorts basic blocks according to the sections in which they are +/// emitted. Basic block sections automatically turn on function sections so +/// the entry block is in the function section. The other sections that are +/// created are: +/// 1) Exception section - basic blocks that are landing pads +/// 2) Cold section - basic blocks that will not have unique sections. +/// 3) Unique section - one per basic block that is emitted in a unique section. +bool MachineFunction::sortBBSections() { + // This should only be done once no matter how many times it is called. + if (this->BBSectionsSorted || !this->getBBSections()) + return false; + + DenseMap MBBOrder; + unsigned MBBOrderN = 0; + + SmallSet S = Target.getBBSectionsSet(F.getName()); + for (auto &MBB : *this) { + // A unique BB section can only be created if this basic block is not + // used for exception table computations. Entry basic block cannot + // start another section because the function starts one already. + if (MBB.getNumber() == this->front().getNumber()) { + if (MBB.isEHPad()) + MBB.setExceptionSection(); + continue; + } + // Also, check if this BB is a cold basic block in which case sections + // are not required with the list option. + bool isColdBB = + ((Target.getBBSections() == llvm::BasicBlockSection::List) && + !S.empty() && !S.count(MBB.getNumber())); + if (MBB.isEHPad()) { + MBB.setExceptionSection(); + } else if (isColdBB) { + MBB.setColdSection(); + } else { + // Place this MBB in a unique section. A unique section begins and ends + // that section. + MBB.setBeginSection(); + MBB.setEndSection(); + } + MBBOrder[&MBB] = MBBOrderN++; + } + + // With -fbasicblock-sections, fall through blocks must be made + // explicitly reachable. Do this after sections is set as + // unnecessary fallthroughs can be avoided. + for (auto &MBB : *this) { + MBB.insertUnconditionalFallthroughBranch(); + } + + // Order : Entry Block, Exception Section, Cold Section, + // Other Unique Sections. + auto SectionType = ([&](MachineBasicBlock &X) { + if (X.getNumber() == this->front().getNumber() && !X.isExceptionSection()) + return 0; + if (X.isExceptionSection()) + return 1; + else if (X.isColdSection()) + return 2; + return 3; + }); + + this->sort(([&](MachineBasicBlock &X, MachineBasicBlock &Y) { + auto TypeX = SectionType(X); + auto TypeY = SectionType(Y); + + return (TypeX != TypeY) ? TypeX < TypeY : MBBOrder[&X] < MBBOrder[&Y]; + })); + + // Set the basic block that begins or ends every section. For unique + // sections, the same basic block begins and ends it. + MachineBasicBlock *PrevMBB = nullptr; + for (auto &MBB : *this) { + // Entry block + if (MBB.getNumber() == this->front().getNumber()) { + PrevMBB = &MBB; + continue; + } + assert(PrevMBB != nullptr && "First block was not a regular block!"); + int TypeP = SectionType(*PrevMBB); + int TypeT = SectionType(MBB); + if (TypeP != TypeT) { + PrevMBB->setEndSection(); + MBB.setBeginSection(); + } + assert((TypeT != 3 || (MBB.isBeginSection() && MBB.isEndSection())) && + "Basic block does not correctly begin or end a section"); + PrevMBB = &MBB; + } + PrevMBB->setEndSection(); + + this->BBSectionsSorted = true; + return true; +} + +/// This is used with -fbasicblock-sections or -fbasicblock-labels option. +/// A unary encoding of basic block labels is done to keep ".strtab" sizes +/// small. +void MachineFunction::setBasicBlockLabels() { + const TargetInstrInfo *TII = getSubtarget().getInstrInfo(); + this->MBBSymbolPrefix.resize(getNumBlockIDs(), 'a'); + for (auto MBBI = begin(), E = end(); MBBI != E; ++MBBI) { + if (MBBI->getNumber() < 0 || MBBI->getNumber() >= (int)getNumBlockIDs()) + report_fatal_error( + "BasicBlock number was out of range: " + Twine(MBBI->getNumber()) + + " [0 -> " + Twine(getNumBlockIDs()) + "]"); + // 'a' - Normal block. + // 'r' - Return block. + // 'l' - Landing Pad. + // 'L' - Return and landing pad. + bool isEHPad = MBBI->isEHPad(); + bool isRetBlock = MBBI->isReturnBlock() && !TII->isTailCall(MBBI->back()); + char type = 'a'; + if (isEHPad && isRetBlock) + type = 'L'; + else if (isEHPad) + type = 'l'; + else if (isRetBlock) + type = 'r'; + MBBSymbolPrefix[MBBI->getNumber()] = type; + } +} + /// Allocate a new MachineInstr. Use this instead of `new MachineInstr'. MachineInstr *MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, Index: llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp =================================================================== --- llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -21,6 +21,8 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/IR/Comdat.h" @@ -52,8 +54,8 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" -#include "llvm/Support/Format.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include @@ -751,6 +753,55 @@ return DataRelROSection; } +/// Returns a unique section for the given machine basic block. +MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, + const TargetMachine &TM) const { + SmallString<128> Name; + Name = (static_cast(MBB.getParent()->getSection())) + ->getSectionName(); + if (TM.getUniqueBBSectionNames()) { + Name += "."; + Name += MBB.getSymbol()->getName(); + } + unsigned UniqueID = NextUniqueID++; + unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; + std::string GroupName = ""; + if (F.hasComdat()) { + Flags |= ELF::SHF_GROUP; + GroupName = F.getComdat()->getName(); + } + return getContext().getELFSection(Name, ELF::SHT_PROGBITS, Flags, + 0 /* Entry Size */, GroupName, UniqueID); +} + +MCSection *TargetLoweringObjectFileELF::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + SmallString<128> Name; + Name = (static_cast(MBB.getParent()->getSection())) + ->getSectionName(); + + // If unique section names is off, explicity add the function name to the + // section name to make sure named sections for functions are unique + // across the module. + if (!TM.getUniqueSectionNames()) { + Name += "."; + Name += MBB.getParent()->getName(); + } + + Name += Suffix; + + unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; + std::string GroupName = ""; + if (F.hasComdat()) { + Flags |= ELF::SHF_GROUP; + GroupName = F.getComdat()->getName(); + } + return getContext().getELFSection(Name, ELF::SHT_PROGBITS, Flags, + 0 /* Entry Size */, GroupName); +} + static MCSectionELF *getStaticStructorSection(MCContext &Ctx, bool UseInitArray, bool IsCtor, unsigned Priority, const MCSymbol *KeySym) { Index: llvm/lib/Target/TargetLoweringObjectFile.cpp =================================================================== --- llvm/lib/Target/TargetLoweringObjectFile.cpp +++ llvm/lib/Target/TargetLoweringObjectFile.cpp @@ -149,6 +149,10 @@ if (isa(GO)) return SectionKind::getText(); + // Basic blocks are classified as text sections. + if (isa(GO)) + return SectionKind::getText(); + // Global variables require more detailed analysis. const auto *GVar = cast(GO); @@ -302,6 +306,18 @@ return DataSection; } +MCSection *TargetLoweringObjectFile::getSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, + const TargetMachine &TM) const { + return nullptr; +} + +MCSection *TargetLoweringObjectFile::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + return nullptr; +} + /// getTTypeGlobalReference - Return an MCExpr to use for a /// reference to the specified global variable from exception /// handling information. Index: llvm/test/CodeGen/X86/basicblock-sections-eh.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-eh.ll @@ -0,0 +1,84 @@ +; Check if landing pads are kept in a separate eh section +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +@_ZTIb = external constant i8* +define i32 @_Z3foob(i1 zeroext %0) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = alloca i8* + %5 = alloca i32 + %6 = alloca i8, align 1 + %7 = zext i1 %0 to i8 + store i8 %7, i8* %3, align 1 + %8 = load i8, i8* %3, align 1 + %9 = trunc i8 %8 to i1 + br i1 %9, label %10, label %11 + +10: ; preds = %1 + store i32 1, i32* %2, align 4 + br label %31 + +11: ; preds = %1 + %12 = call i8* @__cxa_allocate_exception(i64 1) #2 + %13 = load i8, i8* %3, align 1 + %14 = trunc i8 %13 to i1 + %15 = zext i1 %14 to i8 + store i8 %15, i8* %12, align 16 + invoke void @__cxa_throw(i8* %12, i8* bitcast (i8** @_ZTIb to i8*), i8* null) #3 + to label %38 unwind label %16 + +16: ; preds = %11 + %17 = landingpad { i8*, i32 } + catch i8* bitcast (i8** @_ZTIb to i8*) + %18 = extractvalue { i8*, i32 } %17, 0 + store i8* %18, i8** %4, align 8 + %19 = extractvalue { i8*, i32 } %17, 1 + store i32 %19, i32* %5, align 4 + br label %20 + +20: ; preds = %16 + %21 = load i32, i32* %5, align 4 + %22 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIb to i8*)) #2 + %23 = icmp eq i32 %21, %22 + br i1 %23, label %24, label %33 + +24: ; preds = %20 + %25 = load i8*, i8** %4, align 8 + %26 = call i8* @__cxa_begin_catch(i8* %25) #2 + %27 = load i8, i8* %26, align 1 + %28 = trunc i8 %27 to i1 + %29 = zext i1 %28 to i8 + store i8 %29, i8* %6, align 1 + call void @__cxa_end_catch() #2 + br label %30 + +30: ; preds = %24 + store i32 0, i32* %2, align 4 + br label %31 + +31: ; preds = %30, %10 + %32 = load i32, i32* %2, align 4 + ret i32 %32 + +33: ; preds = %20 + %34 = load i8*, i8** %4, align 8 + %35 = load i32, i32* %5, align 4 + %36 = insertvalue { i8*, i32 } undef, i8* %34, 0 + %37 = insertvalue { i8*, i32 } %36, i32 %35, 1 + resume { i8*, i32 } %37 + +38: ; preds = %11 + unreachable +} +declare i8* @__cxa_allocate_exception(i64) +declare void @__cxa_throw(i8*, i8*, i8*) +declare i32 @__gxx_personality_v0(...) +; Function Attrs: nounwind readnone +declare i32 @llvm.eh.typeid.for(i8*) #1 +declare i8* @__cxa_begin_catch(i8*) +declare void @__cxa_end_catch() + +;LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits +;LINUX-SECTIONS: _Z3foob: +;LINUX-SECTIONS: .section .text._Z3foob.eh,"ax",@progbits +;LINUX-SECTIONS: l{{[a|r]*}}.BB._Z3foob: Index: llvm/test/CodeGen/X86/basicblock-sections-labels.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-labels.ll @@ -0,0 +1,33 @@ +; Check the basic block sections labels option +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=labels | FileCheck %s -check-prefix=LINUX-LABELS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +; LINUX-LABELS: .section +; LINUX-LABELS: _Z3bazb: +; LINUX-LABELS-NOT: .section +; LINUX-LABELS: r.BB._Z3bazb: +; LINUX-LABELS-NOT: .section +; LINUX-LABELS: rr.BB._Z3bazb: Index: llvm/test/CodeGen/X86/basicblock-sections-list.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-list.ll @@ -0,0 +1,76 @@ +; Check the basic block sections list option. +; RUN: echo '!_Z3foob' > %t +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define i32 @_Z3foob(i1 zeroext %0) #0 { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + %7 = zext i1 %6 to i32 + %8 = icmp sgt i32 %7, 0 + br i1 %8, label %9, label %11 + +9: ; preds = %1 + %10 = call i32 @_Z3barv() + store i32 %10, i32* %2, align 4 + br label %13 + +11: ; preds = %1 + %12 = call i32 @_Z3bazv() + store i32 %12, i32* %2, align 4 + br label %13 + +13: ; preds = %11, %9 + %14 = load i32, i32* %2, align 4 + ret i32 %14 +} + +declare i32 @_Z3barv() #1 +declare i32 @_Z3bazv() #1 + +define i32 @_Z3zipb(i1 zeroext %0) #0 { + %2 = alloca i32, align 4 + %3 = alloca i8, align 1 + %4 = zext i1 %0 to i8 + store i8 %4, i8* %3, align 1 + %5 = load i8, i8* %3, align 1 + %6 = trunc i8 %5 to i1 + %7 = zext i1 %6 to i32 + %8 = icmp sgt i32 %7, 0 + br i1 %8, label %9, label %11 + +9: ; preds = %1 + %10 = call i32 @_Z3barv() + store i32 %10, i32* %2, align 4 + br label %13 + +11: ; preds = %1 + %12 = call i32 @_Z3bazv() + store i32 %12, i32* %2, align 4 + br label %13 + +13: ; preds = %11, %9 + %14 = load i32, i32* %2, align 4 + ret i32 %14 +} + +; LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits +; LINUX-SECTIONS: _Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.a.BB._Z3foob,"ax",@progbits,unique,1 +; LINUX-SECTIONS: a.BB._Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.aa.BB._Z3foob,"ax",@progbits,unique,2 +; LINUX-SECTIONS: aa.BB._Z3foob: +; LINUX-SECTIONS: .section .text._Z3foob.raa.BB._Z3foob,"ax",@progbits,unique,3 +; LINUX-SECTIONS: raa.BB._Z3foob: + +; LINUX-SECTIONS: .section .text._Z3zipb,"ax",@progbits +; LINUX-SECTIONS: _Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.a.BB._Z3zipb,"ax",@progbits,unique,1 +; LINUX-SECTIONS-NOT: a.BB._Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.aa.BB._Z3zipb,"ax",@progbits,unique,2 +; LINUX-SECTIONS-NOT: aa.BB._Z3zipb: +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.raa.BB._Z3zipb,"ax",@progbits,unique,3 +; LINUX-SECTIONS-NOT: raa.BB._Z3zipb: Index: llvm/test/CodeGen/X86/basicblock-sections-listbb.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections-listbb.ll @@ -0,0 +1,38 @@ +; Fine-grained basic block sections, subset of basic blocks in a function. +; Only basic block with id 2 must get a section. +; RUN: echo '!_Z3bazb' > %t +; RUN: echo '!!2' >> %t +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; Check that the basic block with id 1 doesn't get a section. +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb Index: llvm/test/CodeGen/X86/basicblock-sections.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/X86/basicblock-sections.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basicblock-sections=all -unique-bb-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS + +define void @_Z3bazb(i1 zeroext) { + %2 = alloca i8, align 1 + %3 = zext i1 %0 to i8 + store i8 %3, i8* %2, align 1 + %4 = load i8, i8* %2, align 1 + %5 = trunc i8 %4 to i1 + br i1 %5, label %6, label %8 + +6: ; preds = %1 + %7 = call i32 @_Z3barv() + br label %10 + +8: ; preds = %1 + %9 = call i32 @_Z3foov() + br label %10 + +10: ; preds = %8, %6 + ret void +} + +declare i32 @_Z3barv() #1 + +declare i32 @_Z3foov() #1 + + +; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: _Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique,1 +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique,2 +; LINUX-SECTIONS: rr.BB._Z3bazb: +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb