Index: include/llvm/CodeGen/AsmPrinter.h =================================================================== --- include/llvm/CodeGen/AsmPrinter.h +++ include/llvm/CodeGen/AsmPrinter.h @@ -112,6 +112,12 @@ MCSymbol *CurrentFnEnd; MCSymbol *CurExceptionSym; + MCSymbol *CurrentFnColdBegin; + MCSymbol *CurrentFnColdEnd; + MCSymbol *CurColdExceptionSym; + + const MachineBasicBlock *ColdFragmentStart; + // The garbage collection metadata printer table. void *GCMetadataPrinters; // Really a DenseMap. @@ -158,6 +164,13 @@ MCSymbol *getFunctionBegin() const { return CurrentFnBegin; } MCSymbol *getFunctionEnd() const { return CurrentFnEnd; } MCSymbol *getCurExceptionSym(); + MCSymbol *getFunctionColdBegin() const { return CurrentFnColdBegin; } + MCSymbol *getFunctionColdEnd() const { return CurrentFnColdEnd; } + MCSymbol *getCurColdExceptionSym(); + + const MachineBasicBlock *getColdFragmentStart() const { + return ColdFragmentStart; + } /// Return information about object file lowering. const TargetLoweringObjectFile &getObjFileLowering() const; Index: include/llvm/MC/MCObjectFileInfo.h =================================================================== --- include/llvm/MC/MCObjectFileInfo.h +++ include/llvm/MC/MCObjectFileInfo.h @@ -53,6 +53,9 @@ /// Section directive for standard text. MCSection *TextSection; + /// Section directive for cold text. + MCSection *ColdTextSection; + /// Section directive for standard data. MCSection *DataSection; @@ -224,6 +227,7 @@ } MCSection *getTextSection() const { return TextSection; } + MCSection *getColdTextSection() const { return ColdTextSection; } MCSection *getDataSection() const { return DataSection; } MCSection *getBSSSection() const { return BSSSection; } MCSection *getReadOnlySection() const { return ReadOnlySection; } Index: include/llvm/MC/MCStreamer.h =================================================================== --- include/llvm/MC/MCStreamer.h +++ include/llvm/MC/MCStreamer.h @@ -223,6 +223,8 @@ } bool hasUnfinishedDwarfFrameInfo(); + void pushDwarfFrameInfo(MCDwarfFrameInfo DFI); + MCDwarfFrameInfo popDwarfFrameInfo(); unsigned getNumWinFrameInfos() { return WinFrameInfos.size(); } ArrayRef getWinFrameInfos() const { Index: include/llvm/MC/MCTargetOptions.h =================================================================== --- include/llvm/MC/MCTargetOptions.h +++ include/llvm/MC/MCTargetOptions.h @@ -36,6 +36,7 @@ bool ShowMCEncoding : 1; bool ShowMCInst : 1; bool AsmVerbose : 1; + bool SplitColdCode : 1; int DwarfVersion; /// getABIName - If this returns a non-empty string this represents the /// textual name of the ABI that we want the backend to use, e.g. o32, or @@ -58,6 +59,7 @@ ARE_EQUAL(ShowMCEncoding) && ARE_EQUAL(ShowMCInst) && ARE_EQUAL(AsmVerbose) && + ARE_EQUAL(SplitColdCode) && ARE_EQUAL(DwarfVersion) && ARE_EQUAL(ABIName)); #undef ARE_EQUAL Index: include/llvm/MC/MCTargetOptionsCommandFlags.h =================================================================== --- include/llvm/MC/MCTargetOptionsCommandFlags.h +++ include/llvm/MC/MCTargetOptionsCommandFlags.h @@ -46,6 +46,9 @@ cl::desc("Emit internal instruction representation to " "assembly file")); +cl::opt SplitColdCode("split-cold-code", + cl::desc("Emit cold code in a different section")); + cl::opt FatalWarnings("fatal-warnings", cl::desc("Treat warnings as errors")); @@ -66,6 +69,7 @@ Options.DwarfVersion = DwarfVersion; Options.ShowMCInst = ShowMCInst; Options.ABIName = ABIName; + Options.SplitColdCode = SplitColdCode; Options.MCFatalWarnings = FatalWarnings; Options.MCNoWarn = NoWarn; return Options; Index: include/llvm/MC/SectionKind.h =================================================================== --- include/llvm/MC/SectionKind.h +++ include/llvm/MC/SectionKind.h @@ -28,6 +28,9 @@ /// Text - Text section, used for functions and other executable code. Text, + /// Text - Text section, used for functions and other rarely executed code. + ColdText, + /// ReadOnly - Data that is never written to at program runtime by the /// program or the dynamic linker. Things in the top-level readonly /// SectionKind are not mergeable. @@ -113,6 +116,7 @@ bool isMetadata() const { return K == Metadata; } bool isText() const { return K == Text; } + bool isColdText() const { return K == ColdText; } bool isReadOnly() const { return K == ReadOnly || isMergeableCString() || @@ -172,6 +176,7 @@ static SectionKind getMetadata() { return get(Metadata); } static SectionKind getText() { return get(Text); } + static SectionKind getColdText() { return get(ColdText); } static SectionKind getReadOnly() { return get(ReadOnly); } static SectionKind getMergeable1ByteCString() { return get(Mergeable1ByteCString); Index: lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -21,6 +21,7 @@ #include "llvm/Analysis/ConstantFolding.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GCMetadataPrinter.h" +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -110,6 +111,9 @@ CurExceptionSym = CurrentFnSym = CurrentFnSymForSize = nullptr; CurrentFnBegin = nullptr; CurrentFnEnd = nullptr; + CurrentFnColdBegin = nullptr; + CurrentFnColdEnd = nullptr; + ColdFragmentStart = nullptr; GCMetadataPrinters = nullptr; VerboseAsm = OutStreamer->isVerboseAsm(); } @@ -170,6 +174,8 @@ AU.addRequired(); if (isVerbose()) AU.addRequired(); + if (TM.Options.MCOptions.SplitColdCode) + AU.addRequired(); } bool AsmPrinter::doInitialization(Module &M) { @@ -837,6 +843,18 @@ MCConstantExpr::create(FrameOffset, OutContext)); } +static bool isReachedByFallthrough(const MachineBasicBlock &MBB) { + // The first BB cannot be accessed via fallthrough. + if (&*MBB.getParent()->begin() == &MBB) + return false; + MachineFunction::iterator I(const_cast(&MBB)); + return (--I)->canFallThrough(); +} + +static MCSymbol *getColdExceptionSym(AsmPrinter *Asm) { + return Asm->getCurColdExceptionSym(); +} + /// EmitFunctionBody - This method emits the body and trailer for a /// function. void AsmPrinter::EmitFunctionBody() { @@ -847,9 +865,48 @@ bool ShouldPrintDebugScopes = MMI->hasDebugInfo(); + auto *HotSection = const_cast(getCurrentSection()); + auto *ColdSection = getObjFileLowering().getColdTextSection(); + auto *CurSection = HotSection; + + MachineBlockFrequencyInfo *MBFI; + uint64_t EntryFreq; + + bool SplitColdCode = ColdSection && TM.Options.MCOptions.SplitColdCode; + if (SplitColdCode) { + MBFI = &getAnalysis(); + EntryFreq = MBFI->getEntryFreq(); + } + // Print out code for the function. bool HasAnyRealCode = false; for (auto &MBB : *MF) { + if (SplitColdCode && CurSection != ColdSection && + !isReachedByFallthrough(MBB)) { + bool IsCold = MBB.isEHPad(); + if (!IsCold) { + auto BP = BranchProbability::getBranchProbability( + MBFI->getBlockFreq(&MBB).getFrequency(), EntryFreq); + uint64_t ColdThresold = 1 << 14; + IsCold = BP.getNumerator() < ColdThresold; + } + + if (IsCold) { + CurSection = ColdSection; + ColdFragmentStart = &MBB; + OutStreamer->SwitchSection(ColdSection); + MCSymbol *ColdFnSym = + OutContext.getOrCreateSymbol(CurrentFnSym->getName() + "$cold"); + OutStreamer->EmitLabel(ColdFnSym); + CurrentFnColdBegin = createTempSymbol("func_cold_begin"); + OutStreamer->EmitLabel(CurrentFnColdBegin); + + for (const HandlerInfo &HI : Handlers) { + HI.Handler->beginFragment(&MBB, getColdExceptionSym); + } + } + } + // Print a label for the basic block. EmitBasicBlockStart(MBB); for (auto &MI : MBB) { @@ -917,6 +974,17 @@ EmitBasicBlockEnd(MBB); } + if (CurSection == ColdSection) { + CurSection = HotSection; + CurrentFnColdEnd = createTempSymbol("func_cold_end"); + OutStreamer->EmitLabel(CurrentFnColdEnd); + for (const HandlerInfo &HI : Handlers) { + HI.Handler->endFragment(); + } + + OutStreamer->SwitchSection(HotSection); + } + // If the function is empty and the object file uses .subsections_via_symbols, // then we need to emit *something* to the function body to prevent the // labels from collapsing together. Just emit a noop. @@ -1235,6 +1303,12 @@ return CurExceptionSym; } +MCSymbol *AsmPrinter::getCurColdExceptionSym() { + if (!CurColdExceptionSym) + CurColdExceptionSym = createTempSymbol("cold_exception"); + return CurColdExceptionSym; +} + void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { this->MF = &MF; // Get the function symbol. Index: lib/CodeGen/AsmPrinter/DwarfCFIException.cpp =================================================================== --- lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -136,6 +136,8 @@ /// endFunction - Gather and emit post-function exception information. /// void DwarfCFIException::endFunction(const MachineFunction *) { + assert(InFlightCFIs.empty() && "Remaining inflight CFI"); + if (!shouldEmitPersonality) return; @@ -147,33 +149,43 @@ if (!shouldEmitCFI) return; + bool IsSubFragment = Asm->OutStreamer->hasUnfinishedDwarfFrameInfo(); + if (IsSubFragment) + InFlightCFIs.push_back(Asm->OutStreamer->popDwarfFrameInfo()); + Asm->OutStreamer->EmitCFIStartProc(/*IsSimple=*/false); // Indicate personality routine, if any. - if (!shouldEmitPersonality) - return; - - auto *F = MBB->getParent()->getFunction(); - auto *P = dyn_cast(F->getPersonalityFn()->stripPointerCasts()); - assert(P && "Expected personality function"); - - // If we are forced to emit this personality, make sure to record - // it because it might not appear in any landingpad - if (forceEmitPersonality) - MMI->addPersonality(P); - - const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); - unsigned PerEncoding = TLOF.getPersonalityEncoding(); - const MCSymbol *Sym = - TLOF.getCFIPersonalitySymbol(P, *Asm->Mang, Asm->TM, MMI); - Asm->OutStreamer->EmitCFIPersonality(Sym, PerEncoding); + if (shouldEmitPersonality) { + auto *F = MBB->getParent()->getFunction(); + auto *P = dyn_cast(F->getPersonalityFn()->stripPointerCasts()); + assert(P && "Expected personality function"); + + // If we are forced to emit this personality, make sure to record + // it because it might not appear in any landingpad + if (forceEmitPersonality) + MMI->addPersonality(P); + + const TargetLoweringObjectFile &TLOF = Asm->getObjFileLowering(); + unsigned PerEncoding = TLOF.getPersonalityEncoding(); + const MCSymbol *Sym = + TLOF.getCFIPersonalitySymbol(P, *Asm->Mang, Asm->TM, MMI); + Asm->OutStreamer->EmitCFIPersonality(Sym, PerEncoding); + + // Provide LSDA information. + if (shouldEmitLSDA) + Asm->OutStreamer->EmitCFILsda(ESP(Asm), TLOF.getLSDAEncoding()); + } - // Provide LSDA information. - if (shouldEmitLSDA) - Asm->OutStreamer->EmitCFILsda(ESP(Asm), TLOF.getLSDAEncoding()); + // There must be a better way to do this, but it will do for now. + if (IsSubFragment) + for (const MCCFIInstruction &I : InFlightCFIs.back().Instructions) + Asm->emitCFIInstruction(I); } void DwarfCFIException::endFragment() { if (shouldEmitCFI) Asm->OutStreamer->EmitCFIEndProc(); + if (InFlightCFIs.size()) + Asm->OutStreamer->pushDwarfFrameInfo(InFlightCFIs.pop_back_val()); } Index: lib/CodeGen/AsmPrinter/DwarfException.h =================================================================== --- lib/CodeGen/AsmPrinter/DwarfException.h +++ lib/CodeGen/AsmPrinter/DwarfException.h @@ -47,6 +47,8 @@ AsmPrinter::CFIMoveType moveTypeModule; + SmallVector InFlightCFIs; + public: //===--------------------------------------------------------------------===// // Main entry points. Index: lib/CodeGen/AsmPrinter/EHStreamer.h =================================================================== --- lib/CodeGen/AsmPrinter/EHStreamer.h +++ lib/CodeGen/AsmPrinter/EHStreamer.h @@ -84,9 +84,9 @@ /// zero for the landing pad and the action. Calls marked 'nounwind' have /// no entry and must not be contained in the try-range of any entry - they /// form gaps in the table. Entries must be ordered by try-range address. - void computeCallSiteTable(SmallVectorImpl &CallSites, - const SmallVectorImpl &LPs, - const SmallVectorImpl &FirstActions); + unsigned computeCallSiteTable(SmallVectorImpl &CallSites, + const SmallVectorImpl &LPs, + const SmallVectorImpl &FirstActions); /// Emit landing pads and actions. /// @@ -109,6 +109,10 @@ /// catches in the function. This tables is reversed indexed base 1. void emitExceptionTable(); + void emitLSDAHeader(MCSymbol *LPStart, unsigned SubCallSiteTableLength, + unsigned CallSiteTableLength, unsigned TTypeEncoding, + unsigned SizeActionsAndTypes); + virtual void emitTypeInfos(unsigned TTypeEncoding); // Helpers for for identifying what kind of clause an EH typeid or selector Index: lib/CodeGen/AsmPrinter/EHStreamer.cpp =================================================================== --- lib/CodeGen/AsmPrinter/EHStreamer.cpp +++ lib/CodeGen/AsmPrinter/EHStreamer.cpp @@ -211,7 +211,7 @@ /// the landing pad and the action. Calls marked 'nounwind' have no entry and /// must not be contained in the try-range of any entry - they form gaps in the /// table. Entries must be ordered by try-range address. -void EHStreamer:: +unsigned EHStreamer:: computeCallSiteTable(SmallVectorImpl &CallSites, const SmallVectorImpl &LandingPads, const SmallVectorImpl &FirstActions) { @@ -229,9 +229,24 @@ bool PreviousIsInvoke = false; bool IsSJLJ = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::SjLj; + bool SplitColdCode = Asm->getFunctionColdBegin() != nullptr; + bool IsInCold = false; + + unsigned RegularCallCount = 0; // Visit all instructions in order of address. for (const auto &MBB : *Asm->MF) { + if (!IsInCold && SplitColdCode && LastLabel && Asm->getColdFragmentStart() == &MBB) { + auto FnEnd = Asm->getFunctionEnd(); + CallSiteEntry Site = { LastLabel, FnEnd, nullptr, 0 }; + CallSites.push_back(Site); + RegularCallCount++; + SawPotentiallyThrowing = false; + PreviousIsInvoke = false; + IsInCold = true; + LastLabel = Asm->getFunctionColdBegin(); + } + for (const auto &MI : MBB) { if (!MI.isEHLabel()) { if (MI.isCall()) @@ -263,6 +278,8 @@ CallSiteEntry Site = { LastLabel, BeginLabel, nullptr, 0 }; CallSites.push_back(Site); PreviousIsInvoke = false; + if (!IsInCold) + RegularCallCount++; } LastLabel = LandingPad->EndLabels[P.RangeIndex]; @@ -291,9 +308,11 @@ } // Otherwise, create a new call-site. - if (!IsSJLJ) + if (!IsSJLJ) { CallSites.push_back(Site); - else { + if (!IsInCold) + RegularCallCount++; + } else { // SjLj EH must maintain the call sites in the order assigned // to them by the SjLjPrepare pass. unsigned SiteNo = MMI->getCallSiteBeginLabel(BeginLabel); @@ -312,7 +331,11 @@ if (SawPotentiallyThrowing && !IsSJLJ && LastLabel != nullptr) { CallSiteEntry Site = { LastLabel, nullptr, nullptr, 0 }; CallSites.push_back(Site); + if (!IsInCold) + RegularCallCount++; } + + return RegularCallCount; } /// Emit landing pads and actions. @@ -362,7 +385,8 @@ // Compute the call-site table. SmallVector CallSites; - computeCallSiteTable(CallSites, LandingPads, FirstActions); + unsigned RegularCallCount = computeCallSiteTable(CallSites, LandingPads, + FirstActions); // Final tallies. @@ -370,23 +394,33 @@ bool IsSJLJ = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::SjLj; bool HaveTTData = IsSJLJ ? (!TypeInfos.empty() || !FilterIds.empty()) : true; - unsigned CallSiteTableLength; - if (IsSJLJ) - CallSiteTableLength = 0; - else { + // Cold splitting + MCSymbol *EHFuncColdBeginSym = Asm->getFunctionColdBegin(); + assert(!IsSJLJ || !EHFuncColdBeginSym && "Cold splitting is not supported with SJLJ exceptions"); + + unsigned CallSiteTableLength = 0, ColdCallSiteTableLength = 0; + if (!IsSJLJ) { unsigned SiteStartSize = 4; // dwarf::DW_EH_PE_udata4 unsigned SiteLengthSize = 4; // dwarf::DW_EH_PE_udata4 unsigned LandingPadSize = 4; // dwarf::DW_EH_PE_udata4 - CallSiteTableLength = - CallSites.size() * (SiteStartSize + SiteLengthSize + LandingPadSize); + unsigned CallSiteSize = SiteStartSize + SiteLengthSize + LandingPadSize; + CallSiteTableLength = RegularCallCount * CallSiteSize; + ColdCallSiteTableLength = + (CallSites.size() - RegularCallCount) * CallSiteSize; } - for (unsigned i = 0, e = CallSites.size(); i < e; ++i) { + for (unsigned i = 0, e = RegularCallCount; i < e; ++i) { CallSiteTableLength += getULEB128Size(CallSites[i].Action); if (IsSJLJ) CallSiteTableLength += getULEB128Size(i); } + for (unsigned i = RegularCallCount, e = CallSites.size(); i < e; ++i) { + ColdCallSiteTableLength += getULEB128Size(CallSites[i].Action); + if (IsSJLJ) + ColdCallSiteTableLength += getULEB128Size(i); + } + // Type infos. MCSection *LSDASection = Asm->getObjFileLowering().getLSDASection(); unsigned TTypeEncoding; @@ -444,10 +478,6 @@ Asm->OutStreamer->EmitLabel(GCCETSym); Asm->OutStreamer->EmitLabel(Asm->getCurExceptionSym()); - // Emit the LSDA header. - Asm->EmitEncodingByte(dwarf::DW_EH_PE_omit, "@LPStart"); - Asm->EmitEncodingByte(TTypeEncoding, "@TType"); - // The type infos need to be aligned. GCC does this by inserting padding just // before the type infos. However, this changes the size of the exception // table, so you need to take this into account when you output the exception @@ -464,37 +494,16 @@ // We chose another solution: don't output padding inside the table like GCC // does, instead output it before the table. unsigned SizeTypes = TypeInfos.size() * TypeFormatSize; - unsigned CallSiteTableLengthSize = getULEB128Size(CallSiteTableLength); - unsigned TTypeBaseOffset = - sizeof(int8_t) + // Call site format - CallSiteTableLengthSize + // Call site table length size - CallSiteTableLength + // Call site table length - SizeActions + // Actions size - SizeTypes; - unsigned TTypeBaseOffsetSize = getULEB128Size(TTypeBaseOffset); - unsigned TotalSize = - sizeof(int8_t) + // LPStart format - sizeof(int8_t) + // TType format - (HaveTTData ? TTypeBaseOffsetSize : 0) + // TType base offset size - TTypeBaseOffset; // TType base offset - unsigned SizeAlign = (4 - TotalSize) & 3; - if (HaveTTData) { - // Account for any extra padding that will be added to the call site table - // length. - Asm->EmitULEB128(TTypeBaseOffset, "@TType base offset", SizeAlign); - SizeAlign = 0; - } + // Emit the LSDA header. + emitLSDAHeader(EHFuncColdBeginSym, ColdCallSiteTableLength, + CallSiteTableLength, TTypeEncoding, + SizeActions + SizeTypes); bool VerboseAsm = Asm->OutStreamer->isVerboseAsm(); // SjLj Exception handling if (IsSJLJ) { - Asm->EmitEncodingByte(dwarf::DW_EH_PE_udata4, "Call site"); - - // Add extra padding if it wasn't added to the TType base offset. - Asm->EmitULEB128(CallSiteTableLength, "Call site table length", SizeAlign); - // Emit the landing pad site information. unsigned idx = 0; for (SmallVectorImpl::const_iterator @@ -542,25 +551,57 @@ // A missing entry in the call-site table indicates that a call is not // supposed to throw. - // Emit the landing pad call site table. - Asm->EmitEncodingByte(dwarf::DW_EH_PE_udata4, "Call site"); + bool IsInCold = false; - // Add extra padding if it wasn't added to the TType base offset. - Asm->EmitULEB128(CallSiteTableLength, "Call site table length", SizeAlign); + // Regular Fragment + // --------------------- + // | FunctionBegin: | + // | | + // | ... | + // | | + // | FunctionEnd: | + // --------------------- + // + // .............. + // + // Cold Fragment + // --------------------- + // | FunctionColdBegin: | + // | | + // | ... | + // | | + // | FunctionColdEnd: | + // --------------------- + + MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin(); + MCSymbol *EHFuncEndSym = Asm->getFunctionEnd(); + MCSymbol *PadBaseLabel = EHFuncColdBeginSym + ? EHFuncColdBeginSym + : EHFuncBeginSym; unsigned Entry = 0; for (SmallVectorImpl::const_iterator I = CallSites.begin(), E = CallSites.end(); I != E; ++I) { const CallSiteEntry &S = *I; - MCSymbol *EHFuncBeginSym = Asm->getFunctionBegin(); - MCSymbol *BeginLabel = S.BeginLabel; - if (!BeginLabel) + if (!BeginLabel) { BeginLabel = EHFuncBeginSym; + } else if (BeginLabel->getSection().getKind().isColdText()) { + IsInCold = true; + EHFuncBeginSym = EHFuncColdBeginSym; + EHFuncEndSym = Asm->getFunctionColdEnd(); + + // We emit a second LSDA header withing the first one, so we can + // reuse action and type tables. + Asm->OutStreamer->EmitLabel(Asm->getCurColdExceptionSym()); + emitLSDAHeader(nullptr, 0, ColdCallSiteTableLength, TTypeEncoding, + SizeActions + SizeTypes); + } + MCSymbol *EndLabel = S.EndLabel; if (!EndLabel) - EndLabel = Asm->getFunctionEnd(); + EndLabel = EHFuncEndSym; // Offset of the call site relative to the previous call site, counted in // number of 16-byte bundles. The first call site is counted relative to @@ -584,7 +625,7 @@ if (VerboseAsm) Asm->OutStreamer->AddComment(Twine(" jumps to ") + S.LPad->LandingPadLabel->getName()); - Asm->EmitLabelDifference(S.LPad->LandingPadLabel, EHFuncBeginSym, 4); + Asm->EmitLabelDifference(S.LPad->LandingPadLabel, PadBaseLabel, 4); } // Offset of the first associated action record, relative to the start of @@ -648,6 +689,122 @@ Asm->EmitAlignment(2); } +void EHStreamer::emitLSDAHeader(MCSymbol *LPStart, unsigned SubCallSiteTableLength, + unsigned CallSiteTableLength, unsigned TTypeEncoding, + unsigned SizeActionsAndTypes) { + unsigned ActionTableIndex = CallSiteTableLength; + + unsigned AlignOffset = + sizeof(int8_t) + // LPStart format + sizeof(int8_t); // TType format + + if (LPStart) { + // We we have a nested LSDA after the callsites, acount for it. + // In order to not duplicate Actions and types entries, we nest + // the cold LSDA header after the regular calls. This adjusts + // offsets and alignement to fit that pattern. + // + // LSDA table layout: + // --------------------- + // | @LPStart Encoding | <= Regular LSDA header + // | @LPStart | + // | @Type Encoding | + // | @Type base offset | <=| + // | Call site encoding | |= Used to enfore cold LSDA header alignment + // | Action table index | <=| + // | | + // | Regular call sites | + // | ... | + // | | + // | | + // | @LPStart Encoding | <= Cold LSDA header + // | @LPStart | + // | @Type Encoding | + // | @Type base offset | <=| + // | Call site encoding | |= Used to enfore type table alignement + // | Action table index | <=| + // | | + // | Cold call sites | + // | ... | + // | | + // | | + // | Action table | | Action table grows down + // | ... | V + // | | + // | | + // | ... | ^ + // | Type table | | Type table grows up + // --------------------- + // + if (SubCallSiteTableLength) { + unsigned TTypeSubHeaderOffset = + sizeof(int8_t) + // Call site format + getULEB128Size(SubCallSiteTableLength); // Call site table length + + unsigned TTypeSubBodyOffset = + SubCallSiteTableLength + // Call site table + SizeActionsAndTypes; // Actions and Types size + + unsigned SubLSDAHeaderSize = + sizeof(int8_t) + // LPStart format + sizeof(int8_t) + // TType format + TTypeSubHeaderOffset; + + if (TTypeEncoding != dwarf::DW_EH_PE_omit) + SubLSDAHeaderSize += getULEB128Size(TTypeSubHeaderOffset + TTypeSubBodyOffset); + + // Make sure it is aligned. + unsigned SubLSDAAlignOffet = SubLSDAHeaderSize + TTypeSubBodyOffset; + SubLSDAHeaderSize += ((4 - SubLSDAAlignOffet) & 3); + + ActionTableIndex += SubLSDAHeaderSize + SubCallSiteTableLength; + + // We make sure nested LSDA is aligned properly + AlignOffset -= ActionTableIndex; + AlignOffset -= SizeActionsAndTypes; + AlignOffset += CallSiteTableLength; + } + + auto PtrSize = Asm->getDataLayout().getPointerSize(); + + // LPStart Pointer + AlignOffset += PtrSize; + + // If there is a cold fragment, landing pads are in there. + Asm->EmitEncodingByte(dwarf::DW_EH_PE_absptr, "@LPStart"); + Asm->OutStreamer->EmitSymbolValue(LPStart, PtrSize); + } else { + assert(!SubCallSiteTableLength && "Nested LDSA require explicit LPStart"); + Asm->EmitEncodingByte(dwarf::DW_EH_PE_omit, "@LPStart"); + } + + unsigned TTypeBaseOffset = + sizeof(int8_t) + // Call site format + getULEB128Size(ActionTableIndex) + // Action table index size + ActionTableIndex + // Action table index + SizeActionsAndTypes; // Actions and Types size + + AlignOffset += TTypeBaseOffset; // TType base offset + + Asm->EmitEncodingByte(TTypeEncoding, "@TType"); + if (TTypeEncoding != dwarf::DW_EH_PE_omit) { + AlignOffset += getULEB128Size(TTypeBaseOffset); + + unsigned SizeAlign = (4 - AlignOffset) & 3; + AlignOffset += SizeAlign; + + // Account for any extra padding that will be added to the call site table + // length. + Asm->EmitULEB128(TTypeBaseOffset, "@TType base offset", SizeAlign); + } + + // Emit the landing pad call site table. + Asm->EmitEncodingByte(dwarf::DW_EH_PE_udata4, "Call site"); + + // Add extra padding if it wasn't added to the TType base offset. + Asm->EmitULEB128(ActionTableIndex, "Action table index", (4 - AlignOffset) & 3); +} + void EHStreamer::emitTypeInfos(unsigned TTypeEncoding) { const std::vector &TypeInfos = MMI->getTypeInfos(); const std::vector &FilterIds = MMI->getFilterIds(); Index: lib/MC/MCObjectFileInfo.cpp =================================================================== --- lib/MC/MCObjectFileInfo.cpp +++ lib/MC/MCObjectFileInfo.cpp @@ -75,6 +75,10 @@ = Ctx->getMachOSection("__TEXT", "__text", MachO::S_ATTR_PURE_INSTRUCTIONS, SectionKind::getText()); + ColdTextSection // .text.cold + = Ctx->getMachOSection("__TEXT", "__text.cold", + MachO::S_ATTR_PURE_INSTRUCTIONS, + SectionKind::getColdText()); DataSection // .data = Ctx->getMachOSection("__DATA", "__data", 0, SectionKind::getData()); @@ -441,6 +445,9 @@ TextSection = Ctx->getELFSection(".text", ELF::SHT_PROGBITS, ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); + ColdTextSection = Ctx->getELFSection(".text.cold", ELF::SHT_PROGBITS, + ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); + DataSection = Ctx->getELFSection(".data", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC); @@ -822,6 +829,7 @@ CompactUnwindDwarfEHFrameOnly = 0; EHFrameSection = nullptr; // Created on demand. + ColdTextSection = nullptr; // Only work on some plateforms. CompactUnwindSection = nullptr; // Used only by selected targets. DwarfAccelNamesSection = nullptr; // Used only by selected targets. DwarfAccelObjCSection = nullptr; // Used only by selected targets. Index: lib/MC/MCStreamer.cpp =================================================================== --- lib/MC/MCStreamer.cpp +++ lib/MC/MCStreamer.cpp @@ -179,6 +179,20 @@ return CurFrame && !CurFrame->End; } +void MCStreamer::pushDwarfFrameInfo(MCDwarfFrameInfo DFI) { + if (hasUnfinishedDwarfFrameInfo()) + report_fatal_error("Pushing a frame before finishing the previous one!"); + DwarfFrameInfos.push_back(DFI); +} + +MCDwarfFrameInfo MCStreamer::popDwarfFrameInfo() { + if (!hasUnfinishedDwarfFrameInfo()) + report_fatal_error("Can only pop unfinished frames!"); + auto DFI = DwarfFrameInfos.back(); + DwarfFrameInfos.pop_back(); + return DFI; +} + void MCStreamer::EnsureValidDwarfFrame() { MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo(); if (!CurFrame || CurFrame->End) Index: lib/MC/MCTargetOptions.cpp =================================================================== --- lib/MC/MCTargetOptions.cpp +++ lib/MC/MCTargetOptions.cpp @@ -17,7 +17,7 @@ MCFatalWarnings(false), MCNoWarn(false), MCSaveTempLabels(false), MCUseDwarfDirectory(false), MCIncrementalLinkerCompatible(false), ShowMCEncoding(false), ShowMCInst(false), AsmVerbose(false), - DwarfVersion(0), ABIName() {} + SplitColdCode(false), DwarfVersion(0), ABIName() {} StringRef MCTargetOptions::getABIName() const { return ABIName; Index: test/CodeGen/X86/coldsplit.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/coldsplit.ll @@ -0,0 +1,85 @@ +; RUN: llc -mtriple=x86_64-pc-linux -split-cold-code < %s | FileCheck %s -check-prefix=LINUX +; RUN: llc -mtriple=x86_64-apple-macosx10.11.0 -split-cold-code < %s | FileCheck %s -check-prefix=DARWIN + +declare i32 @foo(); + +declare i32 @bar(); + +define i32 @branchweightcoldsplit(i32 %a) { + %1 = icmp sgt i32 %a, 1 + br i1 %1, label %2, label %4, !prof !0 + +;