diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -34,8 +34,11 @@ #include "bolt/Core/JumpTable.h" #include "bolt/Core/MCPlus.h" #include "bolt/Utils/NameResolver.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator.h" +#include "llvm/ADT/iterator_range.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDwarf.h" @@ -44,9 +47,11 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/Support/raw_ostream.h" #include +#include #include #include #include +#include #include using namespace llvm::object; @@ -142,7 +147,8 @@ uint64_t Action; }; - using CallSitesType = SmallVector; + using CallSitesList = SmallVector, 0>; + using CallSitesRange = iterator_range; using IslandProxiesType = std::map>; @@ -492,8 +498,7 @@ DenseMap> FrameRestoreEquivalents; // For tracking exception handling ranges. - CallSitesType CallSites; - CallSitesType ColdCallSites; + CallSitesList CallSites; /// Binary blobs representing action, type, and type index tables for this /// function' LSDA (exception handling). @@ -507,9 +512,9 @@ /// addressing. LSDATypeTableTy LSDATypeAddressTable; - /// Marking for the beginning of language-specific data area for the function. - MCSymbol *LSDASymbol{nullptr}; - MCSymbol *ColdLSDASymbol{nullptr}; + /// Marking for the beginnings of language-specific data areas for each + /// fragment of the function. + SmallVector LSDASymbols; /// Map to discover which CFIs are attached to a given instruction offset. /// Maps an instruction offset into a FrameInstructions offset. @@ -716,7 +721,6 @@ BB->releaseCFG(); clearList(CallSites); - clearList(ColdCallSites); clearList(LSDATypeTable); clearList(LSDATypeAddressTable); @@ -1418,9 +1422,24 @@ uint8_t getPersonalityEncoding() const { return PersonalityEncoding; } - const CallSitesType &getCallSites() const { return CallSites; } + CallSitesRange getCallSites(const FragmentNum F) const { + return make_range(std::equal_range( + CallSites.begin(), CallSites.end(), std::make_pair(F, CallSite()), + [](const std::pair &Lhs, + const std::pair &Rhs) { + return Lhs.first < Rhs.first; + })); + } - const CallSitesType &getColdCallSites() const { return ColdCallSites; } + void + addCallSites(const ArrayRef> NewCallSites) { + llvm::copy(NewCallSites, std::back_inserter(CallSites)); + llvm::stable_sort(CallSites, + [](const std::pair &Lhs, + const std::pair &Rhs) { + return Lhs.first < Rhs.first; + }); + } ArrayRef getLSDAActionTable() const { return LSDAActionTable; } @@ -1822,9 +1841,11 @@ return *this; } - /// Set LSDA symbol for the function. + /// Set main LSDA symbol for the function. BinaryFunction &setLSDASymbol(MCSymbol *Symbol) { - LSDASymbol = Symbol; + if (LSDASymbols.empty()) + LSDASymbols.resize(1); + LSDASymbols.front() = Symbol; return *this; } @@ -1848,30 +1869,25 @@ uint64_t getLSDAAddress() const { return LSDAAddress; } /// Return symbol pointing to function's LSDA. - MCSymbol *getLSDASymbol() { - if (LSDASymbol) - return LSDASymbol; - if (CallSites.empty()) + MCSymbol *getLSDASymbol(const FragmentNum F) { + if (F.get() < LSDASymbols.size() && LSDASymbols[F.get()] != nullptr) + return LSDASymbols[F.get()]; + if (llvm::empty(getCallSites(F))) return nullptr; - LSDASymbol = BC.Ctx->getOrCreateSymbol( - Twine("GCC_except_table") + Twine::utohexstr(getFunctionNumber())); + if (F.get() >= LSDASymbols.size()) + LSDASymbols.resize(F.get() + 1); - return LSDASymbol; - } - - /// Return symbol pointing to function's LSDA for the cold part. - MCSymbol *getColdLSDASymbol(const FragmentNum Fragment) { - if (ColdLSDASymbol) - return ColdLSDASymbol; - if (ColdCallSites.empty()) - return nullptr; + SmallString<256> SymbolName; + if (F == FragmentNum::main()) + SymbolName = formatv("GCC_except_table{0:x-}", getFunctionNumber()); + else + SymbolName = formatv("GCC_cold_except_table{0:x-}.{1}", + getFunctionNumber(), F.get()); - ColdLSDASymbol = - BC.Ctx->getOrCreateSymbol(formatv("GCC_cold_except_table{0:x-}.{1}", - getFunctionNumber(), Fragment.get())); + LSDASymbols[F.get()] = BC.Ctx->getOrCreateSymbol(SymbolName); - return ColdLSDASymbol; + return LSDASymbols[F.get()]; } void setOutputDataAddress(uint64_t Address) { OutputDataOffset = Address; } diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -154,7 +154,7 @@ void emitCFIInstruction(const MCCFIInstruction &Inst) const; /// Emit exception handling ranges for the function. - void emitLSDA(BinaryFunction &BF, bool EmitColdPart); + void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF); /// Emit line number information corresponding to \p NewLoc. \p PrevLoc /// provides a context for de-duplication of line number info. @@ -333,9 +333,7 @@ if (Function.getPersonalityFunction() != nullptr) Streamer.emitCFIPersonality(Function.getPersonalityFunction(), Function.getPersonalityEncoding()); - MCSymbol *LSDASymbol = FF.isSplitFragment() - ? Function.getColdLSDASymbol(FF.getFragmentNum()) - : Function.getLSDASymbol(); + MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum()); if (LSDASymbol) Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding); else @@ -394,7 +392,7 @@ emitLineInfoEnd(Function, EndSymbol); // Exception handling info for the function. - emitLSDA(Function, FF.isSplitFragment()); + emitLSDA(Function, FF); if (FF.isMainFragment() && opts::JumpTables > JTS_NONE) emitJumpTables(Function); @@ -881,10 +879,10 @@ } // The code is based on EHStreamer::emitExceptionTable(). -void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) { - const BinaryFunction::CallSitesType *Sites = - EmitColdPart ? &BF.getColdCallSites() : &BF.getCallSites(); - if (Sites->empty()) +void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) { + const BinaryFunction::CallSitesRange Sites = + BF.getCallSites(FF.getFragmentNum()); + if (llvm::empty(Sites)) return; // Calculate callsite table size. Size of each callsite entry is: @@ -894,9 +892,9 @@ // or // // sizeof(dwarf::DW_EH_PE_data4) * 3 + sizeof(uleb128(action)) - uint64_t CallSiteTableLength = Sites->size() * 4 * 3; - for (const BinaryFunction::CallSite &CallSite : *Sites) - CallSiteTableLength += getULEB128Size(CallSite.Action); + uint64_t CallSiteTableLength = llvm::size(Sites) * 4 * 3; + for (const auto &FragmentCallSite : Sites) + CallSiteTableLength += getULEB128Size(FragmentCallSite.second.Action); Streamer.switchSection(BC.MOFI->getLSDASection()); @@ -908,15 +906,12 @@ Streamer.emitValueToAlignment(TTypeAlignment); // Emit the LSDA label. - MCSymbol *LSDASymbol = EmitColdPart - ? BF.getColdLSDASymbol(FragmentNum::cold()) - : BF.getLSDASymbol(); + MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum()); assert(LSDASymbol && "no LSDA symbol set"); Streamer.emitLabel(LSDASymbol); // Corresponding FDE start. - const MCSymbol *StartSymbol = - BF.getSymbol(EmitColdPart ? FragmentNum::cold() : FragmentNum::main()); + const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum()); // Emit the LSDA header. @@ -988,7 +983,8 @@ Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1); Streamer.emitULEB128IntValue(CallSiteTableLength); - for (const BinaryFunction::CallSite &CallSite : *Sites) { + for (const auto &FragmentCallSite : Sites) { + const BinaryFunction::CallSite &CallSite = FragmentCallSite.second; const MCSymbol *BeginLabel = CallSite.Start; const MCSymbol *EndLabel = CallSite.End; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -619,13 +619,16 @@ // Dump new exception ranges for the function. if (!CallSites.empty()) { OS << "EH table:\n"; - for (const CallSite &CSI : CallSites) { - OS << " [" << *CSI.Start << ", " << *CSI.End << ") landing pad : "; - if (CSI.LP) - OS << *CSI.LP; - else - OS << "0"; - OS << ", action : " << CSI.Action << '\n'; + for (const FunctionFragment &FF : getLayout().fragments()) { + for (const auto &FCSI : getCallSites(FF.getFragmentNum())) { + const CallSite &CSI = FCSI.second; + OS << " [" << *CSI.Start << ", " << *CSI.End << ") landing pad : "; + if (CSI.LP) + OS << *CSI.LP; + else + OS << "0"; + OS << ", action : " << CSI.Action << '\n'; + } } OS << '\n'; } diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -367,10 +367,10 @@ uint64_t Action; }; - for (FunctionFragment &FF : getLayout().fragments()) { - // Sites to update - either regular or cold. - CallSitesType &Sites = FF.isMainFragment() ? CallSites : ColdCallSites; + // Sites to update. + CallSitesList Sites; + for (FunctionFragment &FF : getLayout().fragments()) { // If previous call can throw, this is its exception handler. EHInfo PreviousEH = {nullptr, 0}; @@ -389,9 +389,6 @@ if (!Throws && !StartRange) continue; - assert(getLayout().isHotColdSplit() && - "Exceptions only supported for hot/cold splitting"); - // Extract exception handling information from the instruction. const MCSymbol *LP = nullptr; uint64_t Action = 0; @@ -434,10 +431,10 @@ } // Close the previous range. - if (EndRange) { + if (EndRange) Sites.emplace_back( + FF.getFragmentNum(), CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action}); - } if (Throws) { // I, II: @@ -451,13 +448,14 @@ // Check if we need to close the range. if (StartRange) { - assert((FF.isMainFragment() || &Sites == &ColdCallSites) && - "sites mismatch"); const MCSymbol *EndRange = getFunctionEndLabel(FF.getFragmentNum()); Sites.emplace_back( + FF.getFragmentNum(), CallSite{StartRange, EndRange, PreviousEH.LP, PreviousEH.Action}); } } + + addCallSites(Sites); } const uint8_t DWARF_CFI_PRIMARY_OPCODE_MASK = 0xc0; diff --git a/bolt/lib/Passes/SplitFunctions.cpp b/bolt/lib/Passes/SplitFunctions.cpp --- a/bolt/lib/Passes/SplitFunctions.cpp +++ b/bolt/lib/Passes/SplitFunctions.cpp @@ -217,7 +217,11 @@ struct SplitAll final : public SplitStrategy { bool canSplit(const BinaryFunction &BF) override { return true; } - bool keepEmpty() override { return false; } + bool keepEmpty() override { + // Keeping empty fragments allows us to test, that empty fragments do not + // generate symbols. + return true; + } void fragment(const BlockIt Start, const BlockIt End) override { unsigned Fragment = 0; diff --git a/bolt/test/X86/fragmented-symbols.s b/bolt/test/X86/fragmented-symbols.s new file mode 100644 --- /dev/null +++ b/bolt/test/X86/fragmented-symbols.s @@ -0,0 +1,118 @@ +# Checks that symbols are allocated in correct sections, and that empty +# fragments are not allocated at all. + +# REQUIRES: x86_64-linux + +# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o +# RUN: %clangxx %cxxflags %t.o -o %t.exe -Wl,-q -no-pie +# RUN: llvm-bolt %t.exe -o %t.bolt --split-functions --split-strategy=all \ +# RUN: --print-split --print-only=_Z3foov 2>&1 | \ +# RUN: FileCheck %s --check-prefix=CHECK-SPLIT +# RUN: llvm-nm %t.bolt | FileCheck %s --check-prefix=CHECK-COLD0 +# RUN: llvm-objdump --syms %t.bolt | \ +# RUN: FileCheck %s --check-prefix=CHECK-SYMS + +# CHECK-SPLIT: .LLP0 (4 instructions, align : 1) +# CHECK-SPLIT: ------- HOT-COLD SPLIT POINT ------- +# CHECK-SPLIT-EMPTY: +# CHECK-SPLIT-NEXT: ------- HOT-COLD SPLIT POINT ------- +# CHECK-SPLIT-EMPTY: +# CHECK-SPLIT-NEXT: .LFT0 (2 instructions, align : 1) + +# CHECK-COLD0-NOT: _Z3foov.cold.0 + +# CHECK-SYMS: .text.cold.1 +# CHECK-SYMS-SAME: _Z3foov.cold.1 +# CHECK-SYMS: .text.cold.2 +# CHECK-SYMS-SAME: _Z3foov.cold.2 +# CHECK-SYMS: .text.cold.3 +# CHECK-SYMS-SAME: _Z3foov.cold.3 + + + .text + .globl _Z3barv + .type _Z3barv, @function +_Z3barv: # void bar(); + .cfi_startproc + ret + .cfi_endproc + .size _Z3barv, .-_Z3barv + + + .globl _Z3bazv + .type _Z3bazv, @function +_Z3bazv: # void baz() noexcept; + .cfi_startproc + ret + .cfi_endproc + .size _Z3bazv, .-_Z3bazv + + + .globl _Z3foov + .type _Z3foov, @function +_Z3foov: # void foo() noexcept; +.LFB1265: # _Z3foov + .cfi_startproc + .cfi_personality 0x3,__gxx_personality_v0 + .cfi_lsda 0x3,.LLSDA1265 + subq $8, %rsp + .cfi_def_cfa_offset 16 +.LEHB0: + call _Z3barv # LP: .L5 +.LEHE0: + jmp .L4 +.L5: # (_Z3foov.cold.0), landing pad, hot + movq %rax, %rdi + cmpq $1, %rdx + je .L3 + call _ZSt9terminatev # _Z3foov.cold.1 +.L3: # _Z3foov.cold.2 + call __cxa_begin_catch + call _Z3bazv + call __cxa_end_catch +.L4: # _Z3foov.cold.3 + call _Z3bazv + addq $8, %rsp + .cfi_def_cfa_offset 8 + ret + .cfi_endproc + .globl __gxx_personality_v0 + .section .gcc_except_table,"a",@progbits + .align 4 +.LLSDA1265: + .byte 0xff + .byte 0x3 + .uleb128 .LLSDATT1265-.LLSDATTD1265 +.LLSDATTD1265: + .byte 0x1 + .uleb128 .LLSDACSE1265-.LLSDACSB1265 +.LLSDACSB1265: + .uleb128 .LEHB0-.LFB1265 + .uleb128 .LEHE0-.LEHB0 + .uleb128 .L5-.LFB1265 + .uleb128 0x3 +.LLSDACSE1265: + .byte 0 + .byte 0 + .byte 0x1 + .byte 0x7d + .align 4 + .long _ZTISt13runtime_error +.LLSDATT1265: + .text + .size _Z3foov, .-_Z3foov + + + .globl main + .type main, @function +main: + .cfi_startproc + subq $8, %rsp + .cfi_def_cfa_offset 16 + call _Z3foov + movl $0, %eax + addq $8, %rsp + .cfi_def_cfa_offset 8 + ret + .cfi_endproc + .size main, .-main diff --git a/bolt/test/runtime/X86/exceptions-pic.test b/bolt/test/runtime/X86/exceptions-pic.test --- a/bolt/test/runtime/X86/exceptions-pic.test +++ b/bolt/test/runtime/X86/exceptions-pic.test @@ -5,9 +5,9 @@ RUN: %clangxx -fpic -Wl,-q %S/Inputs/exception4.cpp -o %t.pic RUN: llvm-bolt %t.pic -o %t && %t 2>&1 | FileCheck %s RUN: llvm-bolt --relocs --use-old-text %t.pic -o %t && %t 2>&1 | FileCheck %s +RUN: llvm-bolt %t.pic -o %t --split-functions --split-strategy=all --split-eh && \ +RUN: %t 2>&1 | FileCheck %s CHECK: catch 2 CHECK-NEXT: catch 1 CHECK-NEXT: caught ExcC - - diff --git a/bolt/test/runtime/X86/rethrow.cpp b/bolt/test/runtime/X86/rethrow.cpp new file mode 100644 --- /dev/null +++ b/bolt/test/runtime/X86/rethrow.cpp @@ -0,0 +1,100 @@ +#include +#include + +void erringFunc() { throw std::runtime_error("Hello"); } + +void libCallA() { erringFunc(); } + +void libCallB() { throw std::runtime_error("World"); } + +void handleEventA() { + try { + libCallA(); + } catch (std::runtime_error &E) { + std::cout << "handleEventA: unhandled error " << E.what() << "\n"; + throw; + } +} + +void handleEventB() { + try { + libCallB(); + } catch (std::runtime_error &E) { + std::cout << "handleEventB: handle error " << E.what() << "\n"; + } +} + +class EventGen { + unsigned RemainingEvents = 5; + +public: + int generateEvent() { + if (RemainingEvents > 0) { + --RemainingEvents; + return (RemainingEvents % 3) + 1; + } + return 0; + } +}; + +class TerminateException : public std::runtime_error { +public: + TerminateException() : std::runtime_error("Time to stop!") {} +}; + +void runEventLoop(EventGen &EG) { + while (true) { + try { + int Ev = EG.generateEvent(); + switch (Ev) { + case 0: + throw TerminateException(); + case 1: + handleEventA(); + break; + case 2: + handleEventB(); + break; + } + } catch (TerminateException &E) { + std::cout << "Terminated?\n"; + throw; + } catch (std::runtime_error &E) { + std::cout << "Unhandled error: " << E.what() << "\n"; + } + } +} + +struct CleanUp { + ~CleanUp() { std::cout << "Cleanup\n"; } +}; + +int main() { + EventGen EG; + try { + CleanUp CU; + runEventLoop(EG); + } catch (TerminateException &E) { + std::cout << "Terminated!\n"; + } + return 0; +} + +/* +REQUIRES: system-linux + +RUN: %clang++ %cflags %s -o %t.exe -Wl,-q +RUN: llvm-bolt %t.exe --split-functions --split-strategy=randomN \ +RUN: --split-all-cold --split-eh --bolt-seed=7 -o %t.bolt +RUN: %t.bolt | FileCheck %s + +CHECK: handleEventB: handle error World +CHECK-NEXT: handleEventA: unhandled error Hello +CHECK-NEXT: Unhandled error: Hello +CHECK-NEXT: handleEventB: handle error World +CHECK-NEXT: handleEventA: unhandled error Hello +CHECK-NEXT: Unhandled error: Hello +CHECK-NEXT: Terminated? +CHECK-NEXT: Cleanup +CHECK-NEXT: Terminated! +*/