diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -59,7 +59,6 @@ namespace bolt { class BinaryFunction; -class ExecutableFileMemoryManager; /// Information on loadable part of the file. struct SegmentInfo { @@ -313,10 +312,6 @@ FilterIterator; using FilteredBinaryDataIterator = FilterIterator; - /// Memory manager for sections and segments. Used to communicate with ORC - /// among other things. - std::shared_ptr EFMM; - StringRef getFilename() const { return Filename; } void setFilename(StringRef Name) { Filename = std::string(Name); } diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -90,7 +90,7 @@ uint64_t OutputFileOffset{0}; // File offset in the rewritten binary file. StringRef OutputContents; // Rewritten section contents. const uint64_t SectionNumber; // Order in which the section was created. - unsigned SectionID{-1u}; // Unique ID used for address mapping. + std::string SectionID; // Unique ID used for address mapping. // Set by ExecutableFileMemoryManager. uint32_t Index{0}; // Section index in the output file. mutable bool IsReordered{false}; // Have the contents been reordered? @@ -430,18 +430,18 @@ } uint64_t getOutputAddress() const { return OutputAddress; } uint64_t getOutputFileOffset() const { return OutputFileOffset; } - unsigned getSectionID() const { + StringRef getSectionID() const { assert(hasValidSectionID() && "trying to use uninitialized section id"); return SectionID; } - bool hasValidSectionID() const { return SectionID != -1u; } + bool hasValidSectionID() const { return !SectionID.empty(); } bool hasValidIndex() { return Index != 0; } uint32_t getIndex() const { return Index; } // mutation void setOutputAddress(uint64_t Address) { OutputAddress = Address; } void setOutputFileOffset(uint64_t Offset) { OutputFileOffset = Offset; } - void setSectionID(unsigned ID) { + void setSectionID(StringRef ID) { assert(!hasValidSectionID() && "trying to set section id twice"); SectionID = ID; } diff --git a/bolt/include/bolt/Core/Linker.h b/bolt/include/bolt/Core/Linker.h new file mode 100644 --- /dev/null +++ b/bolt/include/bolt/Core/Linker.h @@ -0,0 +1,48 @@ +//===- bolt/Core/Linker.h - BOLTLinker interface ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the interface BOLT uses for linking. +// +//===----------------------------------------------------------------------===// + +#ifndef BOLT_CORE_LINKER_H +#define BOLT_CORE_LINKER_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBufferRef.h" + +#include +#include +#include + +namespace llvm { +namespace bolt { + +class BinarySection; + +class BOLTLinker { +public: + using SectionMapper = + std::function; + using SectionsMapper = std::function; + + virtual ~BOLTLinker() = default; + + /// Load and link \p Obj. \p MapSections will be called before the object is + /// linked to allow section addresses to be remapped. When called, the address + /// of a section can be changed by calling the passed SectionMapper. + virtual void loadObject(MemoryBufferRef Obj, SectionsMapper MapSections) = 0; + + /// Return the address of a symbol or std::nullopt if it cannot be found. + virtual std::optional lookupSymbol(StringRef Name) const = 0; +}; + +} // namespace bolt +} // namespace llvm + +#endif // BOLT_CORE_LINKER_H diff --git a/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h b/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h --- a/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h +++ b/bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h @@ -10,7 +10,7 @@ #define BOLT_REWRITE_EXECUTABLE_FILE_MEMORY_MANAGER_H #include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include #include @@ -20,20 +20,12 @@ class BinaryContext; /// Class responsible for allocating and managing code and data sections. -class ExecutableFileMemoryManager : public RuntimeDyld::MemoryManager { +class ExecutableFileMemoryManager : public jitlink::JITLinkMemoryManager { private: - uint8_t *allocateSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsCode, bool IsReadOnly); - BinaryContext &BC; - bool AllowStubs; + void updateSection(const jitlink::Section &Section, uint8_t *Contents, + size_t Size, size_t Alignment); - struct AllocInfo { - uint8_t *Address; - size_t Size; - size_t Alignment; - }; - SmallVector AllocatedSections; + BinaryContext &BC; // All new sections will be identified by the following prefix. std::string NewSecPrefix; @@ -50,48 +42,14 @@ // user-supplied objects into the main input executable. uint32_t ObjectsLoaded{0}; - ExecutableFileMemoryManager(BinaryContext &BC, bool AllowStubs) - : BC(BC), AllowStubs(AllowStubs) {} - - ~ExecutableFileMemoryManager(); - - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - return allocateSection(Size, Alignment, SectionID, SectionName, - /*IsCode=*/true, true); - } - - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - return allocateSection(Size, Alignment, SectionID, SectionName, - /*IsCode=*/false, IsReadOnly); - } - - // Ignore TLS sections by treating them as a regular data section - TLSSection allocateTLSSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - TLSSection Res; - Res.Offset = 0; - Res.InitializationImage = allocateDataSection( - Size, Alignment, SectionID, SectionName, /*IsReadOnly=*/false); - return Res; - } - - bool allowStubAllocation() const override { return AllowStubs; } + ExecutableFileMemoryManager(BinaryContext &BC) : BC(BC) {} - /// Count processed objects and skip memory finalization. - bool finalizeMemory(std::string *ErrMsg) override { - ++ObjectsLoaded; - return false; - } + void allocate(const jitlink::JITLinkDylib *JD, jitlink::LinkGraph &G, + OnAllocatedFunction OnAllocated) override; - /// Ignore EH frames. - void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, - size_t Size) override {} - void deregisterEHFrames() override {} + void deallocate(std::vector Allocs, + OnDeallocatedFunction OnDeallocated) override; + using JITLinkMemoryManager::deallocate; /// Section name management. void setNewSecPrefix(StringRef Prefix) { NewSecPrefix = Prefix; } diff --git a/bolt/include/bolt/Rewrite/JITLinkLinker.h b/bolt/include/bolt/Rewrite/JITLinkLinker.h new file mode 100644 --- /dev/null +++ b/bolt/include/bolt/Rewrite/JITLinkLinker.h @@ -0,0 +1,56 @@ +//===- bolt/Rewrite/JITLinkLinker.h - Linker using JITLink ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// BOLTLinker using JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef BOLT_REWRITE_JITLINK_LINKER_H +#define BOLT_REWRITE_JITLINK_LINKER_H + +#include "bolt/Core/Linker.h" +#include "bolt/Rewrite/ExecutableFileMemoryManager.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h" + +#include +#include +#include + +namespace llvm { +namespace bolt { + +class BinaryContext; + +class JITLinkLinker : public BOLTLinker { +private: + struct Context; + friend struct Context; + + BinaryContext &BC; + std::unique_ptr MM; + jitlink::JITLinkDylib Dylib{"main"}; + std::vector Allocs; + std::map Symtab; + +public: + JITLinkLinker(BinaryContext &BC, + std::unique_ptr MM); + ~JITLinkLinker(); + + void loadObject(MemoryBufferRef Obj, SectionsMapper MapSections) override; + std::optional lookupSymbol(StringRef Name) const override; + + static SmallVector + orderedBlocks(const jitlink::Section &Section); + static size_t sectionSize(const jitlink::Section &Section); +}; + +} // namespace bolt +} // namespace llvm + +#endif // BOLT_REWRITE_JITLINK_LINKER_H diff --git a/bolt/include/bolt/Rewrite/MachORewriteInstance.h b/bolt/include/bolt/Rewrite/MachORewriteInstance.h --- a/bolt/include/bolt/Rewrite/MachORewriteInstance.h +++ b/bolt/include/bolt/Rewrite/MachORewriteInstance.h @@ -13,13 +13,13 @@ #ifndef BOLT_REWRITE_MACHO_REWRITE_INSTANCE_H #define BOLT_REWRITE_MACHO_REWRITE_INSTANCE_H +#include "bolt/Core/Linker.h" #include "bolt/Utils/NameResolver.h" #include "llvm/Support/Error.h" #include namespace llvm { class ToolOutputFile; -class RuntimeDyld; class raw_pwrite_stream; namespace object { class MachOObjectFile; @@ -37,7 +37,7 @@ NameResolver NR; - std::unique_ptr RTDyld; + std::unique_ptr Linker; std::unique_ptr Out; @@ -49,8 +49,9 @@ static StringRef getNewSecPrefix() { return ".bolt.new"; } static StringRef getOrgSecPrefix() { return ".bolt.org"; } - void mapInstrumentationSection(StringRef SectionName); - void mapCodeSections(); + void mapInstrumentationSection(StringRef SectionName, + BOLTLinker::SectionMapper MapSection); + void mapCodeSections(BOLTLinker::SectionMapper MapSection); void adjustCommandLineOptions(); void readSpecialSections(); diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -14,6 +14,7 @@ #define BOLT_REWRITE_REWRITE_INSTANCE_H #include "bolt/Core/BinaryContext.h" +#include "bolt/Core/Linker.h" #include "bolt/Utils/NameResolver.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/MC/StringTableBuilder.h" @@ -206,13 +207,13 @@ std::vector getCodeSections(); /// Map all sections to their final addresses. - void mapFileSections(RuntimeDyld &RTDyld); + void mapFileSections(BOLTLinker::SectionMapper MapSection); /// Map code sections generated by BOLT. - void mapCodeSections(RuntimeDyld &RTDyld); + void mapCodeSections(BOLTLinker::SectionMapper MapSection); /// Map the rest of allocatable sections. - void mapAllocatableSections(RuntimeDyld &RTDyld); + void mapAllocatableSections(BOLTLinker::SectionMapper MapSection); /// Update output object's values based on the final \p Layout. void updateOutputValues(const MCAsmLayout &Layout); @@ -459,7 +460,7 @@ std::unique_ptr CFIRdWrt; // Run ExecutionEngine linker with custom memory manager and symbol resolver. - std::unique_ptr RTDyld; + std::unique_ptr Linker; /// Output file where we mix original code from the input binary and /// optimized code for selected functions. diff --git a/bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h b/bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h --- a/bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h +++ b/bolt/include/bolt/RuntimeLibs/HugifyRuntimeLibrary.h @@ -28,8 +28,8 @@ void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final {} - void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld, - std::function OnLoad) final; + void link(BinaryContext &BC, StringRef ToolPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) override; }; } // namespace bolt diff --git a/bolt/include/bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h b/bolt/include/bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h --- a/bolt/include/bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h +++ b/bolt/include/bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h @@ -35,8 +35,8 @@ void emitBinary(BinaryContext &BC, MCStreamer &Streamer) final; - void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld, - std::function OnLoad) final; + void link(BinaryContext &BC, StringRef ToolPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) override; private: std::string buildTables(BinaryContext &BC); diff --git a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h --- a/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h +++ b/bolt/include/bolt/RuntimeLibs/RuntimeLibrary.h @@ -15,6 +15,7 @@ #ifndef BOLT_RUNTIMELIBS_RUNTIME_LIBRARY_H #define BOLT_RUNTIMELIBS_RUNTIME_LIBRARY_H +#include "bolt/Core/Linker.h" #include "llvm/ADT/StringRef.h" #include #include @@ -22,7 +23,6 @@ namespace llvm { class MCStreamer; -class RuntimeDyld; namespace bolt { @@ -51,8 +51,8 @@ virtual void emitBinary(BinaryContext &BC, MCStreamer &Streamer) = 0; /// Link with the library code. - virtual void link(BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld, - std::function OnLoad) = 0; + virtual void link(BinaryContext &BC, StringRef ToolPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) = 0; protected: /// The fini and init address set by the runtime library. @@ -63,7 +63,8 @@ static std::string getLibPath(StringRef ToolPath, StringRef LibFileName); /// Load a static runtime library specified by \p LibPath. - static void loadLibrary(StringRef LibPath, RuntimeDyld &RTDyld); + static void loadLibrary(StringRef LibPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections); }; } // namespace bolt diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -1975,7 +1975,9 @@ ErrorOr AbsSection = getUniqueSectionByName(""); for (auto SI = Sections.begin(); SI != Sections.end();) { BinarySection *Section = *SI; - if (Section->hasSectionRef() || Section->getOutputSize() || + // We check getOutputData() instead of getOutputSize() because sometimes + // zero-sized .text.cold sections are allocated. + if (Section->hasSectionRef() || Section->getOutputData() || (AbsSection && Section == &AbsSection.get())) { ++SI; continue; diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -189,13 +189,16 @@ if (opts::UpdateDebugSections && BC.isELF()) { // Force the emission of debug line info into allocatable section to ensure - // RuntimeDyld will process it without ProcessAllSections flag. + // JITLink will process it. // // NB: on MachO all sections are required for execution, hence no need // to change flags/attributes. MCSectionELF *ELFDwarfLineSection = static_cast(BC.MOFI->getDwarfLineSection()); ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC); + MCSectionELF *ELFDwarfLineStrSection = + static_cast(BC.MOFI->getDwarfLineStrSection()); + ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC); } if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary()) diff --git a/bolt/lib/Core/DebugData.cpp b/bolt/lib/Core/DebugData.cpp --- a/bolt/lib/Core/DebugData.cpp +++ b/bolt/lib/Core/DebugData.cpp @@ -1635,15 +1635,6 @@ "cannot combine raw data with new line entries"); MCOS->emitLabel(getLabel()); MCOS->emitBytes(RawData); - - // Emit fake relocation for RuntimeDyld to always allocate the section. - // - // FIXME: remove this once RuntimeDyld stops skipping allocatable sections - // without relocations. - MCOS->emitRelocDirective( - *MCConstantExpr::create(0, *BC.Ctx), "BFD_RELOC_NONE", - MCSymbolRefExpr::create(getLabel(), *BC.Ctx), SMLoc(), *BC.STI); - return; } @@ -1725,12 +1716,8 @@ // Still need to write the section out for the ExecutionEngine, and temp in // memory object we are constructing. - if (LineStr) { + if (LineStr) LineStr->emitSection(&Streamer); - SmallString<0> Data = LineStr->getFinalizedData(); - BC.registerOrUpdateNoteSection(".debug_line_str", copyByteArray(Data.str()), - Data.size()); - } } } // namespace bolt diff --git a/bolt/lib/Rewrite/CMakeLists.txt b/bolt/lib/Rewrite/CMakeLists.txt --- a/bolt/lib/Rewrite/CMakeLists.txt +++ b/bolt/lib/Rewrite/CMakeLists.txt @@ -1,7 +1,7 @@ set(LLVM_LINK_COMPONENTS DebugInfoDWARF DWP - ExecutionEngine + JITLink MC Object Support @@ -13,6 +13,7 @@ BoltDiff.cpp DWARFRewriter.cpp ExecutableFileMemoryManager.cpp + JITLinkLinker.cpp MachORewriteInstance.cpp RewriteInstance.cpp diff --git a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp --- a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp +++ b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp @@ -7,7 +7,9 @@ //===----------------------------------------------------------------------===// #include "bolt/Rewrite/ExecutableFileMemoryManager.h" +#include "bolt/Rewrite/JITLinkLinker.h" #include "bolt/Rewrite/RewriteInstance.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Support/MemAlloc.h" #undef DEBUG_TYPE @@ -21,30 +23,105 @@ namespace bolt { -uint8_t *ExecutableFileMemoryManager::allocateSection( - uintptr_t Size, unsigned Alignment, unsigned SectionID, - StringRef SectionName, bool IsCode, bool IsReadOnly) { - uint8_t *Ret = static_cast(llvm::allocate_buffer(Size, Alignment)); - AllocatedSections.push_back(AllocInfo{Ret, Size, Alignment}); +namespace { - // A Size of 1 might mean an empty section for which RuntimeDyld decided to - // allocate 1 byte. In this case, the allocation will never be initialized - // causing non-deterministic output section contents. - if (Size == 1) - *Ret = 0; +SmallVector orderedSections(jitlink::LinkGraph &G) { + SmallVector Sections( + llvm::map_range(G.sections(), [](auto &S) { return &S; })); + llvm::sort(Sections, [](const auto *LHS, const auto *RHS) { + return LHS->getOrdinal() < RHS->getOrdinal(); + }); + return Sections; +} + +size_t sectionAlignment(const jitlink::Section &Section) { + assert(!Section.empty() && "Cannot get alignment for empty section"); + return JITLinkLinker::orderedBlocks(Section).front()->getAlignment(); +} + +StringRef sectionName(const jitlink::Section &Section, + const BinaryContext &BC) { + auto Name = Section.getName(); + + if (BC.isMachO()) { + // JITLink "normalizes" section names as "SegmentName,SectionName" on + // Mach-O. BOLT internally refers to sections just by the section name so + // strip-off the segment name. + auto SegmentEnd = Name.find(','); + assert(SegmentEnd != StringRef::npos && "Mach-O segment not found"); + Name = Name.substr(SegmentEnd + 1); + } + + return Name; +} + +struct SectionAllocInfo { + void *Address; + size_t Size; + size_t Alignment; +}; + +struct AllocInfo { + SmallVector AllocatedSections; + + ~AllocInfo() { + for (auto &Section : AllocatedSections) + deallocate_buffer(Section.Address, Section.Size, Section.Alignment); + } + + SectionAllocInfo allocateSection(const jitlink::Section &Section) { + auto Size = JITLinkLinker::sectionSize(Section); + auto Alignment = sectionAlignment(Section); + auto *Buf = allocate_buffer(Size, Alignment); + SectionAllocInfo Alloc{Buf, Size, Alignment}; + AllocatedSections.push_back(Alloc); + return Alloc; + } +}; + +struct BOLTInFlightAlloc : ExecutableFileMemoryManager::InFlightAlloc { + // Even though this is passed using a raw pointer in FinalizedAlloc, we keep + // it in a unique_ptr as long as possible to enjoy automatic cleanup when + // something goes wrong. + std::unique_ptr Alloc; + +public: + BOLTInFlightAlloc(std::unique_ptr Alloc) + : Alloc(std::move(Alloc)) {} + + virtual void abandon(OnAbandonedFunction OnAbandoned) override { + llvm_unreachable("unexpected abandoned allocation"); + } + + virtual void finalize(OnFinalizedFunction OnFinalized) override { + OnFinalized(ExecutableFileMemoryManager::FinalizedAlloc( + orc::ExecutorAddr::fromPtr(Alloc.release()))); + } +}; + +} // anonymous namespace + +void ExecutableFileMemoryManager::updateSection( + const jitlink::Section &JLSection, uint8_t *Contents, size_t Size, + size_t Alignment) { + auto SectionID = JLSection.getName(); + auto SectionName = sectionName(JLSection, BC); + auto Prot = JLSection.getMemProt(); + auto IsCode = (Prot & orc::MemProt::Exec) != orc::MemProt::None; + auto IsReadOnly = (Prot & orc::MemProt::Write) == orc::MemProt::None; // Register a debug section as a note section. if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) { BinarySection &Section = - BC.registerOrUpdateNoteSection(SectionName, Ret, Size, Alignment); + BC.registerOrUpdateNoteSection(SectionName, Contents, Size, Alignment); Section.setSectionID(SectionID); assert(!Section.isAllocatable() && "note sections cannot be allocatable"); - return Ret; + return; } if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" || SectionName == "" || SectionName.startswith(".rela."))) - return Ret; + return; SmallVector Buf; if (ObjectsLoaded > 0) { @@ -70,7 +147,7 @@ "Original section must exist and be allocatable."); Section = &OrgSection.get(); - Section->updateContents(Ret, Size); + Section->updateContents(Contents, Size); } else { // If the input contains a section with the section name, rename it in the // output file to avoid the section name conflict and emit the new section @@ -87,7 +164,7 @@ // sections in the input file. BinarySection &NewSection = BC.registerOrUpdateSection( UsePrefix ? NewSecPrefix + SectionName : SectionName, ELF::SHT_PROGBITS, - BinarySection::getFlags(IsReadOnly, IsCode, true), Ret, Size, + BinarySection::getFlags(IsReadOnly, IsCode, true), Contents, Size, Alignment); if (UsePrefix) NewSection.setOutputName(SectionName); @@ -100,17 +177,51 @@ << " section : " << Section->getOutputName() << " (" << Section->getName() << ")" << " with size " << Size << ", alignment " << Alignment << " at " - << Ret << ", ID = " << SectionID << "\n"; + << Contents << ", ID = " << SectionID << "\n"; }); Section->setSectionID(SectionID); +} + +void ExecutableFileMemoryManager::allocate(const jitlink::JITLinkDylib *JD, + jitlink::LinkGraph &G, + OnAllocatedFunction OnAllocated) { + auto Alloc = std::make_unique(); + + for (auto *Section : orderedSections(G)) { + if (Section->empty()) + continue; + + auto SectionAlloc = Alloc->allocateSection(*Section); + updateSection(*Section, static_cast(SectionAlloc.Address), + SectionAlloc.Size, SectionAlloc.Alignment); + + size_t CurrentOffset = 0; + auto *Buf = static_cast(SectionAlloc.Address); + for (auto *Block : JITLinkLinker::orderedBlocks(*Section)) { + CurrentOffset = jitlink::alignToBlock(CurrentOffset, *Block); + auto BlockSize = Block->getSize(); + auto *BlockBuf = Buf + CurrentOffset; + + if (Block->isZeroFill()) + std::memset(BlockBuf, 0, BlockSize); + else + std::memcpy(BlockBuf, Block->getContent().data(), BlockSize); + + Block->setMutableContent({BlockBuf, Block->getSize()}); + CurrentOffset += BlockSize; + } + } - return Ret; + OnAllocated(std::make_unique(std::move(Alloc))); } -ExecutableFileMemoryManager::~ExecutableFileMemoryManager() { - for (const AllocInfo &AI : AllocatedSections) - llvm::deallocate_buffer(AI.Address, AI.Size, AI.Alignment); +void ExecutableFileMemoryManager::deallocate( + std::vector Allocs, OnDeallocatedFunction OnDeallocated) { + for (auto &Alloc : Allocs) + delete Alloc.release().toPtr(); + + OnDeallocated(Error::success()); } } // namespace bolt diff --git a/bolt/lib/Rewrite/JITLinkLinker.cpp b/bolt/lib/Rewrite/JITLinkLinker.cpp new file mode 100644 --- /dev/null +++ b/bolt/lib/Rewrite/JITLinkLinker.cpp @@ -0,0 +1,206 @@ +//===- bolt/Rewrite/JITLinkLinker.cpp - BOLTLinker using JITLink ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "bolt/Rewrite/JITLinkLinker.h" +#include "bolt/Core/BinaryData.h" +#include "bolt/Rewrite/RewriteInstance.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorSymbolDef.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bolt" + +namespace llvm { +namespace bolt { + +namespace { + +bool hasSymbols(const jitlink::Block &B) { + return llvm::any_of(B.getSection().symbols(), + [&B](const auto &S) { return &S->getBlock() == &B; }); +} + +/// Liveness in JITLink is based on symbols so sections that do not contain +/// any symbols will always be pruned. This pass adds anonymous symbols to +/// needed sections to prevent pruning. +Error markSectionsLive(jitlink::LinkGraph &G) { + for (auto &Section : G.sections()) { + // We only need allocatable sections. + if (Section.getMemLifetimePolicy() == orc::MemLifetimePolicy::NoAlloc) + continue; + + // Skip empty sections. + if (JITLinkLinker::sectionSize(Section) == 0) + continue; + + for (auto *Block : Section.blocks()) { + // No need to add symbols if it already has some. + if (hasSymbols(*Block)) + continue; + + G.addAnonymousSymbol(*Block, /*Offset=*/0, /*Size=*/0, + /*IsCallable=*/false, /*IsLive=*/true); + } + } + + return jitlink::markAllSymbolsLive(G); +} + +void reassignSectionAddress(jitlink::LinkGraph &LG, + const BinarySection &BinSection, uint64_t Address) { + auto *JLSection = LG.findSectionByName(BinSection.getSectionID()); + assert(JLSection && "cannot find section in LinkGraph"); + + auto BlockAddress = Address; + for (auto *Block : JITLinkLinker::orderedBlocks(*JLSection)) { + // FIXME it would seem to make sense to align here. However, in + // non-relocation mode, we simply use the original address of functions + // which might not be aligned with the minimum alignment used by + // BinaryFunction (2). Example failing test when aligning: + // bolt/test/X86/addr32.s + Block->setAddress(orc::ExecutorAddr(BlockAddress)); + BlockAddress += Block->getSize(); + } +} + +} // anonymous namespace + +struct JITLinkLinker::Context : jitlink::JITLinkContext { + JITLinkLinker &Linker; + JITLinkLinker::SectionsMapper MapSections; + + Context(JITLinkLinker &Linker, JITLinkLinker::SectionsMapper MapSections) + : JITLinkContext(&Linker.Dylib), Linker(Linker), + MapSections(MapSections) {} + + jitlink::JITLinkMemoryManager &getMemoryManager() override { + return *Linker.MM; + } + + bool shouldAddDefaultTargetPasses(const Triple &TT) const override { + // The default passes manipulate DWARF sections in a way incompatible with + // BOLT. + // TODO check if we can actually use these passes to remove some of the + // DWARF manipulation done in BOLT. + return false; + } + + Error modifyPassConfig(jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override { + Config.PrePrunePasses.push_back(markSectionsLive); + return Error::success(); + } + + void notifyFailed(Error Err) override { + errs() << "BOLT-ERROR: JITLink failed: " << Err << '\n'; + exit(1); + } + + void + lookup(const LookupMap &Symbols, + std::unique_ptr LC) override { + jitlink::AsyncLookupResult AllResults; + + for (const auto &Symbol : Symbols) { + std::string SymName = Symbol.first.str(); + LLVM_DEBUG(dbgs() << "BOLT: looking for " << SymName << "\n"); + + if (auto Address = Linker.lookupSymbol(SymName)) { + LLVM_DEBUG(dbgs() << "Resolved to address 0x" + << Twine::utohexstr(*Address) << "\n"); + AllResults[Symbol.first] = orc::ExecutorSymbolDef( + orc::ExecutorAddr(*Address), JITSymbolFlags()); + continue; + } + + if (const BinaryData *I = Linker.BC.getBinaryDataByName(SymName)) { + uint64_t Address = I->isMoved() && !I->isJumpTable() + ? I->getOutputAddress() + : I->getAddress(); + LLVM_DEBUG(dbgs() << "Resolved to address 0x" + << Twine::utohexstr(Address) << "\n"); + AllResults[Symbol.first] = orc::ExecutorSymbolDef( + orc::ExecutorAddr(Address), JITSymbolFlags()); + continue; + } + LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n"); + AllResults[Symbol.first] = + orc::ExecutorSymbolDef(orc::ExecutorAddr(0), JITSymbolFlags()); + } + + LC->run(std::move(AllResults)); + } + + Error notifyResolved(jitlink::LinkGraph &G) override { + MapSections([&G](const BinarySection &Section, uint64_t Address) { + reassignSectionAddress(G, Section, Address); + }); + + for (auto *Symbol : G.defined_symbols()) { + Linker.Symtab.insert( + {Symbol->getName().str(), Symbol->getAddress().getValue()}); + } + + return Error::success(); + } + + void notifyFinalized( + jitlink::JITLinkMemoryManager::FinalizedAlloc Alloc) override { + Linker.Allocs.push_back(std::move(Alloc)); + ++Linker.MM->ObjectsLoaded; + } +}; + +JITLinkLinker::JITLinkLinker(BinaryContext &BC, + std::unique_ptr MM) + : BC(BC), MM(std::move(MM)) {} + +JITLinkLinker::~JITLinkLinker() { cantFail(MM->deallocate(std::move(Allocs))); } + +void JITLinkLinker::loadObject(MemoryBufferRef Obj, + SectionsMapper MapSections) { + auto LG = jitlink::createLinkGraphFromObject(Obj); + if (auto E = LG.takeError()) { + errs() << "BOLT-ERROR: JITLink failed: " << E << '\n'; + exit(1); + } + + auto Ctx = std::make_unique(*this, MapSections); + jitlink::link(std::move(*LG), std::move(Ctx)); +} + +std::optional JITLinkLinker::lookupSymbol(StringRef Name) const { + auto It = Symtab.find(Name.data()); + if (It == Symtab.end()) + return std::nullopt; + + return It->second; +} + +SmallVector +JITLinkLinker::orderedBlocks(const jitlink::Section &Section) { + SmallVector Blocks(Section.blocks()); + llvm::sort(Blocks, [](const auto *LHS, const auto *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + return Blocks; +} + +size_t JITLinkLinker::sectionSize(const jitlink::Section &Section) { + size_t Size = 0; + + for (const auto *Block : orderedBlocks(Section)) { + Size = jitlink::alignToBlock(Size, *Block); + Size += Block->getSize(); + } + + return Size; +} + +} // namespace bolt +} // namespace llvm diff --git a/bolt/lib/Rewrite/MachORewriteInstance.cpp b/bolt/lib/Rewrite/MachORewriteInstance.cpp --- a/bolt/lib/Rewrite/MachORewriteInstance.cpp +++ b/bolt/lib/Rewrite/MachORewriteInstance.cpp @@ -17,6 +17,7 @@ #include "bolt/Profile/DataReader.h" #include "bolt/Rewrite/BinaryPassManager.h" #include "bolt/Rewrite/ExecutableFileMemoryManager.h" +#include "bolt/Rewrite/JITLinkLinker.h" #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h" #include "bolt/Utils/Utils.h" #include "llvm/MC/MCAsmLayout.h" @@ -386,7 +387,8 @@ Manager.runPasses(); } -void MachORewriteInstance::mapInstrumentationSection(StringRef SectionName) { +void MachORewriteInstance::mapInstrumentationSection( + StringRef SectionName, BOLTLinker::SectionMapper MapSection) { if (!opts::Instrument) return; ErrorOr Section = BC->getUniqueSectionByName(SectionName); @@ -396,11 +398,11 @@ } if (!Section->hasValidSectionID()) return; - RTDyld->reassignSectionAddress(Section->getSectionID(), - Section->getAddress()); + MapSection(*Section, Section->getAddress()); } -void MachORewriteInstance::mapCodeSections() { +void MachORewriteInstance::mapCodeSections( + BOLTLinker::SectionMapper MapSection) { for (BinaryFunction *Function : BC->getAllBinaryFunctions()) { if (!Function->isEmitted()) continue; @@ -417,8 +419,7 @@ LLVM_DEBUG(dbgs() << "BOLT: mapping 0x" << Twine::utohexstr(FuncSection->getAllocAddress()) << " to 0x" << Twine::utohexstr(Function->getOutputAddress()) << '\n'); - RTDyld->reassignSectionAddress(FuncSection->getSectionID(), - Function->getOutputAddress()); + MapSection(*FuncSection, Function->getOutputAddress()); Function->setImageAddress(FuncSection->getAllocAddress()); Function->setImageSize(FuncSection->getOutputSize()); } @@ -439,7 +440,7 @@ assert(FuncSection && "cannot find section for function"); Addr = llvm::alignTo(Addr, 4); FuncSection->setOutputAddress(Addr); - RTDyld->reassignSectionAddress(FuncSection->getSectionID(), Addr); + MapSection(*FuncSection, Addr); Function->setFileOffset(Addr - BOLT->getAddress() + BOLT->getInputFileOffset()); Function->setImageAddress(FuncSection->getAllocAddress()); @@ -450,34 +451,6 @@ } } -namespace { - -class BOLTSymbolResolver : public LegacyJITSymbolResolver { - BinaryContext &BC; -public: - BOLTSymbolResolver(BinaryContext &BC) : BC(BC) {} - - JITSymbol findSymbolInLogicalDylib(const std::string &Name) override { - return JITSymbol(nullptr); - } - - JITSymbol findSymbol(const std::string &Name) override { - LLVM_DEBUG(dbgs() << "BOLT: looking for " << Name << "\n"); - if (BinaryData *I = BC.getBinaryDataByName(Name)) { - const uint64_t Address = I->isMoved() && !I->isJumpTable() - ? I->getOutputAddress() - : I->getAddress(); - LLVM_DEBUG(dbgs() << "Resolved to address 0x" << Twine::utohexstr(Address) - << "\n"); - return JITSymbol(Address, JITSymbolFlags()); - } - LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n"); - return JITSymbol(nullptr); - } -}; - -} // end anonymous namespace - void MachORewriteInstance::emitAndLink() { std::error_code EC; std::unique_ptr<::llvm::ToolOutputFile> TempOut = @@ -503,45 +476,38 @@ "error creating in-memory object"); assert(Obj && "createObjectFile cannot return nullptr"); - BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC); - MCAsmLayout FinalLayout( static_cast(Streamer.get())->getAssembler()); - BC->EFMM.reset(new ExecutableFileMemoryManager(*BC, /*AllowStubs*/ false)); - BC->EFMM->setOrgSecPrefix(getOrgSecPrefix()); - BC->EFMM->setNewSecPrefix(getNewSecPrefix()); - - RTDyld.reset(new decltype(RTDyld)::element_type(*BC->EFMM, Resolver)); - RTDyld->setProcessAllSections(true); - RTDyld->loadObject(*Obj); - if (RTDyld->hasError()) { - outs() << "BOLT-ERROR: RTDyld failed.\n"; - exit(1); - } - - // Assign addresses to all sections. If key corresponds to the object - // created by ourselves, call our regular mapping function. If we are - // loading additional objects as part of runtime libraries for - // instrumentation, treat them as extra sections. - mapCodeSections(); - mapInstrumentationSection("__counters"); - mapInstrumentationSection("__tables"); - RTDyld->finalizeWithMemoryManagerLocking(); - - // TODO: Refactor addRuntimeLibSections to work properly on Mach-O - // and use it here. - //FIXME! Put this in RtLibrary->link -// mapInstrumentationSection("I__setup"); -// mapInstrumentationSection("I__fini"); -// mapInstrumentationSection("I__data"); -// mapInstrumentationSection("I__text"); -// mapInstrumentationSection("I__cstring"); -// mapInstrumentationSection("I__literal16"); - -// if (auto *RtLibrary = BC->getRuntimeLibrary()) { -// RtLibrary->link(*BC, ToolPath, *ES, *OLT); -// } + auto EFMM = std::make_unique(*BC); + EFMM->setNewSecPrefix(getNewSecPrefix()); + EFMM->setOrgSecPrefix(getOrgSecPrefix()); + + Linker = std::make_unique(*BC, std::move(EFMM)); + Linker->loadObject(ObjectMemBuffer->getMemBufferRef(), + [this](auto MapSection) { + // Assign addresses to all sections. If key corresponds + // to the object created by ourselves, call our regular + // mapping function. If we are loading additional objects + // as part of runtime libraries for instrumentation, + // treat them as extra sections. + mapCodeSections(MapSection); + mapInstrumentationSection("__counters", MapSection); + mapInstrumentationSection("__tables", MapSection); + }); + + // TODO: Refactor addRuntimeLibSections to work properly on Mach-O + // and use it here. + // if (auto *RtLibrary = BC->getRuntimeLibrary()) { + // RtLibrary->link(*BC, ToolPath, *Linker, [this](auto MapSection) { + // mapInstrumentationSection("I__setup", MapSection); + // mapInstrumentationSection("I__fini", MapSection); + // mapInstrumentationSection("I__data", MapSection); + // mapInstrumentationSection("I__text", MapSection); + // mapInstrumentationSection("I__cstring", MapSection); + // mapInstrumentationSection("I__literal16", MapSection); + // }); + // } } void MachORewriteInstance::writeInstrumentationSection(StringRef SectionName, diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -26,6 +26,7 @@ #include "bolt/Rewrite/BinaryPassManager.h" #include "bolt/Rewrite/DWARFRewriter.h" #include "bolt/Rewrite/ExecutableFileMemoryManager.h" +#include "bolt/Rewrite/JITLinkLinker.h" #include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h" #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h" #include "bolt/Utils/CommandLineOpts.h" @@ -33,7 +34,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Function.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" @@ -3299,69 +3299,6 @@ BinaryFunctionPassManager::runAllPasses(*BC); } -namespace { - -class BOLTSymbolResolver : public JITSymbolResolver { - BinaryContext &BC; - -public: - BOLTSymbolResolver(BinaryContext &BC) : BC(BC) {} - - // We are responsible for all symbols - Expected getResponsibilitySet(const LookupSet &Symbols) override { - return Symbols; - } - - // Some of our symbols may resolve to zero and this should not be an error - bool allowsZeroSymbols() override { return true; } - - /// Resolves the address of each symbol requested - void lookup(const LookupSet &Symbols, - OnResolvedFunction OnResolved) override { - JITSymbolResolver::LookupResult AllResults; - - if (BC.EFMM->ObjectsLoaded) { - for (const StringRef &Symbol : Symbols) { - std::string SymName = Symbol.str(); - LLVM_DEBUG(dbgs() << "BOLT: looking for " << SymName << "\n"); - // Resolve to a PLT entry if possible - if (const BinaryData *I = BC.getPLTBinaryDataByName(SymName)) { - AllResults[Symbol] = - JITEvaluatedSymbol(I->getAddress(), JITSymbolFlags()); - continue; - } - OnResolved(make_error( - "Symbol not found required by runtime: " + Symbol, - inconvertibleErrorCode())); - return; - } - OnResolved(std::move(AllResults)); - return; - } - - for (const StringRef &Symbol : Symbols) { - std::string SymName = Symbol.str(); - LLVM_DEBUG(dbgs() << "BOLT: looking for " << SymName << "\n"); - - if (BinaryData *I = BC.getBinaryDataByName(SymName)) { - uint64_t Address = I->isMoved() && !I->isJumpTable() - ? I->getOutputAddress() - : I->getAddress(); - LLVM_DEBUG(dbgs() << "Resolved to address 0x" - << Twine::utohexstr(Address) << "\n"); - AllResults[Symbol] = JITEvaluatedSymbol(Address, JITSymbolFlags()); - continue; - } - LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n"); - AllResults[Symbol] = JITEvaluatedSymbol(0, JITSymbolFlags()); - } - - OnResolved(std::move(AllResults)); - } -}; - -} // anonymous namespace - void RewriteInstance::preregisterSections() { // Preregister sections before emission to set their order in the output. const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true, @@ -3435,39 +3372,18 @@ // Get output object as ObjectFile. std::unique_ptr ObjectMemBuffer = MemoryBuffer::getMemBuffer(BOS->str(), "in-memory object file", false); - std::unique_ptr Obj = cantFail( - object::ObjectFile::createObjectFile(ObjectMemBuffer->getMemBufferRef()), - "error creating in-memory object"); - BOLTSymbolResolver Resolver = BOLTSymbolResolver(*BC); + auto EFMM = std::make_unique(*BC); + EFMM->setNewSecPrefix(getNewSecPrefix()); + EFMM->setOrgSecPrefix(getOrgSecPrefix()); + + Linker = std::make_unique(*BC, std::move(EFMM)); + Linker->loadObject(ObjectMemBuffer->getMemBufferRef(), + [this](auto MapSection) { mapFileSections(MapSection); }); MCAsmLayout FinalLayout( static_cast(Streamer.get())->getAssembler()); - // Disable stubs because RuntimeDyld may try to increase the size of - // sections accounting for stubs. We need those sections to match the - // same size seen in the input binary, in case this section is a copy - // of the original one seen in the binary. - BC->EFMM.reset(new ExecutableFileMemoryManager(*BC, /*AllowStubs=*/false)); - BC->EFMM->setNewSecPrefix(getNewSecPrefix()); - BC->EFMM->setOrgSecPrefix(getOrgSecPrefix()); - - RTDyld.reset(new decltype(RTDyld)::element_type(*BC->EFMM, Resolver)); - RTDyld->setProcessAllSections(false); - RTDyld->loadObject(*Obj); - - // Assign addresses to all sections. If key corresponds to the object - // created by ourselves, call our regular mapping function. If we are - // loading additional objects as part of runtime libraries for - // instrumentation, treat them as extra sections. - mapFileSections(*RTDyld); - - RTDyld->finalizeWithMemoryManagerLocking(); - if (RTDyld->hasError()) { - errs() << "BOLT-ERROR: RTDyld failed: " << RTDyld->getErrorString() << "\n"; - exit(1); - } - // Update output addresses based on the new section map and // layout. Only do this for the object created by ourselves. updateOutputValues(FinalLayout); @@ -3476,9 +3392,9 @@ DebugInfoRewriter->updateLineTableOffsets(FinalLayout); if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) - RtLibrary->link(*BC, ToolPath, *RTDyld, [this](RuntimeDyld &R) { + RtLibrary->link(*BC, ToolPath, *Linker, [this](auto MapSection) { // Map newly registered sections. - this->mapAllocatableSections(*RTDyld); + this->mapAllocatableSections(MapSection); }); // Once the code is emitted, we can rename function sections to actual @@ -3874,7 +3790,7 @@ << '\n'; } -void RewriteInstance::mapFileSections(RuntimeDyld &RTDyld) { +void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) { BC->deregisterUnusedSections(); // If no new .eh_frame was written, remove relocated original .eh_frame. @@ -3884,19 +3800,18 @@ BinarySection *NewEHFrameSection = getSection(getNewSecPrefix() + getEHFrameSectionName()); if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) { - // RTDyld will still have to process relocations for the section, hence + // JITLink will still have to process relocations for the section, hence // we need to assign it the address that wouldn't result in relocation // processing failure. - RTDyld.reassignSectionAddress(RelocatedEHFrameSection->getSectionID(), - NextAvailableAddress); + MapSection(*RelocatedEHFrameSection, NextAvailableAddress); BC->deregisterSection(*RelocatedEHFrameSection); } } - mapCodeSections(RTDyld); + mapCodeSections(MapSection); // Map the rest of the sections. - mapAllocatableSections(RTDyld); + mapAllocatableSections(MapSection); } std::vector RewriteInstance::getCodeSections() { @@ -3925,7 +3840,7 @@ return CodeSections; } -void RewriteInstance::mapCodeSections(RuntimeDyld &RTDyld) { +void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) { if (BC->HasRelocations) { // Map sections for functions with pre-assigned addresses. for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) { @@ -3937,8 +3852,7 @@ InjectedFunction->getCodeSection(); assert(FunctionSection && "function should have section"); FunctionSection->setOutputAddress(OutputAddress); - RTDyld.reassignSectionAddress(FunctionSection->getSectionID(), - OutputAddress); + MapSection(*FunctionSection, OutputAddress); InjectedFunction->setImageAddress(FunctionSection->getAllocAddress()); InjectedFunction->setImageSize(FunctionSection->getOutputSize()); } @@ -4014,8 +3928,7 @@ dbgs() << "BOLT: mapping " << Section->getName() << " at 0x" << Twine::utohexstr(Section->getAllocAddress()) << " to 0x" << Twine::utohexstr(Section->getOutputAddress()) << '\n'); - RTDyld.reassignSectionAddress(Section->getSectionID(), - Section->getOutputAddress()); + MapSection(*Section, Section->getOutputAddress()); Section->setOutputFileOffset( getFileOffsetForAddress(Section->getOutputAddress())); } @@ -4045,8 +3958,7 @@ << Twine::utohexstr(FuncSection->getAllocAddress()) << " to 0x" << Twine::utohexstr(Function.getAddress()) << '\n'); - RTDyld.reassignSectionAddress(FuncSection->getSectionID(), - Function.getAddress()); + MapSection(*FuncSection, Function.getAddress()); Function.setImageAddress(FuncSection->getAllocAddress()); Function.setImageSize(FuncSection->getOutputSize()); if (Function.getImageSize() > Function.getMaxSize()) { @@ -4064,7 +3976,7 @@ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: mapping JT " << Section.getName() << " to 0x" << Twine::utohexstr(JT->getAddress()) << '\n'); - RTDyld.reassignSectionAddress(Section.getSectionID(), JT->getAddress()); + MapSection(Section, JT->getAddress()); } } @@ -4100,7 +4012,7 @@ dbgs() << formatv( "BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n", FF.getImageAddress(), FF.getAddress(), FF.getImageSize())); - RTDyld.reassignSectionAddress(ColdSection->getSectionID(), FF.getAddress()); + MapSection(*ColdSection, FF.getAddress()); if (TooLarge) BC->deregisterSection(*ColdSection); @@ -4130,7 +4042,8 @@ } } -void RewriteInstance::mapAllocatableSections(RuntimeDyld &RTDyld) { +void RewriteInstance::mapAllocatableSections( + BOLTLinker::SectionMapper MapSection) { // Allocate read-only sections first, then writable sections. enum : uint8_t { ST_READONLY, ST_READWRITE }; for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) { @@ -4164,8 +4077,7 @@ }); Section.setOutputAddress(Section.getAddress()); Section.setOutputFileOffset(Section.getInputFileOffset()); - RTDyld.reassignSectionAddress(Section.getSectionID(), - Section.getAddress()); + MapSection(Section, Section.getAddress()); } else { NextAvailableAddress = alignTo(NextAvailableAddress, Section.getAlignment()); @@ -4178,8 +4090,7 @@ << '\n'; }); - RTDyld.reassignSectionAddress(Section.getSectionID(), - NextAvailableAddress); + MapSection(Section, NextAvailableAddress); Section.setOutputAddress(NextAvailableAddress); Section.setOutputFileOffset( getFileOffsetForAddress(NextAvailableAddress)); @@ -5992,9 +5903,9 @@ } uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) { - uint64_t Value = RTDyld->getSymbol(Name).getAddress(); - if (Value != 0) - return Value; + auto Value = Linker->lookupSymbol(Name); + if (Value) + return *Value; // Return the original value if we haven't emitted the symbol. BinaryData *BD = BC->getBinaryDataByName(Name); diff --git a/bolt/lib/RuntimeLibs/CMakeLists.txt b/bolt/lib/RuntimeLibs/CMakeLists.txt --- a/bolt/lib/RuntimeLibs/CMakeLists.txt +++ b/bolt/lib/RuntimeLibs/CMakeLists.txt @@ -1,8 +1,8 @@ set(LLVM_LINK_COMPONENTS BinaryFormat + JITLink MC Object - RuntimeDyld Support ) diff --git a/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp --- a/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/HugifyRuntimeLibrary.cpp @@ -12,7 +12,7 @@ #include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h" #include "bolt/Core/BinaryFunction.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "bolt/Core/Linker.h" #include "llvm/MC/MCStreamer.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/CommandLine.h" @@ -61,20 +61,15 @@ } void HugifyRuntimeLibrary::link(BinaryContext &BC, StringRef ToolPath, - RuntimeDyld &RTDyld, - std::function OnLoad) { + BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) { + std::string LibPath = getLibPath(ToolPath, opts::RuntimeHugifyLib); - loadLibrary(LibPath, RTDyld); - OnLoad(RTDyld); - RTDyld.finalizeWithMemoryManagerLocking(); - if (RTDyld.hasError()) { - outs() << "BOLT-ERROR: RTDyld failed: " << RTDyld.getErrorString() << "\n"; - exit(1); - } + loadLibrary(LibPath, Linker, MapSections); assert(!RuntimeStartAddress && "We don't currently support linking multiple runtime libraries"); - RuntimeStartAddress = RTDyld.getSymbol("__bolt_hugify_self").getAddress(); + RuntimeStartAddress = Linker.lookupSymbol("__bolt_hugify_self").value_or(0); if (!RuntimeStartAddress) { errs() << "BOLT-ERROR: instrumentation library does not define " "__bolt_hugify_self: " diff --git a/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp --- a/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/InstrumentationRuntimeLibrary.cpp @@ -13,8 +13,8 @@ #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h" #include "bolt/Core/BinaryFunction.h" #include "bolt/Core/JumpTable.h" +#include "bolt/Core/Linker.h" #include "bolt/Utils/CommandLineOpts.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/MC/MCStreamer.h" #include "llvm/Support/Alignment.h" #include "llvm/Support/CommandLine.h" @@ -191,28 +191,22 @@ } void InstrumentationRuntimeLibrary::link( - BinaryContext &BC, StringRef ToolPath, RuntimeDyld &RTDyld, - std::function OnLoad) { + BinaryContext &BC, StringRef ToolPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) { std::string LibPath = getLibPath(ToolPath, opts::RuntimeInstrumentationLib); - loadLibrary(LibPath, RTDyld); - OnLoad(RTDyld); - RTDyld.finalizeWithMemoryManagerLocking(); - if (RTDyld.hasError()) { - outs() << "BOLT-ERROR: RTDyld failed: " << RTDyld.getErrorString() << "\n"; - exit(1); - } + loadLibrary(LibPath, Linker, MapSections); if (BC.isMachO()) return; - RuntimeFiniAddress = RTDyld.getSymbol("__bolt_instr_fini").getAddress(); + RuntimeFiniAddress = Linker.lookupSymbol("__bolt_instr_fini").value_or(0); if (!RuntimeFiniAddress) { errs() << "BOLT-ERROR: instrumentation library does not define " "__bolt_instr_fini: " << LibPath << "\n"; exit(1); } - RuntimeStartAddress = RTDyld.getSymbol("__bolt_instr_start").getAddress(); + RuntimeStartAddress = Linker.lookupSymbol("__bolt_instr_start").value_or(0); if (!RuntimeStartAddress) { errs() << "BOLT-ERROR: instrumentation library does not define " "__bolt_instr_start: " @@ -224,7 +218,7 @@ << Twine::utohexstr(RuntimeFiniAddress) << "\n"; outs() << "BOLT-INFO: clear procedure is 0x" << Twine::utohexstr( - RTDyld.getSymbol("__bolt_instr_clear_counters").getAddress()) + Linker.lookupSymbol("__bolt_instr_clear_counters").value_or(0)) << "\n"; emitTablesAsELFNote(BC); diff --git a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp --- a/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp +++ b/bolt/lib/RuntimeLibs/RuntimeLibrary.cpp @@ -11,11 +11,12 @@ //===----------------------------------------------------------------------===// #include "bolt/RuntimeLibs/RuntimeLibrary.h" +#include "bolt/Core/Linker.h" #include "bolt/RuntimeLibs/RuntimeLibraryVariables.inc" #include "bolt/Utils/Utils.h" #include "llvm/BinaryFormat/Magic.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/Object/Archive.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/Support/Path.h" #define DEBUG_TYPE "bolt-rtlib" @@ -44,7 +45,8 @@ return std::string(LibPath.str()); } -void RuntimeLibrary::loadLibrary(StringRef LibPath, RuntimeDyld &RTDyld) { +void RuntimeLibrary::loadLibrary(StringRef LibPath, BOLTLinker &Linker, + BOLTLinker::SectionsMapper MapSections) { ErrorOr> MaybeBuf = MemoryBuffer::getFile(LibPath, false, false); check_error(MaybeBuf.getError(), LibPath); @@ -57,7 +59,7 @@ for (const object::Archive::Child &C : Archive.children(Err)) { std::unique_ptr Bin = cantFail(C.getAsBinary()); if (object::ObjectFile *Obj = dyn_cast(&*Bin)) - RTDyld.loadObject(*Obj); + Linker.loadObject(Obj->getMemoryBufferRef(), MapSections); } check_error(std::move(Err), B->getBufferIdentifier()); } else if (Magic == file_magic::elf_relocatable || @@ -65,7 +67,7 @@ std::unique_ptr Obj = cantFail( object::ObjectFile::createObjectFile(B.get()->getMemBufferRef()), "error creating in-memory object"); - RTDyld.loadObject(*Obj); + Linker.loadObject(Obj->getMemoryBufferRef(), MapSections); } else { errs() << "BOLT-ERROR: unrecognized library format: " << LibPath << "\n"; exit(1);