Index: bolt/include/bolt/Core/BinaryContext.h =================================================================== --- bolt/include/bolt/Core/BinaryContext.h +++ bolt/include/bolt/Core/BinaryContext.h @@ -59,25 +59,76 @@ class BinaryFunction; -/// Information on loadable part of the file. -struct SegmentInfo { - uint64_t Address; /// Address of the segment in memory. - uint64_t Size; /// Size of the segment in memory. - uint64_t FileOffset; /// Offset in the file. - uint64_t FileSize; /// Size in file. - uint64_t Alignment; /// Alignment of the segment. - +// We create a copy of ELF64LE::Phdr type because ELF64LE::Phdr uses +// endian-aware members which use memcpy for assignments and other operations. +// We will use this struct instead to avoid memcpy and convert to ELF64LE::Phdr +// only before writing to file. +struct ProgramHeader { + uint32_t p_type{0}; + uint32_t p_flags{0}; + uint64_t p_offset{0}; + uint64_t p_vaddr{0}; + uint64_t p_paddr{0}; + uint64_t p_filesz{0}; + uint64_t p_memsz{0}; + uint64_t p_align{0}; + + ProgramHeader(const ELF64LE::Phdr &Hdr) { + p_type = Hdr.p_type; + p_flags = Hdr.p_flags; + p_offset = Hdr.p_offset; + p_vaddr = Hdr.p_vaddr; + p_paddr = Hdr.p_paddr; + p_filesz = Hdr.p_filesz; + p_memsz = Hdr.p_memsz; + p_align = Hdr.p_align; + } + ProgramHeader() {} + + ProgramHeader(uint32_t p_type, uint32_t p_flags, uint64_t p_offset, + uint64_t p_vaddr, uint64_t p_paddr, uint64_t p_filesz, + uint64_t p_memsz, uint64_t p_align) + : p_type(p_type), p_flags(p_flags), p_offset(p_offset), p_vaddr(p_vaddr), + p_paddr(p_paddr), p_filesz(p_filesz), p_memsz(p_memsz), + p_align(p_align) {} + explicit operator ELF64LE::Phdr() const { + ELF64LE::Phdr Res; + Res.p_type = p_type; + Res.p_flags = p_flags; + Res.p_offset = p_offset; + Res.p_vaddr = p_vaddr; + Res.p_paddr = p_paddr; + Res.p_filesz = p_filesz; + Res.p_memsz = p_memsz; + Res.p_align = p_align; + return Res; + } void print(raw_ostream &OS) const { - OS << "SegmentInfo { Address: 0x" - << Twine::utohexstr(Address) << ", Size: 0x" - << Twine::utohexstr(Size) << ", FileOffset: 0x" - << Twine::utohexstr(FileOffset) << ", FileSize: 0x" - << Twine::utohexstr(FileSize) << ", Alignment: 0x" - << Twine::utohexstr(Alignment) << "}"; - }; + OS << "SegmentInfo { Address: 0x" << Twine::utohexstr(p_vaddr) + << ", Size: 0x" << Twine::utohexstr(p_memsz) << ", FileOffset: 0x" + << Twine::utohexstr(p_offset) << ", FileSize: 0x" + << Twine::utohexstr(p_filesz) << ", Alignment: 0x" + << Twine::utohexstr(p_align) << "}"; + } + bool contains(const BinarySection &Section) const { + return (Section.getAddress() && Section.getInputFileOffset() && + Section.getAddress() >= p_vaddr && + (Section.getAddress() + !Section.isTBSS() * Section.getSize() <= + p_vaddr + p_memsz)); + } + + bool isTLS() const { return p_type == ELF::PT_TLS; } + bool isLOAD() const { return p_type == ELF::PT_LOAD; } + bool isExec() const { return p_flags & ELF::PF_X; } + // intended for loadable segments + unsigned getSectionFlags() const { + return ELF::SHF_ALLOC * (p_type == ELF::PT_LOAD) | + ELF::SHF_EXECINSTR * !!(p_flags & ELF::PF_X) | + ELF::SHF_WRITE * !!(p_flags & ELF::PF_W); + } }; -inline raw_ostream &operator<<(raw_ostream &OS, const SegmentInfo &SegInfo) { +inline raw_ostream &operator<<(raw_ostream &OS, const ProgramHeader &SegInfo) { SegInfo.print(OS); return OS; } @@ -240,6 +291,8 @@ /// overwritten, but it is okay to re-generate debug info for them. std::set ProcessedCUs; + StringMap EndSymbols; + // Setup MCPlus target builder void initializeTarget(std::unique_ptr TargetBuilder) { MIB = std::move(TargetBuilder); @@ -286,9 +339,18 @@ std::optional Source, unsigned CUID, unsigned DWARFVersion); - /// [start memory address] -> [segment info] mapping. - std::map SegmentMapInfo; + /// [start memory address] -> [program header] mapping for loadable input + /// segments + std::map SegmentMapInfo; + // [address] -> [offset] mappings for output loadable segments + std::map OutputAddressToOffsetMap; + std::vector OutputSegments; + + using SegmentIterator = decltype(OutputSegments)::iterator; + using FilteredSegmentIterator = FilterIterator; + + std::vector InputSegments; /// Symbols that are expected to be undefined in MCContext during emission. std::unordered_set UndefinedSymbols; @@ -658,15 +720,12 @@ // Address of the first allocated segment. uint64_t FirstAllocAddress{std::numeric_limits::max()}; - /// Track next available address for new allocatable sections. RewriteInstance - /// sets this prior to running BOLT passes, so layout passes are aware of the - /// final addresses functions will have. - uint64_t LayoutStartAddress{0}; - - /// Old .text info. + // The end of the last loadable segment in the input + uint64_t InputAddressSpaceEnd{0}; uint64_t OldTextSectionAddress{0}; - uint64_t OldTextSectionOffset{0}; - uint64_t OldTextSectionSize{0}; + // Computed in advance when not using --rewrite, otherwise unused + uint64_t NewTextSectionAddress{0}; + uint64_t MaxPHDRSize{0}; /// Address of the code/function that is executed before any other code in /// the binary. @@ -725,6 +784,7 @@ } bool isRISCV() const { return TheTriple->getArch() == llvm::Triple::riscv64; } + bool isRISC() const { return isRISCV() || isAArch64(); } // AArch64-specific functions to check if symbol is used to delimit // code/data in .text. Code is marked by $x, data by $d. @@ -769,6 +829,20 @@ FilteredBinaryDataIterator(pred, End, End)); } + std::vector getNewSectionsByFlags(unsigned Flags, + bool ROwithRX = false); + + std::vector getAllNewSections() { + + std::vector Result = getNewSectionsByFlags( + ELF::SHF_ALLOC | ELF::SHF_EXECINSTR, /*ROwithRX*/ true); + std::vector RWSections = + getNewSectionsByFlags(ELF::SHF_ALLOC | ELF::SHF_WRITE); + + Result.insert(Result.end(), RWSections.begin(), RWSections.end()); + + return Result; + } /// Iterate over all the sub-symbols of /p BD (if any). iterator_range getSubBinaryData(BinaryData *BD); @@ -874,6 +948,11 @@ Name.startswith("HOLEat"); } + // true if a section wasn't present in the input binary + bool isExtra(BinarySection &Section) const { + return !Section.getInputFileOffset(); + } + MCSymbol *getHotTextStartSymbol() const { return Ctx->getOrCreateSymbol("__hot_start"); } @@ -1027,6 +1106,24 @@ FilteredSectionIterator(isAllocatable, Sections.end(), Sections.end())); } + iterator_range loadableSegments() { + auto IsLoad = [](const SegmentIterator &Phdr) { return Phdr->isLOAD(); }; + return make_range(FilteredSegmentIterator(IsLoad, InputSegments.begin(), + InputSegments.end()), + FilteredSegmentIterator(IsLoad, InputSegments.end(), + InputSegments.end())); + } + + iterator_range nonLoadableSegments() { + auto IsNotLoad = [](const SegmentIterator &Phdr) { + return !Phdr->isLOAD(); + }; + return make_range(FilteredSegmentIterator(IsNotLoad, InputSegments.begin(), + InputSegments.end()), + FilteredSegmentIterator(IsNotLoad, InputSegments.end(), + InputSegments.end())); + } + /// Iterate over all registered code sections. iterator_range textSections() { auto isText = [](const SectionIterator &Itr) { @@ -1079,7 +1176,7 @@ /// Check if the address belongs to this binary's static allocation space. bool containsAddress(uint64_t Address) const { - return Address >= FirstAllocAddress && Address < LayoutStartAddress; + return Address >= FirstAllocAddress && Address < InputAddressSpaceEnd; } /// Return section name containing the given \p Address. Index: bolt/include/bolt/Core/BinaryFunction.h =================================================================== --- bolt/include/bolt/Core/BinaryFunction.h +++ bolt/include/bolt/Core/BinaryFunction.h @@ -1309,7 +1309,7 @@ /// Return true if the function body is non-contiguous. bool isSplit() const { return isSimple() && getLayout().isSplit(); } - bool shouldPreserveNops() const { return PreserveNops; } + bool shouldPreserveNops() const { return isPseudo() || PreserveNops; } /// Return true if the function has exception handling tables. bool hasEHRanges() const { return HasEHRanges; } @@ -2066,6 +2066,7 @@ // adjustments. void handleAArch64IndirectCall(MCInst &Instruction, const uint64_t Offset); + bool handlePLT(); /// Scan function for references to other functions. In relocation mode, /// add relocations for external references. In non-relocation mode, detect /// and mark new entry points. Index: bolt/include/bolt/Core/BinarySection.h =================================================================== --- bolt/include/bolt/Core/BinarySection.h +++ bolt/include/bolt/Core/BinarySection.h @@ -159,7 +159,8 @@ : BC(BC), Name(getName(Section)), Section(Section), Contents(getContents(Section)), Address(Section.getAddress()), Size(Section.getSize()), Alignment(Section.getAlignment().value()), - OutputName(Name), SectionNumber(++Count) { + OutputName(Name), OutputSize(Size), OutputContents(Contents), + SectionNumber(++Count) { if (isELF()) { ELFType = ELFSectionRef(Section).getType(); ELFFlags = ELFSectionRef(Section).getFlags(); @@ -214,34 +215,19 @@ // Order sections by their immutable properties. bool operator<(const BinarySection &Other) const { - // Allocatable before non-allocatable. - if (isAllocatable() != Other.isAllocatable()) - return isAllocatable() > Other.isAllocatable(); - - // Input sections take precedence. - if (hasSectionRef() != Other.hasSectionRef()) - return hasSectionRef() > Other.hasSectionRef(); - - // Compare allocatable input sections by their address. - if (hasSectionRef() && getAddress() != Other.getAddress()) + if (getAddress() != Other.getAddress()) return getAddress() < Other.getAddress(); - if (hasSectionRef() && getAddress() && getSize() != Other.getSize()) - return getSize() < Other.getSize(); - // Code before data. - if (isText() != Other.isText()) - return isText() > Other.isText(); + // We want to keep tdata/tbss together to properly calculate + // TLS segment size, and since .tbss may have the same address + // as the section after it, we have this check + if (isTBSS() || Other.isTBSS()) + return isTBSS() && !Other.isTBSS(); - // Read-only before writable. - if (isWritable() != Other.isWritable()) - return isWritable() < Other.isWritable(); - - // BSS at the end. - if (isBSS() != Other.isBSS()) - return isBSS() < Other.isBSS(); + if (getSize() != Other.getSize()) + return getSize() < Other.getSize(); - // Otherwise, preserve the order of creation. - return SectionNumber < Other.SectionNumber; + return getName() < Other.getName(); } /// @@ -253,6 +239,7 @@ StringRef getName() const { return Name; } uint64_t getAddress() const { return Address; } uint64_t getEndAddress() const { return Address + Size; } + uint64_t getOutputEndAddress() const { return OutputAddress + OutputSize; } uint64_t getSize() const { return Size; } uint64_t getInputFileOffset() const { return InputFileOffset; } Align getAlign() const { return Align(Alignment); } @@ -280,7 +267,7 @@ bool isWritable() const { return (ELFFlags & ELF::SHF_WRITE); } bool isAllocatable() const { if (isELF()) { - return (ELFFlags & ELF::SHF_ALLOC) && !isTBSS(); + return (ELFFlags & ELF::SHF_ALLOC); } else { // On non-ELF assume all sections are allocatable. return true; @@ -344,6 +331,8 @@ /// Does this section have any pending relocations? bool hasPendingRelocations() const { return !PendingRelocations.empty(); } + bool hasDynamicRelocations() const { return !DynamicRelocations.empty(); } + /// Remove non-pending relocation with the given /p Offset. bool removeRelocationAt(uint64_t Offset) { auto Itr = Relocations.find(Offset); @@ -357,18 +346,8 @@ void clearRelocations(); - /// Add a new relocation at the given /p Offset. void addRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0, - bool Pending = false) { - assert(Offset < getSize() && "offset not within section bounds"); - if (!Pending) { - Relocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); - } else { - PendingRelocations.emplace_back( - Relocation{Offset, Symbol, Type, Addend, Value}); - } - } + uint64_t Addend, uint64_t Value = 0, bool Pending = false); /// Add a dynamic relocation at the given /p Offset. void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, @@ -401,7 +380,9 @@ return Itr != Relocations.end() ? &*Itr : nullptr; } - /// Lookup the relocation (if any) at the given /p Offset. + uint64_t getNewEndSymbolValue(uint64_t RelocationOffset) const; + + /// Lookup the dynamic relocation (if any) at the given /p Offset. const Relocation *getDynamicRelocationAt(uint64_t Offset) const { Relocation Key{Offset, 0, 0, 0, 0}; auto Itr = DynamicRelocations.find(Key); Index: bolt/include/bolt/Core/MCPlusBuilder.h =================================================================== --- bolt/include/bolt/Core/MCPlusBuilder.h +++ bolt/include/bolt/Core/MCPlusBuilder.h @@ -595,8 +595,18 @@ return false; } + virtual bool isCall64m(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + virtual bool isLeave(const MCInst &Inst) const { return false; } + virtual bool isADD(const MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + virtual bool isADRP(const MCInst &Inst) const { llvm_unreachable("not implemented"); return false; @@ -1075,6 +1085,11 @@ return false; } + virtual bool relaxLdrToAdd(MCInst &Inst) const { + llvm_unreachable("not implemented"); + return false; + } + /// Replace the compound memory operand of Inst with an immediate operand. /// The value of the immediate operand is computed by reading the \p /// ConstantData array starting from \p offset and assuming little-endianess. @@ -1438,8 +1453,7 @@ /// Analyze branch \p Instruction in PLT section and try to determine /// associated got entry address. - virtual uint64_t analyzePLTEntry(MCInst &Instruction, - InstructionIterator Begin, + virtual uint64_t analyzePLTEntry(InstructionIterator Begin, InstructionIterator End, uint64_t BeginPC) const { llvm_unreachable("not implemented"); @@ -1487,11 +1501,19 @@ llvm_unreachable("not implemented"); } - virtual bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const { + // match adrp+add or adrp+ldr + virtual bool matchAdrpPair(const MCInst &Adrp, const MCInst &AddOrLdr) const { llvm_unreachable("not implemented"); return false; } + virtual bool patchPLTInstructions(InstructionIterator Begin, + InstructionIterator End, + const MCSymbol *Target, MCContext *Ctx, + const MCSymbol *BFSymbol) const { + llvm_unreachable("not implemented"); + } + virtual int getShortJmpEncodingSize() const { llvm_unreachable("not implemented"); } Index: bolt/include/bolt/Passes/FixRelaxationPass.h =================================================================== --- bolt/include/bolt/Passes/FixRelaxationPass.h +++ /dev/null @@ -1,40 +0,0 @@ -//===- bolt/Passes/ADRRelaxationPass.h --------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file declares the FixRelaxations class, which locates instructions with -// wrong targets and fixes them. Such problems usually occures when linker -// relaxes (changes) instructions, but doesn't fix relocations types properly -// for them. -// -//===----------------------------------------------------------------------===// - -#ifndef BOLT_PASSES_FIXRELAXATIONPASS_H -#define BOLT_PASSES_FIXRELAXATIONPASS_H - -#include "bolt/Passes/BinaryPasses.h" - -namespace llvm { -namespace bolt { - -class FixRelaxations : public BinaryFunctionPass { - void runOnFunction(BinaryFunction &Function); - -public: - explicit FixRelaxations(const cl::opt &PrintPass) - : BinaryFunctionPass(PrintPass) {} - - const char *getName() const override { return "fix-relaxations"; } - - /// Pass entry point - void runOnFunctions(BinaryContext &BC) override; -}; - -} // namespace bolt -} // namespace llvm - -#endif Index: bolt/include/bolt/Passes/LongJmp.h =================================================================== --- bolt/include/bolt/Passes/LongJmp.h +++ bolt/include/bolt/Passes/LongJmp.h @@ -53,6 +53,7 @@ /// Hold tentative addresses FuncAddressesMapTy HotAddresses; FuncAddressesMapTy ColdAddresses; + uint64_t TentativePLTAddress{0}; DenseMap BBAddresses; /// Used to identify the stub size Index: bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h =================================================================== --- bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h +++ bolt/include/bolt/Rewrite/ExecutableFileMemoryManager.h @@ -51,9 +51,6 @@ OnDeallocatedFunction OnDeallocated) override; using JITLinkMemoryManager::deallocate; - /// Section name management. - void setNewSecPrefix(StringRef Prefix) { NewSecPrefix = Prefix; } - void setOrgSecPrefix(StringRef Prefix) { OrgSecPrefix = Prefix; } }; } // namespace bolt Index: bolt/include/bolt/Rewrite/RewriteInstance.h =================================================================== --- bolt/include/bolt/Rewrite/RewriteInstance.h +++ bolt/include/bolt/Rewrite/RewriteInstance.h @@ -129,7 +129,7 @@ void readRelocations(const object::SectionRef &Section); /// Handle one relocation. - void handleRelocation(const object::SectionRef &RelocatedSection, + void handleRelocation(BinarySection &RelocatedSection, const RelocationRef &Rel); /// Mark functions that are not meant for processing as ignored. @@ -155,7 +155,7 @@ void postProcessFunctions(); - void preregisterSections(); + void renameAndPreregisterSections(); /// Run optimizations that operate at the binary, or post-linker, level. void runOptimizationPasses(); @@ -177,17 +177,52 @@ /// Update debug and other auxiliary information in the file. void updateMetadata(); - /// Return the list of code sections in the output order. - std::vector getCodeSections(); + // Get sections within address space of a provided segment + std::vector getSectionsForSegment(const ProgramHeader &Phdr, + bool IncludeNew = true); - /// Map all sections to their final addresses. - void mapFileSections(BOLTLinker::SectionMapper MapSection); + // Put a non-load segment in the vector of output segments + void createNonLoadSegment(const std::vector &Sections, + const unsigned p_type, const unsigned p_flags, + const unsigned p_align); - /// Map code sections generated by BOLT. - void mapCodeSections(BOLTLinker::SectionMapper MapSection); + // Put a load segment in the vector of output segments and + // create a mapping for it + void + createLoadSegment(const std::vector &Sections, + BOLTLinker::SectionMapper AssignAddress, + const unsigned p_flags, const unsigned p_align, + const std::optional ForceAddress = std::nullopt); - /// Map the rest of allocatable sections. - void mapAllocatableSections(BOLTLinker::SectionMapper MapSection); + // Assign addresses to a group of sections, return size of nobits + uint64_t mapSectionGroup(const std::vector &Sections, + BOLTLinker::SectionMapper AssignAddress); + + void mapFunctionsNonRelocMode(BOLTLinker::SectionMapper AssignAddress); + + // Assign address to a single section + void mapSection(BinarySection &Section); + + // Assign addresses to all allocatable sections + void mapAllocatableSections(BOLTLinker::SectionMapper AssignAddress); + + /// Assign addresses and create segments for the loadable part of executable + void mapLoadableSegments(BOLTLinker::SectionMapper AssignAddress); + + /// Assign addresses and create segments for the runtime library if present + void mapRuntimeLibrary(BOLTLinker::SectionMapper AssignAddress); + + // Copy segment from input to output as it is + void copySegment(const ProgramHeader &Segment, + BOLTLinker::SectionMapper AssignAddress); + + // If we don't rewrite, we remap old segments to the same addresses + // and copy their respective sections. Then create new segments and map + // new sections into them. + void remapLoadableSegments(BOLTLinker::SectionMapper AssignAddress); + + // Copy non-loadable segments to the output, updating offsets/addresses + void mapNonLoadableSegments(); /// Update output object's values based on the final \p Layout. void updateOutputValues(const MCAsmLayout &Layout); @@ -243,14 +278,19 @@ /// Write .eh_frame_hdr. void writeEHFrameHeader(); + void mapEhFrameAndHeader(BOLTLinker::SectionMapper AssignAddress); + + void createGOTPLTRelocations(); + /// Disassemble and create function entries for PLT. void disassemblePLT(); /// Auxiliary function to create .plt BinaryFunction on \p EntryAddres /// with the \p EntrySize size. \p TargetAddress is the .got entry /// associated address. - void createPLTBinaryFunction(uint64_t TargetAddress, uint64_t EntryAddress, - uint64_t EntrySize); + BinaryFunction *createPLTBinaryFunction(uint64_t TargetAddress, + uint64_t EntryAddress, + uint64_t EntrySize); /// Disassemble aarch64-specific .plt \p Section auxiliary function void disassemblePLTSectionAArch64(BinarySection &Section); @@ -276,8 +316,7 @@ return FUNC(ELF64BE); \ } - /// Patch ELF book-keeping info. - void patchELFPHDRTable(); + void writeELFPHDRTable(); /// Create section header table. ELF_FUNCTION(void, patchELFSectionHeaderTable); @@ -291,9 +330,6 @@ /// Patch dynamic section/segment of ELF. ELF_FUNCTION(void, patchELFDynamic); - /// Patch .got - ELF_FUNCTION(void, patchELFGOT); - /// Patch allocatable relocation sections. ELF_FUNCTION(void, patchELFAllocatableRelaSections); @@ -313,6 +349,7 @@ /// Return true if \p Section should be stripped from the output binary. template bool shouldStrip(const ELFShdrTy &Section, StringRef SectionName); + bool shouldStrip(const BinarySection &Section); /// Write ELF symbol table using \p Write and \p AddToStrTab functions /// based on the input file symbol table passed in \p SymTabSection. @@ -349,18 +386,11 @@ /// rewritten binary. void patchBuildID(); - /// Return file offset corresponding to a given virtual address. - uint64_t getFileOffsetFor(uint64_t Address) { - assert(Address >= NewTextSegmentAddress && - "address in not in the new text segment"); - return Address - NewTextSegmentAddress + NewTextSegmentOffset; - } - /// Return file offset corresponding to a virtual \p Address. /// Return 0 if the address has no mapping in the file, including being /// part of .bss section. - uint64_t getFileOffsetForAddress(uint64_t Address) const; - + uint64_t getFileOffsetForAddress(const uint64_t Address) const; + uint64_t getOutputFileOffsetForAddress(const uint64_t Address) const; /// Return true if we will overwrite contents of the section instead /// of appending contents to it. bool willOverwriteSection(StringRef SectionName); @@ -422,6 +452,7 @@ /// Common section names. static StringRef getEHFrameSectionName() { return ".eh_frame"; } + static StringRef getEHFrameHeaderSectionName() { return ".eh_frame_hdr"; } /// An instance of the input binary we are processing, externally owned. llvm::object::ELFObjectFileBase *InputFile; @@ -444,21 +475,12 @@ /// Offset in the input file where non-allocatable sections start. uint64_t FirstNonAllocatableOffset{0}; - /// Information about program header table. uint64_t PHDRTableAddress{0}; uint64_t PHDRTableOffset{0}; - unsigned Phnum{0}; - - /// New code segment info. - uint64_t NewTextSegmentAddress{0}; - uint64_t NewTextSegmentOffset{0}; - uint64_t NewTextSegmentSize{0}; - - /// New writable segment info. - uint64_t NewWritableSegmentAddress{0}; - uint64_t NewWritableSegmentSize{0}; - + uint64_t BaseAddress{0}; + bool HasProgramHeaderSegment{false}; + bool HasReadOnlySegment{false}; /// Track next available address for new allocatable sections. uint64_t NextAvailableAddress{0}; @@ -483,6 +505,12 @@ /// is filled and used only with the relocations-related symbols. std::unordered_map SymbolIndex; + // A map to get GOT entry address by symbol name for AArch64. + // Used to resolve non-relaxed GOT accesses. + // It is not strictly GOT and may include entries from init_array + // or similar sections since references to them use GOT-typed relocations. + std::unordered_map GOTSymbolsByName; + /// Store all non-zero symbols in this map for a quick address lookup. std::map FileSymRefs; Index: bolt/include/bolt/Utils/CommandLineOpts.h =================================================================== --- bolt/include/bolt/Utils/CommandLineOpts.h +++ bolt/include/bolt/Utils/CommandLineOpts.h @@ -59,6 +59,7 @@ extern llvm::cl::opt StrictMode; extern llvm::cl::opt TimeOpts; extern llvm::cl::opt UseOldText; +extern llvm::cl::opt Rewrite; extern llvm::cl::opt UpdateDebugSections; // The default verbosity level (0) is pretty terse, level 1 is fairly Index: bolt/lib/Core/BinaryContext.cpp =================================================================== --- bolt/lib/Core/BinaryContext.cpp +++ bolt/lib/Core/BinaryContext.cpp @@ -49,7 +49,7 @@ namespace opts { -cl::opt NoHugePages("no-huge-pages", +cl::opt NoHugePages("no-huge-pages", cl::init(true), cl::desc("use regular size pages for code alignment"), cl::Hidden, cl::cat(BoltCategory)); @@ -375,6 +375,47 @@ return make_range(Start, End); } +std::vector +BinaryContext::getNewSectionsByFlags(unsigned Flags, bool ROwithRX) { + std::vector Result; + for (BinarySection &Section : allocatableSections()) { + if (!isExtra(Section)) + continue; + if (Section.getOutputAddress() || !Section.hasValidSectionID()) + continue; + + const unsigned SecFlags = Section.getELFFlags(); + if (SecFlags == Flags || (ROwithRX && SecFlags == ELF::SHF_ALLOC && + Flags == (ELF::SHF_ALLOC | ELF::SHF_EXECINSTR))) + Result.push_back(&Section); + } + + if (Flags & ELF::SHF_EXECINSTR) { + + auto CompareSections = [&](const BinarySection *A, const BinarySection *B) { + // executable first + if (A->isText() != B->isText()) + return A->isText() > B->isText(); + // non-executable last + if (!A->isText()) + return false; + // Place movers before anything else. + if (A->getName() == getHotTextMoverSectionName()) + return true; + if (B->getName() == getHotTextMoverSectionName()) + return false; + + // Depending on the option, put main text at the beginning or at the end. + if (opts::HotFunctionsAtEnd) + return B->getName() == getMainCodeSectionName(); + else + return A->getName() == getMainCodeSectionName(); + }; + llvm::stable_sort(Result, CompareSections); + } + return Result; +} + std::pair BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF, bool IsPCRel) { @@ -1887,10 +1928,10 @@ uint64_t FileOffset) const { // Find a segment with a matching file offset. for (auto &KV : SegmentMapInfo) { - const SegmentInfo &SegInfo = KV.second; - if (alignDown(SegInfo.FileOffset, SegInfo.Alignment) == FileOffset) { + const ProgramHeader &SegInfo = KV.second; + if (alignDown(SegInfo.p_offset, SegInfo.p_align) == FileOffset) { // Use segment's aligned memory offset to calculate the base address. - const uint64_t MemOffset = alignDown(SegInfo.Address, SegInfo.Alignment); + const uint64_t MemOffset = alignDown(SegInfo.p_vaddr, SegInfo.p_align); return MMapAddress - MemOffset; } } Index: bolt/lib/Core/BinaryData.cpp =================================================================== --- bolt/lib/Core/BinaryData.cpp +++ bolt/lib/Core/BinaryData.cpp @@ -92,7 +92,9 @@ } bool BinaryData::isMoved() const { - return (getOffset() != OutputOffset || OutputSection != Section); + assert(OutputSection && Section); + return (getOffset() != OutputOffset || + OutputSection->getOutputAddress() != Section->getAddress()); } void BinaryData::print(raw_ostream &OS) const { printBrief(OS); } Index: bolt/lib/Core/BinaryEmitter.cpp =================================================================== --- bolt/lib/Core/BinaryEmitter.cpp +++ bolt/lib/Core/BinaryEmitter.cpp @@ -139,6 +139,7 @@ /// Emit a single function. bool emitFunction(BinaryFunction &BF, FunctionFragment &FF); + void emitPLTFunction(BinaryFunction &BF); /// Helper for emitFunctionBody to write data inside a function /// (used for AArch64) void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart, @@ -216,12 +217,25 @@ emitDataSections(OrgSecPrefix); } +void BinaryEmitter::emitPLTFunction(BinaryFunction &BF) { + assert(BF.begin() + 1 == BF.end()); + MCSection *Section = BC.getCodeSection(BF.getOriginSection()->getName()); + Streamer.switchSection(Section); + Section->setAlignment(Align(BF.getOriginSection()->getAlignment())); + Section->setHasInstructions(true); + Streamer.emitLabel(BF.getSymbol()); + for (auto &BB : BF) + for (auto Instr : BB) + Streamer.emitInstruction(Instr, *BC.STI); +} + void BinaryEmitter::emitFunctions() { auto emit = [&](const std::vector &Functions) { const bool HasProfile = BC.NumProfiledFuncs > 0; const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding(); for (BinaryFunction *Function : Functions) { - if (!BC.shouldEmit(*Function)) + if (!BC.shouldEmit(*Function) && + !(Function->isPLTFunction() && opts::Rewrite && BC.isRISC())) continue; LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function @@ -231,10 +245,18 @@ bool Emitted = false; // Turn off Intel JCC Erratum mitigation for cold code if requested - if (HasProfile && opts::X86AlignBranchBoundaryHotOnly && - !Function->hasValidProfile()) + if ((HasProfile && opts::X86AlignBranchBoundaryHotOnly && + !Function->hasValidProfile()) || + Function->isPLTFunction()) Streamer.setAllowAutoPadding(false); + if (Function->isPLTFunction()) { + emitPLTFunction(*Function); + Function->setEmitted(/*KeepCFG=*/false); + Streamer.setAllowAutoPadding(OriginalAllowAutoPadding); + continue; + } + FunctionLayout &Layout = Function->getLayout(); Emitted |= emitFunction(*Function, Layout.getMainFragment()); @@ -1043,16 +1065,16 @@ for (int Index = TypeTable.size() - 1; Index >= 0; --Index) { const uint64_t TypeAddress = TypeTable[Index]; + const MCSymbol *TypeSymbol = + BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment); switch (TTypeEncoding & 0x70) { default: llvm_unreachable("unsupported TTypeEncoding"); case dwarf::DW_EH_PE_absptr: - Streamer.emitIntValue(TypeAddress, TTypeEncodingSize); + Streamer.emitSymbolValue(TypeSymbol, TTypeEncodingSize); break; case dwarf::DW_EH_PE_pcrel: { if (TypeAddress) { - const MCSymbol *TypeSymbol = - BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment); MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol(); Streamer.emitLabel(DotSymbol); const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub( @@ -1163,9 +1185,17 @@ if (!Section.hasRelocations()) continue; - StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : ""; - Section.emitAsData(Streamer, Prefix + Section.getName()); - Section.clearRelocations(); + assert(Section.getOutputName() != BC.getMainCodeSectionName() && + Section.getOutputName() != + (OrgSecPrefix + BC.getMainCodeSectionName()).str() && + ".text should not have relocations!"); + + Section.emitAsData(Streamer, Section.getOutputName()); + // because dynamic relocations usually contain addresses without symbols, + // we neeed to preserve usual relocations to detect end-of section + // references + if (!Section.hasDynamicRelocations()) + Section.clearRelocations(); } } Index: bolt/lib/Core/BinaryFunction.cpp =================================================================== --- bolt/lib/Core/BinaryFunction.cpp +++ bolt/lib/Core/BinaryFunction.cpp @@ -16,6 +16,7 @@ #include "bolt/Core/DynoStats.h" #include "bolt/Core/HashUtilities.h" #include "bolt/Core/MCPlusBuilder.h" +#include "bolt/Utils/CommandLineOpts.h" #include "bolt/Utils/NameResolver.h" #include "bolt/Utils/NameShortener.h" #include "bolt/Utils/Utils.h" @@ -1156,6 +1157,13 @@ } } +bool BinaryFunction::handlePLT() { + assert(BasicBlocks.size() == 1); + return BC.MIB->patchPLTInstructions(BasicBlocks[0]->begin(), + BasicBlocks[0]->end(), getPLTSymbol(), + BC.Ctx.get(), getSymbol()); +} + bool BinaryFunction::disassemble() { NamedRegionTimer T("disassemble", "Disassemble function", "buildfuncs", "Build Binary Functions", opts::TimeBuild); @@ -1319,24 +1327,110 @@ // Indirect call. We only need to fix it if the operand is RIP-relative. if (IsSimple && MIB->hasPCRelOperand(Instruction)) handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size); - - if (BC.isAArch64()) + else if (opts::Rewrite && BC.isX86() && + BC.MIB->isCall64m(Instruction)) { + // Indirect call with possible absolute immediate. Here we assume + // that a relocation is present if we have such immediate and replace + // the immediate with symbol ref. + if (const Relocation *Rel = + getRelocationInRange(Offset, Offset + Size)) { + assert(Rel->Symbol && "Indirect call without referenced symbol!"); + int64_t Value = Rel->Value; + bool Ok = BC.MIB->replaceImmWithSymbolRef(Instruction, Rel->Symbol, + Rel->Addend, Ctx.get(), + Value, Rel->Type); + assert(Ok && "Failed to replace immediate with symbol ref!"); + } + } else if (BC.isAArch64()) { handleAArch64IndirectCall(Instruction, Offset); + } } } else if (BC.isAArch64() || BC.isRISCV()) { // Check if there's a relocation associated with this instruction. bool UsedReloc = false; - for (auto Itr = Relocations.lower_bound(Offset), - ItrE = Relocations.lower_bound(Offset + Size); - Itr != ItrE; ++Itr) { + if (auto Itr = Relocations.find(Offset); Itr != Relocations.end()) { const Relocation &Relocation = Itr->second; - int64_t Value = Relocation.Value; - const bool Result = BC.MIB->replaceImmWithSymbolRef( - Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), Value, - Relocation.Type); + bool Result = false; + + bool TLSAccessNotHandled = + Relocation.Type == ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC; + enum AccessType { Default, RelaxedGOTAccess, TLSGOT }; + + AccessType Type = [&]() { + if (!Instructions.size() || Itr == Relocations.begin()) + return Default; + if (std::prev(Itr)->second.Type == ELF::R_AARCH64_ADR_GOT_PAGE && + Relocation.Type == ELF::R_AARCH64_ADD_ABS_LO12_NC && + BC.MIB->matchAdrpPair(Instructions.rbegin()->second, Instruction)) + return RelaxedGOTAccess; + + // We usually can't determine .got address for such relocations, + // so we assert that instructions and relocations are consecutive + // and extract address from instructions. + if (Relocation.Type == ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { + assert(std::prev(Itr)->second.Type == + ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 && + "Couldn't find the corresponding ADRP relocation for " + "R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC"); + assert(BC.MIB->matchAdrpPair(Instructions.rbegin()->second, + Instruction) && + "Can't match TLS access instructions"); + return TLSGOT; + } + return Default; + }(); + if (Type != Default) { + + const uint64_t PageAddress = std::prev(Itr)->second.Value; + const uint64_t PageOffset = Relocation.Value; + + assert((PageAddress & 0xfff) == 0 && "Invalid ADRP operand"); + assert((PageOffset & 0xfff) == PageOffset && + "Invalid ADD/LDR operand"); + + const uint64_t TargetAddress = PageAddress + PageOffset; + + assert(TargetAddress && "ADRP + ADD/LDR references zero"); + + const MCSymbol *Symbol = [&]() -> const MCSymbol * { + if (Type == RelaxedGOTAccess) { + assert(Relocation.Symbol && "ADRP+ADD references unknown symbol"); + return Relocation.Symbol; + } + return BC.getOrCreateGlobalSymbol(TargetAddress, "DATAat"); + }(); + + Result = BC.MIB->setOperandToSymbolRef( + Instructions.rbegin()->second, /*OpNum*/ 1, Symbol, 0, + BC.Ctx.get(), ELF::R_AARCH64_ADR_PREL_PG_HI21); + + Result = + BC.MIB->setOperandToSymbolRef(Instruction, /*OpNum*/ 2, Symbol, 0, + BC.Ctx.get(), Relocation.Type) && + Result; + TLSAccessNotHandled = false; + } else { + + int64_t Value = Relocation.Value; + Result = BC.MIB->replaceImmWithSymbolRef( + Instruction, Relocation.Symbol, Relocation.Addend, Ctx.get(), + Value, Relocation.Type); + if (Relocation.Type == ELF::R_AARCH64_ADD_ABS_LO12_NC && + BC.MIB->isLoad(Instruction)) { + // if we see ldr with add-related relocation, it means we + // couldn't find the .got entry for referneced symbol during + // relocation processing. In that case we have to load symbol + // address directly, avoiding .got access altogether. It usually + // happens with linker-inserted symbols such as + // __{section}_{start,end} and shouldn't cause problems. + BC.MIB->relaxLdrToAdd(Instruction); + } + } (void)Result; assert(Result && "cannot replace immediate with relocation"); - + assert(!TLSAccessNotHandled && + "Unhandled R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC"); + (void)TLSAccessNotHandled; // For aarch64, if we replaced an immediate with a symbol from a // relocation, we mark it so we do not try to further process a // pc-relative operand. All we need is the symbol. @@ -1345,6 +1439,19 @@ if (!BC.isRISCV() && MIB->hasPCRelOperand(Instruction) && !UsedReloc) handlePCRelOperand(Instruction, AbsoluteInstrAddr, Size); + } else if (const Relocation *Rel = + getRelocationInRange(Offset, Offset + Size)) { + if (auto *Sym = Rel->Symbol) + if (auto It = BC.EndSymbols.find(Sym->getName()); + It != BC.EndSymbols.end()) { + int64_t Value; + LLVM_DEBUG( + dbgs() << formatv( + "BOLT-INFO: end symbol {0} referenced in {1} + {2:x}\n", + Sym->getName(), getOneName(), Offset);); + BC.MIB->replaceImmWithSymbolRef(Instruction, Sym, Rel->Addend, + Ctx.get(), Value, Rel->Type); + } } add_instruction: @@ -4023,6 +4130,11 @@ } void BinaryFunction::updateOutputValues(const MCAsmLayout &Layout) { + if (isPLTFunction()) { + const uint64_t Offset = getAddress() - OriginSection->getAddress(); + setOutputAddress(OriginSection->getOutputAddress() + Offset); + return; + } if (!isEmitted()) { assert(!isInjected() && "injected function should be emitted"); setOutputAddress(getAddress()); Index: bolt/lib/Core/BinarySection.cpp =================================================================== --- bolt/lib/Core/BinarySection.cpp +++ bolt/lib/Core/BinarySection.cpp @@ -32,6 +32,43 @@ bool BinarySection::isMachO() const { return BC.isMachO(); } +// If there is a Relocation at Offset which references end-of-section symbol, +// return the new value for it +uint64_t BinarySection::getNewEndSymbolValue(uint64_t RelocationOffset) const { + if (const Relocation *R = getRelocationAt(RelocationOffset)) + if (auto *Sym = R->Symbol) + if (auto BSecIt = BC.EndSymbols.find(Sym->getName()); + BSecIt != BC.EndSymbols.end()) { + BinarySection *BSec = BSecIt->second; + assert(BSec->getOutputAddress() && "Unmapped section!"); + uint64_t NewValue = BSec->getOutputEndAddress(); + return NewValue; + } + return 0; +} + +/// Add a new relocation at the given /p Offset. +void BinarySection::addRelocation(uint64_t Offset, MCSymbol *Symbol, + uint64_t Type, uint64_t Addend, + uint64_t Value, bool Pending) { + if (Name == BC.getMainCodeSectionName()) { + errs() << formatv( + "BOLT-ERROR: Adding reloc to .text at {0:x}, symbol {1}\n", Offset, + Symbol ? Symbol->getName() : ""); + // assert false for stacktrace + assert(false && + "Relocations for .text should be handled by BinaryFunction!"); + exit(1); + } + assert(Offset < getSize() && "offset not within section bounds"); + if (!Pending) { + Relocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); + } else { + PendingRelocations.emplace_back( + Relocation{Offset, Symbol, Type, Addend, Value}); + } +} + uint64_t BinarySection::hash(const BinaryData &BD, std::map &Cache) const { @@ -152,8 +189,7 @@ // this means using their input file offsets, since the output file offset // could change (e.g. for new instance of .text). For non-allocatable // sections, the output offset should always be a valid one. - const uint64_t SectionFileOffset = - isAllocatable() ? getInputFileOffset() : getOutputFileOffset(); + const uint64_t SectionFileOffset = getOutputFileOffset(); LLVM_DEBUG( dbgs() << "BOLT-DEBUG: flushing pending relocations for section " << getName() << '\n' Index: bolt/lib/Core/Relocation.cpp =================================================================== --- bolt/lib/Core/Relocation.cpp +++ bolt/lib/Core/Relocation.cpp @@ -105,6 +105,7 @@ case ELF::R_RISCV_RVC_BRANCH: case ELF::R_RISCV_ADD32: case ELF::R_RISCV_SUB32: + case ELF::R_RISCV_64: return true; } } @@ -202,6 +203,7 @@ case ELF::R_RISCV_SUB32: return 4; case ELF::R_RISCV_GOT_HI20: + case ELF::R_RISCV_64: // See extractValueRISCV for why this is necessary. return 8; } @@ -212,7 +214,8 @@ } static bool skipRelocationTypeAArch64(uint64_t Type) { - return Type == ELF::R_AARCH64_NONE || Type == ELF::R_AARCH64_LD_PREL_LO19; + return Type == ELF::R_AARCH64_NONE || Type == ELF::R_AARCH64_LD_PREL_LO19 || + Type == ELF::R_AARCH64_TLSDESC_CALL; } static bool skipRelocationTypeRISCV(uint64_t Type) { @@ -309,8 +312,10 @@ case ELF::R_AARCH64_ADD_ABS_LO12_NC: case ELF::R_AARCH64_ADR_GOT_PAGE: case ELF::R_AARCH64_LD64_GOT_LO12_NC: - if (IsAdr(Contents)) - return true; + if (IsAdr(Contents)) { + Type = ELF::R_AARCH64_ADR_PREL_LO21; + return false; + } } return false; @@ -373,7 +378,6 @@ return static_cast(PC) + SignExtend64<32>(Contents & 0xffffffff); case ELF::R_AARCH64_PREL64: return static_cast(PC) + Contents; - case ELF::R_AARCH64_TLSDESC_CALL: case ELF::R_AARCH64_JUMP26: case ELF::R_AARCH64_CALL26: // Immediate goes in bits 25:0 of B and BL. @@ -515,6 +519,7 @@ return SignExtend64<8>(((Contents >> 2) & 0x1f) | ((Contents >> 5) & 0xe0)); case ELF::R_RISCV_ADD32: case ELF::R_RISCV_SUB32: + case ELF::R_RISCV_64: return Contents; } } @@ -677,6 +682,7 @@ llvm_unreachable("Unknown relocation type"); case ELF::R_RISCV_ADD32: case ELF::R_RISCV_SUB32: + case ELF::R_RISCV_64: return false; case ELF::R_RISCV_JAL: case ELF::R_RISCV_CALL: @@ -816,6 +822,8 @@ uint64_t Relocation::getAbs64() { if (Arch == Triple::aarch64) return ELF::R_AARCH64_ABS64; + if (Arch == Triple::riscv64) + return ELF::R_RISCV_64; return ELF::R_X86_64_64; } Index: bolt/lib/Passes/CMakeLists.txt =================================================================== --- bolt/lib/Passes/CMakeLists.txt +++ bolt/lib/Passes/CMakeLists.txt @@ -13,7 +13,6 @@ DataflowInfoManager.cpp FrameAnalysis.cpp FrameOptimizer.cpp - FixRelaxationPass.cpp FixRISCVCallsPass.cpp HFSort.cpp HFSortPlus.cpp Index: bolt/lib/Passes/FixRelaxationPass.cpp =================================================================== --- bolt/lib/Passes/FixRelaxationPass.cpp +++ /dev/null @@ -1,64 +0,0 @@ -#include "bolt/Passes/FixRelaxationPass.h" -#include "bolt/Core/ParallelUtilities.h" - -using namespace llvm; - -namespace llvm { -namespace bolt { - -// This function finds ADRP+ADD instruction sequences that originally before -// linker relaxations were ADRP+LDR. We've modified LDR/ADD relocation properly -// during relocation reading, so its targeting right symbol. As for ADRP its -// target is wrong before this pass since we won't be able to recognize and -// properly change R_AARCH64_ADR_GOT_PAGE relocation to -// R_AARCH64_ADR_PREL_PG_HI21 during relocation reading. Now we're searching for -// ADRP+ADD sequences, checking that ADRP points to the GOT-table symbol and the -// target of ADD is another symbol. When found change ADRP symbol reference to -// the ADDs one. -void FixRelaxations::runOnFunction(BinaryFunction &BF) { - BinaryContext &BC = BF.getBinaryContext(); - for (BinaryBasicBlock &BB : BF) { - for (auto II = BB.begin(); II != BB.end(); ++II) { - MCInst &Adrp = *II; - if (BC.MIB->isPseudo(Adrp) || !BC.MIB->isADRP(Adrp)) - continue; - - const MCSymbol *AdrpSymbol = BC.MIB->getTargetSymbol(Adrp); - if (!AdrpSymbol || AdrpSymbol->getName() != "__BOLT_got_zero") - continue; - - auto NextII = std::next(II); - if (NextII == BB.end()) - continue; - - const MCInst &Add = *NextII; - if (!BC.MIB->matchAdrpAddPair(Adrp, Add)) - continue; - - const MCSymbol *Symbol = BC.MIB->getTargetSymbol(Add); - if (!Symbol || AdrpSymbol == Symbol) - continue; - - auto L = BC.scopeLock(); - const int64_t Addend = BC.MIB->getTargetAddend(Add); - BC.MIB->setOperandToSymbolRef(Adrp, /*OpNum*/ 1, Symbol, Addend, - BC.Ctx.get(), ELF::R_AARCH64_NONE); - } - } -} - -void FixRelaxations::runOnFunctions(BinaryContext &BC) { - if (!BC.isAArch64() || !BC.HasRelocations) - return; - - ParallelUtilities::WorkFuncTy WorkFun = [&](BinaryFunction &BF) { - runOnFunction(BF); - }; - - ParallelUtilities::runOnEachFunction( - BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun, nullptr, - "FixRelaxations"); -} - -} // namespace bolt -} // namespace llvm Index: bolt/lib/Passes/LongJmp.cpp =================================================================== --- bolt/lib/Passes/LongJmp.cpp +++ bolt/lib/Passes/LongJmp.cpp @@ -20,7 +20,7 @@ extern cl::OptionCategory BoltOptCategory; extern llvm::cl::opt AlignText; extern cl::opt AlignFunctions; -extern cl::opt UseOldText; +extern cl::opt Rewrite; extern cl::opt HotFunctionsAtEnd; static cl::opt GroupStubs("group-stubs", @@ -375,7 +375,7 @@ void LongJmpPass::tentativeLayout( const BinaryContext &BC, std::vector &SortedFunctions) { - uint64_t DotAddress = BC.LayoutStartAddress; + uint64_t DotAddress = BC.NewTextSectionAddress; if (!BC.HasRelocations) { for (BinaryFunction *Func : SortedFunctions) { @@ -391,24 +391,35 @@ } // Relocation mode - uint64_t EstimatedTextSize = 0; - if (opts::UseOldText) { - EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0); - - // Initial padding - if (EstimatedTextSize <= BC.OldTextSectionSize) { - DotAddress = BC.OldTextSectionAddress; - uint64_t Pad = - offsetToAlignment(DotAddress, llvm::Align(opts::AlignText)); - if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) { - DotAddress += Pad; - } + if (opts::Rewrite) { + + DotAddress = alignTo(BC.OldTextSectionAddress, opts::AlignText); + auto PLTSec = BC.getUniqueSectionByName(".plt"); + + if (PLTSec) { + // PLT Section is always put before .text when we rewrite. + // If we don't do that, we won't be able to estimate PLT position + // relative to .text, which means we won't be able to insert stubs + // for PLT calls properly. Also, having PLT just before .text ensures + // that we have minimal amount of stubs inserted in hot text, + // unless -hot-functions-at-end is used. + // If PLT was after text previously, move estimated text address forward + // by it's size with alignment. The real text address will likely be + // different after linking, but it doesn't matter for us because we only + // care about relative positions in the executable segment. + if (PLTSec->getAddress() > BC.OldTextSectionAddress) + DotAddress = alignTo(BC.OldTextSectionAddress + PLTSec->getSize(), + opts::AlignText); + + // alignTo(PLTSec->getSize(), opts::AlignText) is the most + // conservative estimation for distance between start of PLT and start of + // text. It cannot be more far away than that, so it should work for stub + // insertion. + TentativePLTAddress = + DotAddress - alignTo(PLTSec->getSize(), opts::AlignText); } } - - if (!EstimatedTextSize || EstimatedTextSize > BC.OldTextSectionSize) - DotAddress = alignTo(BC.LayoutStartAddress, opts::AlignText); - + assert(DotAddress && "No tentative text address!"); tentativeLayoutRelocMode(BC, SortedFunctions, DotAddress); } @@ -432,6 +443,13 @@ } uint64_t EntryID = 0; const BinaryFunction *TargetFunc = BC.getFunctionForSymbol(Target, &EntryID); + if (opts::Rewrite && TargetFunc->isPLTFunction()) { + uint64_t Offset = + TargetFunc->getAddress() - TargetFunc->getOriginSection()->getAddress(); + assert(TentativePLTAddress && "No PLT address!"); + return TentativePLTAddress + Offset; + } + auto Iter = HotAddresses.find(TargetFunc); if (Iter == HotAddresses.end() || (TargetFunc && EntryID)) { // Look at BinaryContext's resolution for this symbol - this is a symbol not @@ -621,6 +639,8 @@ tentativeLayout(BC, Sorted); updateStubGroups(); for (BinaryFunction *Func : Sorted) { + if (Func->isPLTFunction()) + continue; if (relax(*Func)) { // Don't ruin non-simple functions, they can't afford to have the layout // changed. Index: bolt/lib/Passes/ReorderData.cpp =================================================================== --- bolt/lib/Passes/ReorderData.cpp +++ bolt/lib/Passes/ReorderData.cpp @@ -504,9 +504,6 @@ // Rename sections. BinarySection &Hot = BC.registerSection(Section->getName() + ".hot", *Section); - Hot.setOutputName(Section->getName()); - Section->setOutputName(".bolt.org" + Section->getName()); - // Reorder contents of original section. setSectionOrder(BC, Hot, Order.begin(), SplitPoint); Index: bolt/lib/Passes/ReorderFunctions.cpp =================================================================== --- bolt/lib/Passes/ReorderFunctions.cpp +++ bolt/lib/Passes/ReorderFunctions.cpp @@ -124,7 +124,7 @@ // Assign valid index for functions with valid profile. for (auto &It : BFs) { BinaryFunction &BF = It.second; - if (!BF.hasValidIndex() && BF.hasValidProfile()) + if (!BF.hasValidIndex() && BF.hasValidProfile() && !BF.isPseudo()) BF.setIndex(Index++); } @@ -264,7 +264,7 @@ Cg = buildCallGraph( BC, [](const BinaryFunction &BF) { - if (!BF.hasProfile()) + if (!BF.hasProfile() || BF.isPseudo()) return true; if (BF.getState() != BinaryFunction::State::CFG) return true; @@ -308,7 +308,7 @@ }); uint32_t Index = 0; for (BinaryFunction *BF : SortedFunctions) - if (BF->hasProfile()) { + if (BF->hasProfile() || !BF->isPseudo()) { BF->setIndex(Index++); LLVM_DEBUG(if (opts::Verbosity > 1) { dbgs() << "BOLT-INFO: hot func " << BF->getPrintName() << " (" @@ -414,7 +414,7 @@ ++InvalidEntries; break; } - if (!BF->hasValidIndex()) + if (!BF->hasValidIndex() && !BF->isPseudo()) BF->setIndex(Index++); else if (opts::Verbosity > 0) errs() << "BOLT-WARNING: Duplicate reorder entry for " << Function Index: bolt/lib/Profile/DataAggregator.cpp =================================================================== --- bolt/lib/Profile/DataAggregator.cpp +++ bolt/lib/Profile/DataAggregator.cpp @@ -2004,13 +2004,13 @@ // Check that the binary mapping matches one of the segments. bool MatchFound = llvm::any_of( llvm::make_second_range(BC->SegmentMapInfo), - [&](SegmentInfo &SegInfo) { + [&](ProgramHeader &SegInfo) { // The mapping is page-aligned and hence the MMapAddress could be // different from the segment start address. We cannot know the page // size of the mapping, but we know it should not exceed the segment // alignment value. Hence we are performing an approximate check. - return SegInfo.Address >= MMapInfo.MMapAddress && - SegInfo.Address - MMapInfo.MMapAddress < SegInfo.Alignment; + return SegInfo.p_vaddr >= MMapInfo.MMapAddress && + SegInfo.p_vaddr - MMapInfo.MMapAddress < SegInfo.p_align; }); if (!MatchFound) { errs() << "PERF2BOLT-WARNING: ignoring mapping of " << NameToUse Index: bolt/lib/Rewrite/BinaryPassManager.cpp =================================================================== --- bolt/lib/Rewrite/BinaryPassManager.cpp +++ bolt/lib/Rewrite/BinaryPassManager.cpp @@ -13,7 +13,6 @@ #include "bolt/Passes/AsmDump.h" #include "bolt/Passes/CMOVConversion.h" #include "bolt/Passes/FixRISCVCallsPass.h" -#include "bolt/Passes/FixRelaxationPass.h" #include "bolt/Passes/FrameOptimizer.h" #include "bolt/Passes/Hugify.h" #include "bolt/Passes/IdenticalCodeFolding.h" @@ -181,11 +180,6 @@ PrintStoke("print-stoke", cl::desc("print functions after stoke analysis"), cl::cat(BoltOptCategory)); -static cl::opt - PrintFixRelaxations("print-fix-relaxations", - cl::desc("print functions after fix relaxations pass"), - cl::cat(BoltOptCategory)); - static cl::opt PrintFixRISCVCalls("print-fix-riscv-calls", cl::desc("print functions after fix RISCV calls pass"), @@ -328,8 +322,6 @@ opts::AsmDump.getNumOccurrences()); if (BC.isAArch64()) { - Manager.registerPass(std::make_unique(PrintFixRelaxations)); - Manager.registerPass( std::make_unique(PrintVeneerElimination)); } @@ -482,7 +474,7 @@ Manager.registerPass(std::make_unique()); // Patch original function entries - if (BC.HasRelocations) + if (BC.HasRelocations && !opts::Rewrite) Manager.registerPass(std::make_unique()); // This pass turns tail calls into jumps which makes them invisible to Index: bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp =================================================================== --- bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp +++ bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp @@ -138,38 +138,11 @@ } } - BinarySection *Section = nullptr; - if (!OrgSecPrefix.empty() && SectionName.startswith(OrgSecPrefix)) { - // Update the original section contents. - ErrorOr OrgSection = - BC.getUniqueSectionByName(SectionName.substr(OrgSecPrefix.length())); - assert(OrgSection && OrgSection->isAllocatable() && - "Original section must exist and be allocatable."); - - Section = &OrgSection.get(); - Section->updateContents(Contents, Size); - } else { - // If the input contains a section with the section name, rename it in the - // output file to avoid the section name conflict and emit the new section - // under a unique internal name. - ErrorOr OrgSection = - BC.getUniqueSectionByName(SectionName); - bool UsePrefix = false; - if (OrgSection && OrgSection->hasSectionRef()) { - OrgSection->setOutputName(OrgSecPrefix + SectionName); - UsePrefix = true; - } - - // Register the new section under a unique name to avoid name collision with - // sections in the input file. - BinarySection &NewSection = BC.registerOrUpdateSection( - UsePrefix ? NewSecPrefix + SectionName : SectionName, ELF::SHT_PROGBITS, - BinarySection::getFlags(IsReadOnly, IsCode, true), Contents, Size, - Alignment); - if (UsePrefix) - NewSection.setOutputName(SectionName); - Section = &NewSection; - } + BinarySection *Section = &BC.registerOrUpdateSection( + SectionName, ELF::SHT_PROGBITS, + BinarySection::getFlags(IsReadOnly, IsCode, true), Contents, Size, Alignment); + assert(Section->isAllocatable() && + "verify that allocatable is marked as allocatable"); LLVM_DEBUG({ dbgs() << "BOLT: allocating " Index: bolt/lib/Rewrite/JITLinkLinker.cpp =================================================================== --- bolt/lib/Rewrite/JITLinkLinker.cpp +++ bolt/lib/Rewrite/JITLinkLinker.cpp @@ -110,6 +110,19 @@ std::string SymName = Symbol.first.str(); LLVM_DEBUG(dbgs() << "BOLT: looking for " << SymName << "\n"); + if (auto BSecIt = Linker.BC.EndSymbols.find(SymName); + BSecIt != Linker.BC.EndSymbols.end()) { + BinarySection *BSec = BSecIt->second; + assert(BSec->getOutputAddress() && "Unmapped section!"); + uint64_t Address = BSec->getOutputEndAddress(); + AllResults[Symbol.first] = orc::ExecutorSymbolDef( + orc::ExecutorAddr(Address), JITSymbolFlags()); + LLVM_DEBUG( + dbgs() << formatv("Resolved {0} as the end of {1} at {2:x}\n", + SymName, BSec->getOutputName(), Address);); + continue; + } + if (auto Address = Linker.lookupSymbol(SymName)) { LLVM_DEBUG(dbgs() << "Resolved to address 0x" << Twine::utohexstr(*Address) << "\n"); Index: bolt/lib/Rewrite/MachORewriteInstance.cpp =================================================================== --- bolt/lib/Rewrite/MachORewriteInstance.cpp +++ bolt/lib/Rewrite/MachORewriteInstance.cpp @@ -480,8 +480,6 @@ static_cast(Streamer.get())->getAssembler()); auto EFMM = std::make_unique(*BC); - EFMM->setNewSecPrefix(getNewSecPrefix()); - EFMM->setOrgSecPrefix(getOrgSecPrefix()); Linker = std::make_unique(*BC, std::move(EFMM)); Linker->loadObject(ObjectMemBuffer->getMemBufferRef(), Index: bolt/lib/Rewrite/RewriteInstance.cpp =================================================================== --- bolt/lib/Rewrite/RewriteInstance.cpp +++ bolt/lib/Rewrite/RewriteInstance.cpp @@ -10,6 +10,7 @@ #include "bolt/Core/BinaryContext.h" #include "bolt/Core/BinaryEmitter.h" #include "bolt/Core/BinaryFunction.h" +#include "bolt/Core/BinarySection.h" #include "bolt/Core/DebugData.h" #include "bolt/Core/Exceptions.h" #include "bolt/Core/FunctionLayout.h" @@ -34,6 +35,8 @@ #include "bolt/Utils/Utils.h" #include "llvm/ADT/AddressRanges.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/ELF.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h" #include "llvm/MC/MCAsmBackend.h" @@ -53,11 +56,13 @@ #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MathExtras.h" #include "llvm/Support/Regex.h" #include "llvm/Support/Timer.h" #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/raw_ostream.h" #include +#include #include #include #include @@ -229,11 +234,16 @@ cl::cat(BoltCategory)); static cl::opt -UseGnuStack("use-gnu-stack", - cl::desc("use GNU_STACK program header for new segment (workaround for " - "issues with strip/objcopy)"), - cl::ZeroOrMore, - cl::cat(BoltCategory)); + UseGnuStack("use-gnu-stack", + cl::desc("deprecated and not supported anymore: use GNU_STACK " + "program header for new segment (workaround for " + "issues with strip/objcopy)"), + cl::ZeroOrMore, cl::cat(BoltCategory), + cl::callback([](const bool &Option) { + errs() << "BOLT-WARNING: use-gnu-stack is deprecated, please " + "use -rewrite to " + "avoid stripping issues\n"; + })); static cl::opt TimeRewrite("time-rewrite", @@ -485,35 +495,43 @@ BC->StartFunctionAddress = Obj.getHeader().e_entry; - NextAvailableAddress = 0; - uint64_t NextAvailableOffset = 0; Expected PHsOrErr = Obj.program_headers(); if (Error E = PHsOrErr.takeError()) return E; ELF64LE::PhdrRange PHs = PHsOrErr.get(); + unsigned Phnum = PHs.end() - PHs.begin(); + uint64_t FirstAllocOffset = ~0ULL; + uint64_t EndOfLoadablePartOffset = 0; for (const ELF64LE::Phdr &Phdr : PHs) { switch (Phdr.p_type) { case ELF::PT_LOAD: BC->FirstAllocAddress = std::min(BC->FirstAllocAddress, static_cast(Phdr.p_vaddr)); - NextAvailableAddress = std::max(NextAvailableAddress, - Phdr.p_vaddr + Phdr.p_memsz); - NextAvailableOffset = std::max(NextAvailableOffset, - Phdr.p_offset + Phdr.p_filesz); - - BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{Phdr.p_vaddr, - Phdr.p_memsz, - Phdr.p_offset, - Phdr.p_filesz, - Phdr.p_align}; + FirstAllocOffset = + std::min(FirstAllocOffset, static_cast(Phdr.p_offset)); + BC->InputAddressSpaceEnd = + std::max(BC->InputAddressSpaceEnd, + alignTo(Phdr.p_vaddr + Phdr.p_memsz, BC->PageAlign)); + EndOfLoadablePartOffset = + std::max(EndOfLoadablePartOffset, + alignTo(Phdr.p_offset + Phdr.p_memsz, BC->PageAlign)); + BC->SegmentMapInfo[Phdr.p_vaddr] = ProgramHeader(Phdr); + if (Phdr.p_flags == ELF::PF_R) + HasReadOnlySegment = true; break; case ELF::PT_INTERP: BC->HasInterpHeader = true; break; + case ELF::PT_PHDR: + HasProgramHeaderSegment = true; + break; } + BC->InputSegments.push_back(Phdr); } + // Address of a first byte in a first segment (usually beginning of a file) + BaseAddress = BC->FirstAllocAddress - FirstAllocOffset; for (const SectionRef &Section : InputFile->sections()) { Expected SectionNameOrErr = Section.getName(); if (Error E = SectionNameOrErr.takeError()) @@ -521,14 +539,6 @@ StringRef SectionName = SectionNameOrErr.get(); if (SectionName == ".text") { BC->OldTextSectionAddress = Section.getAddress(); - BC->OldTextSectionSize = Section.getSize(); - - Expected SectionContentsOrErr = Section.getContents(); - if (Error E = SectionContentsOrErr.takeError()) - return E; - StringRef SectionContents = SectionContentsOrErr.get(); - BC->OldTextSectionOffset = - SectionContents.data() - InputFile->getData().data(); } if (!opts::HeatmapMode && @@ -540,34 +550,15 @@ "BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize"); } - if (!NextAvailableAddress || !NextAvailableOffset) - return createStringError(errc::executable_format_error, - "no PT_LOAD pheader seen"); - outs() << "BOLT-INFO: first alloc address is 0x" << Twine::utohexstr(BC->FirstAllocAddress) << '\n'; - FirstNonAllocatableOffset = NextAvailableOffset; - - NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign); - NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign); - - // Hugify: Additional huge page from left side due to - // weird ASLR mapping addresses (4KB aligned) - if (opts::Hugify && !BC->HasFixedLoadAddress) - NextAvailableAddress += BC->PageAlign; - - if (!opts::UseGnuStack) { - // This is where the black magic happens. Creating PHDR table in a segment - // other than that containing ELF header is tricky. Some loaders and/or - // parts of loaders will apply e_phoff from ELF header assuming both are in - // the same segment, while others will do the proper calculation. - // We create the new PHDR table in such a way that both of the methods - // of loading and locating the table work. There's a slight file size - // overhead because of that. - // - // NB: bfd's strip command cannot do the above and will corrupt the - // binary during the process of stripping non-allocatable sections. + if (!opts::Rewrite && !opts::UseOldText) { + // when not rewriting, compute the new text section and segment addresses in + // advance. Needed for LongJMP. + FirstNonAllocatableOffset = EndOfLoadablePartOffset; + uint64_t NextAvailableOffset = EndOfLoadablePartOffset; + NextAvailableAddress = BC->InputAddressSpaceEnd; if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress) NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress; else @@ -577,29 +568,24 @@ NextAvailableAddress - BC->FirstAllocAddress && "PHDR table address calculation error"); - outs() << "BOLT-INFO: creating new program header table at address 0x" - << Twine::utohexstr(NextAvailableAddress) << ", offset 0x" - << Twine::utohexstr(NextAvailableOffset) << '\n'; + BC->OutputAddressToOffsetMap[NextAvailableAddress] = NextAvailableOffset; + + unsigned MaxPhnum = Phnum + !HasProgramHeaderSegment + 2; + + BC->MaxPHDRSize = MaxPhnum * sizeof(ELF64LE::Phdr); PHDRTableAddress = NextAvailableAddress; PHDRTableOffset = NextAvailableOffset; + NextAvailableAddress += BC->MaxPHDRSize; + NextAvailableOffset += BC->MaxPHDRSize; - // Reserve space for 3 extra pheaders. - unsigned Phnum = Obj.getHeader().e_phnum; - Phnum += 3; - - NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy); - NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy); + NextAvailableAddress = alignTo( + NextAvailableAddress, std::max(opts::AlignText, opts::AlignFunctions)); + BC->NewTextSectionAddress = NextAvailableAddress; + outs() << "BOLT-INFO: creating new program header table at address 0x" + << Twine::utohexstr(PHDRTableAddress) << ", offset 0x" + << Twine::utohexstr(PHDRTableOffset) << '\n'; } - - // Align at cache line. - NextAvailableAddress = alignTo(NextAvailableAddress, 64); - NextAvailableOffset = alignTo(NextAvailableOffset, 64); - - NewTextSegmentAddress = NextAvailableAddress; - NewTextSegmentOffset = NextAvailableOffset; - BC->LayoutStartAddress = NextAvailableAddress; - // Tools such as objcopy can strip section contents but leave header // entries. Check that at least .text is mapped in the file. if (!getFileOffsetForAddress(BC->OldTextSectionAddress)) @@ -670,7 +656,8 @@ size_t IDOffset = BuildIDSection->getContents().rfind(BuildID); assert(IDOffset != StringRef::npos && "failed to patch build-id"); - uint64_t FileOffset = getFileOffsetForAddress(BuildIDSection->getAddress()); + uint64_t FileOffset = + getOutputFileOffsetForAddress(BuildIDSection->getOutputAddress()); if (!FileOffset) { errs() << "BOLT-WARNING: Non-allocatable build-id will not be updated.\n"; return; @@ -698,7 +685,6 @@ return E; adjustCommandLineOptions(); discoverFileObjects(); - preprocessProfileData(); // Skip disassembling if we have a translation table and we are running an @@ -727,7 +713,7 @@ if (opts::DiffOnly) return Error::success(); - preregisterSections(); + renameAndPreregisterSections(); runOptimizationPasses(); @@ -1002,7 +988,7 @@ section_iterator Section = cantFail(Symbol.getSection(), "cannot get symbol section"); - if (Section == InputFile->section_end()) { + if (Section == InputFile->section_end() || !Section->getName()) { // Could be an absolute symbol. Used on RISC-V for __global_pointer$ so we // need to record it to handle relocations against it. For other instances // of absolute symbols, we record for pretty printing. @@ -1016,13 +1002,24 @@ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName << " for function\n"); - if (Address == Section->getAddress() + Section->getSize()) { + if (SymbolType != SymbolRef::ST_Debug && + Address == Section->getAddress() + Section->getSize()) { assert(SymbolSize == 0 && - "unexpect non-zero sized symbol at end of section"); - LLVM_DEBUG( - dbgs() - << "BOLT-DEBUG: rejecting as symbol points to end of its section\n"); - registerName(SymbolSize); + "unexpected non-zero sized symbol at end of section"); + + if (auto BSec = + BC->getUniqueSectionByName(cantFail(Section->getName()))) { + // we don't register name because it can collide with data from the next + // section on the same address. We'll instead create local MCSymbols + // when we encounter relocations against such symbols + BC->EndSymbols[Name] = &*BSec; + LLVM_DEBUG(dbgs() << formatv("BOLT-INFO: {0} is in the end of {1}\n", + Name, BSec->getName())); + } else { + LLVM_DEBUG( + dbgs() + << "BOLT-INFO: rejecting as symbol points to end of its section\n"); + } continue; } @@ -1277,6 +1274,7 @@ if (!opts::LinuxKernelMode) { // Read all relocations now that we have binary functions mapped. + createGOTPLTRelocations(); processRelocations(); } @@ -1332,39 +1330,62 @@ } } -void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress, - uint64_t EntryAddress, - uint64_t EntrySize) { +BinaryFunction *RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress, + uint64_t EntryAddress, + uint64_t EntrySize) { if (!TargetAddress) - return; + return nullptr; auto setPLTSymbol = [&](BinaryFunction *BF, StringRef Name) { const unsigned PtrSize = BC->AsmInfo->getCodePointerSize(); MCSymbol *TargetSymbol = BC->registerNameAtAddress( Name.str() + "@GOT", TargetAddress, PtrSize, PtrSize); BF->setPLTSymbol(TargetSymbol); + BF->setPseudo(true); + BF->setIgnored(); }; BinaryFunction *BF = BC->getBinaryFunctionAtAddress(EntryAddress); if (BF && BC->isAArch64()) { // Handle IFUNC trampoline setPLTSymbol(BF, BF->getOneName()); - return; + return BF; } - const Relocation *Rel = BC->getDynamicRelocationAt(TargetAddress); - if (!Rel || !Rel->Symbol) - return; + MCSymbol *Symbol = [&]() -> MCSymbol * { + const Relocation *Rel = BC->getDynamicRelocationAt(TargetAddress); + if (Rel && Rel->Symbol) + return Rel->Symbol; + if (auto Addr = + BC->getUnsignedValueAtAddress(TargetAddress, sizeof(uint64_t))) { + if (auto *Data = BC->getBinaryDataAtAddress(*Addr)) + return Data->getSymbol(); + } + return nullptr; + }(); ErrorOr Section = BC->getSectionForAddress(EntryAddress); assert(Section && "cannot get section for address"); + if (!Symbol) { + assert(EntryAddress == Section->getAddress() && + "Unknown PLT entry after PLT header"); + if (!BF) + BF = BC->createBinaryFunction("__BOLT_PSEUDO_" + Section->getName().str(), + *Section, EntryAddress, 0, EntrySize, + Section->getAlignment()); + BF->setPseudo(true); + BF->setIgnored(); + BF->setPLTSymbol(BC->getOrCreateGlobalSymbol(TargetAddress, "DATAat")); + return BF; + } if (!BF) - BF = BC->createBinaryFunction(Rel->Symbol->getName().str() + "@PLT", - *Section, EntryAddress, 0, EntrySize, + BF = BC->createBinaryFunction(Symbol->getName().str() + "@PLT", *Section, + EntryAddress, 0, EntrySize, Section->getAlignment()); else - BF->addAlternativeName(Rel->Symbol->getName().str() + "@PLT"); - setPLTSymbol(BF, Rel->Symbol->getName()); + BF->addAlternativeName(Symbol->getName().str() + "@PLT"); + setPLTSymbol(BF, Symbol->getName()); + return BF; } void RewriteInstance::disassemblePLTSectionAArch64(BinarySection &Section) { @@ -1399,22 +1420,31 @@ while (InstrOffset < SectionSize) { disassembleInstruction(InstrOffset, Instruction, InstrSize); EntrySize += InstrSize; - if (!BC->MIB->isIndirectBranch(Instruction)) { - Instructions.emplace_back(Instruction); - InstrOffset += InstrSize; - continue; + Instructions.emplace_back(Instruction); + InstrOffset += InstrSize; + if (BC->MIB->isIndirectBranch(Instruction)) { + break; } - - const uint64_t EntryAddress = SectionAddress + EntryOffset; - const uint64_t TargetAddress = BC->MIB->analyzePLTEntry( - Instruction, Instructions.begin(), Instructions.end(), EntryAddress); - - createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize); - break; } - - // Branch instruction - InstrOffset += InstrSize; + const uint64_t EntryAddress = SectionAddress + EntryOffset; + const uint64_t TargetAddress = BC->MIB->analyzePLTEntry( + Instructions.begin(), Instructions.end(), EntryAddress); + BinaryFunction *BF = + createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize); + assert(BF && "Failed to create PLT function"); + if (opts::Rewrite) { + assert(BF && BF->empty()); + BF->setSize(EntrySize); + BF->updateState(BinaryFunction::State::Disassembled); + BinaryBasicBlock *BB = BF->addBasicBlockAt(0, BF->getSymbol()); + for (auto &Inst : Instructions) { + BB->addInstruction(Inst); + } + bool Ok = BF->handlePLT(); + assert(Ok && "Failed to handle PLT entry"); + (void)Ok; + BF->updateState(BinaryFunction::State::CFG); + } // Skip nops if any while (InstrOffset < SectionSize) { @@ -1423,10 +1453,56 @@ break; InstrOffset += InstrSize; + if (opts::Rewrite) { + BF->getBasicBlockAtOffset(0)->addInstruction(Instruction); + BF->setSize(BF->getSize() + InstrSize); + } } } } +void RewriteInstance::createGOTPLTRelocations() { + const uint64_t RelType = Relocation::getAbs64(); + auto CreateRelocations = [this, RelType](BinarySection &Section, bool IsGot) { + DataExtractor DE(Section.getContents(), BC->AsmInfo->isLittleEndian(), + BC->AsmInfo->getCodePointerSize()); + const uint64_t Size = Section.getSize(); + assert(Size % BC->AsmInfo->getCodePointerSize() == 0 && "Invalid GOT size"); + for (uint64_t DataOffset = 0; DataOffset < Size;) { + const uint64_t Offset = DataOffset; + const uint64_t Address = DE.getU64(&DataOffset); + if (Address == 0 || Address == ~0ULL) + continue; + MCSymbol *Sym = BC->getOrCreateGlobalSymbol(Address, "SYMBOLat"); + Section.addRelocation(Offset, Sym, RelType, 0); + if (!opts::Rewrite || !IsGot || !BC->isRISC()) + continue; + if (BinaryData *BD = BC->getBinaryDataAtAddress(Address)) { + for (auto *Sym : BD->getSymbols()) { + StringRef Name = Sym->getName(); + Name = Name.substr(0, Name.find('/')); + if (Name.empty()) + continue; + GOTSymbolsByName[Name.str()] = Section.getAddress() + Offset; + + LLVM_DEBUG( + outs() << formatv( + "BOLT-INFO: BD {0} with address {1:x} at got entry {2:x}\n", + Sym->getName(), Address, Section.getAddress() + Offset);); + } + } + } + }; + + auto GOTPLTSection = BC->getUniqueSectionByName(".got.plt"); + if (opts::Rewrite && GOTPLTSection) + CreateRelocations(*GOTPLTSection, false); + auto GOTSection = BC->getUniqueSectionByName(".got"); + if (GOTSection) + CreateRelocations(*GOTSection, true); +} + + void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) { const uint64_t SectionAddress = Section.getAddress(); const uint64_t SectionSize = Section.getSize(); @@ -1447,14 +1523,13 @@ } }; - // Skip the first special entry since no relocation points to it. - uint64_t InstrOffset = 32; + uint64_t InstrOffset = 0; while (InstrOffset < SectionSize) { InstructionListType Instructions; MCInst Instruction; const uint64_t EntryOffset = InstrOffset; - const uint64_t EntrySize = 16; + uint64_t EntrySize = EntryOffset == 0 ? 32 : 16; uint64_t InstrSize; while (InstrOffset < EntryOffset + EntrySize) { @@ -1465,9 +1540,36 @@ const uint64_t EntryAddress = SectionAddress + EntryOffset; const uint64_t TargetAddress = BC->MIB->analyzePLTEntry( - Instruction, Instructions.begin(), Instructions.end(), EntryAddress); + Instructions.begin(), Instructions.end(), EntryAddress); + + BinaryFunction *BF = + createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize); + assert(BF && "Failed to create PLT function"); + if (opts::Rewrite) { + assert(BF && BF->empty()); + BF->setSize(EntrySize); + BF->updateState(BinaryFunction::State::Disassembled); + BinaryBasicBlock *BB = BF->addBasicBlockAt(0, BF->getSymbol()); + for (auto &Inst : Instructions) { + BB->addInstruction(Inst); + } + bool Ok = BF->handlePLT(); + assert(Ok && "Failed to handle PLT entry"); + (void)Ok; + BF->updateState(BinaryFunction::State::CFG); + } + // Skip nops if any + while (InstrOffset < SectionSize) { + disassembleInstruction(InstrOffset, Instruction, InstrSize); + if (!BC->MIB->isNoop(Instruction)) + break; - createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize); + InstrOffset += InstrSize; + if (opts::Rewrite) { + BF->getBasicBlockAtOffset(0)->addInstruction(Instruction); + BF->setSize(BF->getSize() + InstrSize); + } + } } } @@ -1497,32 +1599,44 @@ MCInst Instruction; uint64_t InstrSize, InstrOffset = EntryOffset; while (InstrOffset < EntryOffset + EntrySize) { + uint64_t TargetAddress = 0; disassembleInstruction(InstrOffset, Instruction, InstrSize); // Check if the entry size needs adjustment. if (EntryOffset == 0 && BC->MIB->isTerminateBranch(Instruction) && EntrySize == 8) EntrySize = 16; - if (BC->MIB->isIndirectBranch(Instruction)) - break; - + if (BC->MIB->hasPCRelOperand(Instruction)) { + bool Ok = BC->MIB->evaluateMemOperandTarget( + Instruction, TargetAddress, SectionAddress + InstrOffset, + InstrSize); + assert(Ok && "Failed to evaluate PLT operand!"); + (void)Ok; + if (opts::Rewrite) { + MCSymbol *Sym = BC->getOrCreateGlobalSymbol(TargetAddress, "DATAat"); + // This is a hack to work around the fact that we don't actually + // disassemble plt. We bluntly assume immediate size of 4 and create + // addend because the address is relative to the next instruction + // This should be properly done by disassembling plt, replacing the + // operand with symbol ref and emitting it back, but it's harder to + // do on X86 because entries may terminate differently depending on + // PLT type, so we just patch immediates - at least for now. + const uint64_t SizeOfImm = 4; + assert(InstrSize > SizeOfImm); + const uint64_t OffsetOfImm = InstrOffset + InstrSize - SizeOfImm; + const uint64_t Addend = -SizeOfImm; + Section.addRelocation(OffsetOfImm, Sym, ELF::R_X86_64_PC32, Addend); + } + } + if (BC->MIB->isIndirectBranch(Instruction)) { + assert(TargetAddress); + const uint64_t EntryAddress = SectionAddress + EntryOffset; + BinaryFunction *BF = + createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize); + assert(BF && "Failed to create PLT function"); + } InstrOffset += InstrSize; } - - if (InstrOffset + InstrSize > EntryOffset + EntrySize) - continue; - - uint64_t TargetAddress; - if (!BC->MIB->evaluateMemOperandTarget(Instruction, TargetAddress, - SectionAddress + InstrOffset, - InstrSize)) { - errs() << "BOLT-ERROR: error evaluating PLT instruction at offset 0x" - << Twine::utohexstr(SectionAddress + InstrOffset) << '\n'; - exit(1); - } - - createPLTBinaryFunction(TargetAddress, SectionAddress + EntryOffset, - EntrySize); } } @@ -1544,15 +1658,9 @@ BinaryFunction *PltBF; auto BFIter = BC->getBinaryFunctions().find(Section.getAddress()); - if (BFIter != BC->getBinaryFunctions().end()) { - PltBF = &BFIter->second; - } else { - // If we did not register any function at the start of the section, - // then it must be a general PLT entry. Add a function at the location. - PltBF = BC->createBinaryFunction( - "__BOLT_PSEUDO_" + Section.getName().str(), Section, - Section.getAddress(), 0, PLTSI->EntrySize, Section.getAlignment()); - } + assert(BFIter != BC->getBinaryFunctions().end() && + "Failed to create PLT header function"); + PltBF = &BFIter->second; PltBF->setPseudo(true); } } @@ -1858,37 +1966,43 @@ opts::HotTextMoveSections.addValue(".never_hugify"); } - if (opts::UseOldText && !BC->OldTextSectionAddress) { - errs() << "BOLT-WARNING: cannot use old .text as the section was not found" - "\n"; - opts::UseOldText = false; + if (opts::Hugify) { + opts::AlignText = BinaryContext::HugePageSize; + BC->PageAlign = BinaryContext::HugePageSize; } - if (opts::UseOldText && !BC->HasRelocations) { - errs() << "BOLT-WARNING: cannot use old .text in non-relocation mode\n"; + + if (opts::AlignText < opts::AlignFunctions) + opts::AlignText = (unsigned)opts::AlignFunctions; + + if (opts::UseOldText) { + errs() << "BOLT-WARNING: '-use-old-text' is deprecated, " + "using -rewrite instead.\n"; opts::UseOldText = false; + opts::Rewrite = true; } - if (!opts::AlignText.getNumOccurrences()) - opts::AlignText = BC->PageAlign; + if (opts::Rewrite && !BC->HasRelocations) { + errs() << "BOLT-ERROR: cannot rewrite in non-relocation mode\n"; + exit(1); + } if (opts::AlignText < opts::AlignFunctions) opts::AlignText = (unsigned)opts::AlignFunctions; if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode && - !opts::UseOldText) + !opts::Rewrite) opts::Lite = true; - if (opts::Lite && opts::UseOldText) { - errs() << "BOLT-WARNING: cannot combine -lite with -use-old-text. " - "Disabling -use-old-text.\n"; - opts::UseOldText = false; - } - if (opts::Lite && opts::StrictMode) { errs() << "BOLT-ERROR: -strict and -lite cannot be used at the same time\n"; exit(1); } + if (opts::Lite && opts::Rewrite) { + errs() << "BOLT-ERROR: cannot combine -lite with -rewrite.\n"; + exit(1); + } + if (opts::Lite) outs() << "BOLT-INFO: enabling lite mode\n"; @@ -1973,15 +2087,16 @@ ErrorOr Value = BC->getUnsignedValueAtAddress(Rel.getOffset(), RelSize); assert(Value && "failed to extract relocated value"); - if ((Skip = Relocation::skipRelocationProcess(RType, *Value))) + Skip = Relocation::skipRelocationProcess(RType, *Value); + if (Skip) return true; - ExtractedValue = Relocation::extractValue(RType, *Value, Rel.getOffset()); Addend = getRelocationAddend(InputFile, Rel); const bool IsPCRelative = Relocation::isPCRelative(RType); const uint64_t PCRelOffset = IsPCRelative && !IsAArch64 ? Rel.getOffset() : 0; bool SkipVerification = false; + auto SymbolIter = Rel.getSymbol(); if (SymbolIter == InputFile->symbol_end()) { SymbolAddress = ExtractedValue - Addend + PCRelOffset; @@ -1992,12 +2107,50 @@ } else { const SymbolRef &Symbol = *SymbolIter; SymbolName = std::string(cantFail(Symbol.getName())); + SymbolAddress = cantFail(Symbol.getAddress()); SkipVerification = (cantFail(Symbol.getType()) == SymbolRef::ST_Other); // Section symbols are marked as ST_Debug. IsSectionRelocation = (cantFail(Symbol.getType()) == SymbolRef::ST_Debug); // Check for PLT entry registered with symbol name - if (!SymbolAddress && (IsAArch64 || BC->isRISCV())) { + if (opts::Rewrite && Relocation::isGOT(RType) && BC->isRISC()) { + std::string StrippedName = SymbolName.substr(0, SymbolName.find("@")); + if (auto It = GOTSymbolsByName.find(StrippedName); + It != GOTSymbolsByName.end()) { + SymbolAddress = It->second; + } else { + // we can't determine which .got entry is referenced by that + // instruction, so we have to relax .got access to direct access. For + // adrp it is enough to just change relocation type to load page address + // of the symbol itself, while LDR will be handled during disassembly - + // we'll change LDR to ADD when we see ldr with + // R_AARCH64_ADD_ABS_LO12_NC relocation. + switch (RType) { + case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: + case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: + // for TLS relocs, we'll match instructions during disassembly to + // determine .got entry. + break; + case ELF::R_AARCH64_ADR_GOT_PAGE: + outs() << formatv("BOLT-WARNING: cannot find .got entry for symbol " + "{0} referenced by R_AARCH64_ADR_GOT_PAGE at {1:x}. Will relax to adrp+add\n", + StrippedName, Rel.getOffset()); + RType = ELF::R_AARCH64_ADR_PREL_PG_HI21; + break; + case ELF::R_AARCH64_LD64_GOT_LO12_NC: + outs() << formatv("BOLT-WARNING: cannot find .got entry for symbol " + "{0} referenced by R_AARCH64_LD64_GOT_LO12_NC at {1:x}. Will relax to adrp+add\n", + StrippedName, Rel.getOffset()); + RType = ELF::R_AARCH64_ADD_ABS_LO12_NC; + break; + default: + errs() << formatv("BOLT-ERROR: Unexpected .got access relocation at {0:x}: can't find .got entry for {1}!\n", + Rel.getOffset(), StrippedName); + exit(1); + } + } + } + if (!SymbolAddress && IsAArch64) { const BinaryData *BD = BC->getPLTBinaryDataByName(SymbolName); SymbolAddress = BD ? BD->getAddress() : 0; } @@ -2033,7 +2186,7 @@ // For GOT relocs, do not subtract addend as the addend does not refer // to this instruction's target, but it refers to the target in the GOT // entry. - if (Relocation::isGOT(RType)) { + if (Relocation::isGOT(RType) && (BC->isX86() || !opts::Rewrite)) { Addend = 0; SymbolAddress = ExtractedValue + PCRelOffset; } else if (Relocation::isTLS(RType)) { @@ -2060,7 +2213,7 @@ if (SkipVerification) return true; - if (IsAArch64 || BC->isRISCV()) + if (BC->isRISC()) return true; if (SymbolName == "__hot_start" || SymbolName == "__hot_end") @@ -2186,6 +2339,32 @@ if (Symbol) SymbolIndex[Symbol] = getRelocationSymbol(InputFile, Rel); + auto IsGotLikeReloc = [this](const uint64_t RType) { + return ((BC->isAArch64() && (RType == ELF::R_AARCH64_GLOB_DAT || + RType == ELF::R_AARCH64_RELATIVE || + RType == ELF::R_AARCH64_TLS_TPREL64)) || + (BC->isRISCV() && RType == ELF::R_RISCV_64)); + }; + + if (opts::Rewrite && IsGotLikeReloc(RType)) { + if (Symbol && !SymbolName.empty()) { + GOTSymbolsByName[SymbolName.str()] = Rel.getOffset(); + LLVM_DEBUG(dbgs() << formatv("BOLT-INFO: GOT entry at {0:x} contains " + "symbol {1} with address {2:x}\n", + Rel.getOffset(), SymbolName, Addend);); + } else if (BinaryData *BD = BC->getBinaryDataAtAddress(Addend)) { + LLVM_DEBUG(outs() << formatv("BOLT-INFO: GOT entry at {0:x} contains " + "symbol {1} with address {2:x}\n", + Rel.getOffset(), BD->getName(), Addend);); + + for (MCSymbol *Sym : BD->getSymbols()) { + StringRef Name = Sym->getName().substr(0, Sym->getName().find('/')); + if (Name.empty()) + continue; + GOTSymbolsByName[Name.str()] = Rel.getOffset(); + } + } + } BC->addDynamicRelocation(Rel.getOffset(), Symbol, RType, Addend); } } @@ -2285,8 +2464,11 @@ StringRef RelocatedSectionName = cantFail(RelocatedSection.getName()); LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is " << RelocatedSectionName << '\n'); - - if (!BinarySection(*BC, RelocatedSection).isAllocatable()) { + auto RelocatedBinarySectionOrErr = + BC->getUniqueSectionByName(RelocatedSectionName); + assert(RelocatedBinarySectionOrErr); + BinarySection &RelocatedBinarySection = *RelocatedBinarySectionOrErr; + if (!RelocatedBinarySection.isAllocatable()) { LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocations against " << "non-allocatable section\n"); return; @@ -2302,10 +2484,10 @@ } for (const RelocationRef &Rel : Section.relocations()) - handleRelocation(RelocatedSection, Rel); + handleRelocation(RelocatedBinarySection, Rel); } -void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection, +void RewriteInstance::handleRelocation(BinarySection &RelocatedSection, const RelocationRef &Rel) { const bool IsAArch64 = BC->isAArch64(); const bool IsFromCode = RelocatedSection.isText(); @@ -2322,7 +2504,6 @@ dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n"; RType &= ~ELF::R_X86_64_converted_reloc_bit; } - if (Relocation::isTLS(RType)) { // No special handling required for TLS relocations on X86. if (BC->isX86()) @@ -2333,13 +2514,6 @@ return; } - if (!IsAArch64 && BC->getDynamicRelocationAt(Rel.getOffset())) { - LLVM_DEBUG({ - dbgs() << formatv("BOLT-DEBUG: address {0:x} has a ", Rel.getOffset()) - << "dynamic relocation against it. Ignoring static relocation.\n"; - }); - return; - } std::string SymbolName; uint64_t SymbolAddress; @@ -2390,7 +2564,14 @@ } MCSymbol *ReferencedSymbol = nullptr; - if (!IsSectionRelocation) + bool IsToSectionEnd = false; + if (auto It = BC->EndSymbols.find(SymbolName); It != BC->EndSymbols.end()) { + // We need to create local symbol because using registerName will collide + // with the next section if it's on the same address. This requires us to + // check for EndSymbols in a lot of places, but what else can we do? + ReferencedSymbol = BC->Ctx->getOrCreateSymbol(SymbolName); + IsToSectionEnd = true; + } else if (!IsSectionRelocation) if (BinaryData *BD = BC->getBinaryDataByName(SymbolName)) ReferencedSymbol = BD->getSymbol(); @@ -2443,10 +2624,25 @@ // "symbol + addend" references an object different from "symbol". ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, Addend, ExtractedValue); + } else if (opts::Rewrite && !IsToCode && !IsFromCode) { + if (!ReferencedSymbol) { + ReferencedSymbol = + BC->getOrCreateGlobalSymbol(SymbolAddress, "SYMBOLat"); + } + + RelocatedSection.addRelocation( + Rel.getOffset() - RelocatedSection.getAddress(), ReferencedSymbol, + RType, Addend, ExtractedValue); + LLVM_DEBUG({ + dbgs() << formatv("BOLT-DEBUG: creating PC-relative data relocation at " + "{0:x} for {1}\n", + Rel.getOffset(), SymbolName); + }); } else { LLVM_DEBUG({ - dbgs() << "BOLT-DEBUG: not creating PC-relative relocation at" - << formatv("{0:x} for {1}\n", Rel.getOffset(), SymbolName); + dbgs() << formatv("BOLT-DEBUG: ignoring PC-relative relocation at " + "{0:x} for {1}\n", + Rel.getOffset(), SymbolName); }); } @@ -2517,15 +2713,24 @@ } } - if (ForceRelocation) { - std::string Name = - Relocation::isGOT(RType) ? "__BOLT_got_zero" : SymbolName; - ReferencedSymbol = BC->registerNameAtAddress(Name, 0, 0, 0); - SymbolAddress = 0; - if (Relocation::isGOT(RType)) - Addend = Address; - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: forcing relocation against symbol " - << SymbolName << " with addend " << Addend << '\n'); + if (IsToSectionEnd) { + // do nothing, add it as it is later + } else if (ForceRelocation) { + std::string Name = SymbolName; + if (Relocation::isGOT(RType)) { + if (opts::Rewrite) { + Name = SymbolName + "@GOT"; + } else { + Addend = Address; + SymbolAddress = 0; + Name = "__BOLT_got_zero"; + } + } + ReferencedSymbol = BC->registerNameAtAddress(Name, SymbolAddress, 0, 0); + LLVM_DEBUG( + dbgs() << formatv("BOLT-DEBUG: forcing relocation against symbol {0} " + "with addend {1:x} and address {2:x}\n", + SymbolName, Addend, SymbolAddress)); } else if (ReferencedBF) { ReferencedSymbol = ReferencedBF->getSymbol(); uint64_t RefFunctionOffset = 0; @@ -2549,8 +2754,7 @@ if (!ContainingBF) ReferencedBF->registerReferencedOffset(RefFunctionOffset); } - if (opts::Verbosity > 1 && - BinarySection(*BC, RelocatedSection).isWritable()) + if (opts::Verbosity > 1 && RelocatedSection.isWritable()) errs() << "BOLT-WARNING: writable reference into the middle of the " << formatv("function {0} detected at address {1:x}\n", *ReferencedBF, Rel.getOffset()); @@ -2656,9 +2860,9 @@ NumDataRelocations < opts::MaxDataRelocations); }; - if ((ReferencedSection && refersToReorderedSection(ReferencedSection)) || + if (opts::Rewrite || + (ReferencedSection && refersToReorderedSection(ReferencedSection)) || (opts::ForceToDataRelocations && checkMaxDataRelocations()) || - // RISC-V has ADD/SUB data-to-data relocations BC->isRISCV()) ForceRelocation = true; @@ -3152,24 +3356,30 @@ BinaryFunctionPassManager::runAllPasses(*BC); } -void RewriteInstance::preregisterSections() { - // Preregister sections before emission to set their order in the output. - const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true, - /*IsText*/ false, - /*IsAllocatable*/ true); - if (BinarySection *EHFrameSection = getSection(getEHFrameSectionName())) { - // New .eh_frame. - BC->registerOrUpdateSection(getNewSecPrefix() + getEHFrameSectionName(), - ELF::SHT_PROGBITS, ROFlags); - // Fully register a relocatable copy of the original .eh_frame. - BC->registerSection(".relocated.eh_frame", *EHFrameSection); - } - BC->registerOrUpdateSection(getNewSecPrefix() + ".gcc_except_table", - ELF::SHT_PROGBITS, ROFlags); - BC->registerOrUpdateSection(getNewSecPrefix() + ".rodata", ELF::SHT_PROGBITS, - ROFlags); - BC->registerOrUpdateSection(getNewSecPrefix() + ".rodata.cold", - ELF::SHT_PROGBITS, ROFlags); +void RewriteInstance::renameAndPreregisterSections() { + + auto Rename = [this](BinarySection *Section) { + if (!Section) + return; + std::string OldName = Section->getName().str(); + const std::string NewName = (getOrgSecPrefix() + Section->getName()).str(); + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: renaming input section " + << Section->getName() << " to " << NewName << "\n";); + BC->renameSection(*Section, NewName); + BC->registerOrUpdateSection(OldName, ELF::SHT_PROGBITS, + Section->getELFFlags()); + }; + if (!opts::Rewrite) { + if (BC->HasRelocations) + Rename(getSection(BC->getMainCodeSectionName())); + Rename(getSection(getEHFrameSectionName())); + if (EHFrameSection) + BC->registerSection(".relocated.eh_frame", *EHFrameSection); + Rename(getSection(".eh_frame_hdr")); + Rename(getSection(".gcc_except_table")); + } + if (opts::JumpTables > JTS_BASIC) + Rename(getSection(".rodata")); } void RewriteInstance::emitAndLink() { @@ -3185,7 +3395,7 @@ std::unique_ptr Streamer = BC->createStreamer(OS); if (EHFrameSection) { - if (opts::UseOldText || opts::StrictMode) { + if (opts::Rewrite || opts::StrictMode) { // The section is going to be regenerated from scratch. // Empty the contents, but keep the section reference. EHFrameSection->clearContents(); @@ -3215,11 +3425,6 @@ << OutObjectPath << "\n"; } - ErrorOr TextSection = - BC->getUniqueSectionByName(BC->getMainCodeSectionName()); - if (BC->HasRelocations && TextSection) - BC->renameSection(*TextSection, getOrgSecPrefix() + ".text"); - ////////////////////////////////////////////////////////////////////////////// // Assign addresses to new sections. ////////////////////////////////////////////////////////////////////////////// @@ -3229,12 +3434,11 @@ MemoryBuffer::getMemBuffer(ObjectBuffer, "in-memory object file", false); auto EFMM = std::make_unique(*BC); - EFMM->setNewSecPrefix(getNewSecPrefix()); - EFMM->setOrgSecPrefix(getOrgSecPrefix()); Linker = std::make_unique(*BC, std::move(EFMM)); - Linker->loadObject(ObjectMemBuffer->getMemBufferRef(), - [this](auto MapSection) { mapFileSections(MapSection); }); + Linker->loadObject( + ObjectMemBuffer->getMemBufferRef(), + [this](auto AssignAddress) { mapAllocatableSections(AssignAddress); }); MCAsmLayout FinalLayout( static_cast(Streamer.get())->getAssembler()); @@ -3247,11 +3451,20 @@ DebugInfoRewriter->updateLineTableOffsets(FinalLayout); if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) - RtLibrary->link(*BC, ToolPath, *Linker, [this](auto MapSection) { + RtLibrary->link(*BC, ToolPath, *Linker, [this](auto AssignAddress) { // Map newly registered sections. - this->mapAllocatableSections(MapSection); + this->mapRuntimeLibrary(AssignAddress); }); + if (opts::Rewrite) { + // map the first non-allocatable address to the first non-allocatable + // offset so that BSS doesn't occupy file space. + FirstNonAllocatableOffset = + BC->OutputSegments.back().p_offset + BC->OutputSegments.back().p_filesz; + BC->OutputAddressToOffsetMap[NextAvailableAddress] = + FirstNonAllocatableOffset; + } + mapNonLoadableSegments(); // Once the code is emitted, we can rename function sections to actual // output sections and de-register sections used for emission. for (BinaryFunction *Function : BC->getAllBinaryFunctions()) { @@ -3295,72 +3508,77 @@ addBoltInfoSection(); } -void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) { +void RewriteInstance::mapAllocatableSections( + BOLTLinker::SectionMapper AssignAddress) { + if (opts::Rewrite) { + // Assign new address to every section and + // put new sections in the end of old segments, + // matching sections with segments by their flags. + mapLoadableSegments(AssignAddress); + } else { + // Reassign old addresses to old sections and segments, + // and create new segments for new sections. + remapLoadableSegments(AssignAddress); + } + ProgramHeader LastSegment = BC->OutputSegments.back(); + const uint64_t EndOfLastSegmentInMem = + LastSegment.p_vaddr + LastSegment.p_memsz; + const uint64_t EndOfLastSegmentInFile = + LastSegment.p_offset + LastSegment.p_filesz; + for (BinarySection &Section : BC->allocatableSections()) { + if (!Section.getOutputAddress()) { + // because of strip, zero-sized sections such as .bss may end up outside + // of any segments and stay unmapped. Just put them in end of the last + // segment. + if (Section.getInputFileOffset() && Section.getSize() == 0) { + dbgs() << formatv("BOLT-WARNING: zero-sized allocatable section {0} is " + "outside of any segment.\n", + Section.getName()); + + Section.setOutputAddress(EndOfLastSegmentInMem); + Section.setOutputFileOffset(EndOfLastSegmentInFile); + if (Section.hasValidSectionID()) + AssignAddress(Section, Section.getOutputAddress()); + Section.setIsFinalized(); + } + } + } +} + +void RewriteInstance::remapLoadableSegments( + BOLTLinker::SectionMapper AssignAddress) { BC->deregisterUnusedSections(); + auto RestoreName = [this](BinarySection *Section, StringRef Name) { + if (!Section) + return; + LLVM_DEBUG(dbgs() << formatv("BOLT-DEBUG: original section {0} was " + "not duplicated, restoring name\n", + Name)); + assert(!getSection(Name) && "Unexpected section emitted"); + BC->renameSection(*Section, Name); + }; // If no new .eh_frame was written, remove relocated original .eh_frame. BinarySection *RelocatedEHFrameSection = getSection(".relocated" + getEHFrameSectionName()); if (RelocatedEHFrameSection && RelocatedEHFrameSection->hasValidSectionID()) { - BinarySection *NewEHFrameSection = - getSection(getNewSecPrefix() + getEHFrameSectionName()); + BinarySection *NewEHFrameSection = getSection(getEHFrameSectionName()); if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) { // JITLink will still have to process relocations for the section, hence // we need to assign it the address that wouldn't result in relocation // processing failure. - MapSection(*RelocatedEHFrameSection, NextAvailableAddress); + AssignAddress(*RelocatedEHFrameSection, NextAvailableAddress); BC->deregisterSection(*RelocatedEHFrameSection); + + // Remove org prefix from sections since they weren't duplicated + RestoreName(&*EHFrameSection, getEHFrameSectionName()); + RestoreName(getSection(getOrgSecPrefix() + getEHFrameHeaderSectionName()), + getEHFrameHeaderSectionName()); + RestoreName(getSection(getOrgSecPrefix() + ".gcc_except_table"), + ".gcc_except_table"); } } - mapCodeSections(MapSection); - - // Map the rest of the sections. - mapAllocatableSections(MapSection); -} - -std::vector RewriteInstance::getCodeSections() { - std::vector CodeSections; - for (BinarySection &Section : BC->textSections()) - if (Section.hasValidSectionID()) - CodeSections.emplace_back(&Section); - - auto compareSections = [&](const BinarySection *A, const BinarySection *B) { - // If both A and B have names starting with ".text.cold", then - // - if opts::HotFunctionsAtEnd is true, we want order - // ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold" - // - if opts::HotFunctionsAtEnd is false, we want order - // ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T" - if (A->getName().startswith(BC->getColdCodeSectionName()) && - B->getName().startswith(BC->getColdCodeSectionName())) { - if (A->getName().size() != B->getName().size()) - return (opts::HotFunctionsAtEnd) - ? (A->getName().size() > B->getName().size()) - : (A->getName().size() < B->getName().size()); - return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName()) - : (A->getName() < B->getName()); - } - - // Place movers before anything else. - if (A->getName() == BC->getHotTextMoverSectionName()) - return true; - if (B->getName() == BC->getHotTextMoverSectionName()) - return false; - - // Depending on the option, put main text at the beginning or at the end. - if (opts::HotFunctionsAtEnd) - return B->getName() == BC->getMainCodeSectionName(); - else - return A->getName() == BC->getMainCodeSectionName(); - }; - - // Determine the order of sections. - llvm::stable_sort(CodeSections, compareSections); - - return CodeSections; -} - -void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) { if (BC->HasRelocations) { // Map sections for functions with pre-assigned addresses. for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) { @@ -3372,97 +3590,49 @@ InjectedFunction->getCodeSection(); assert(FunctionSection && "function should have section"); FunctionSection->setOutputAddress(OutputAddress); - MapSection(*FunctionSection, OutputAddress); + AssignAddress(*FunctionSection, OutputAddress); InjectedFunction->setImageAddress(FunctionSection->getAllocAddress()); InjectedFunction->setImageSize(FunctionSection->getOutputSize()); } + } - // Populate the list of sections to be allocated. - std::vector CodeSections = getCodeSections(); - - // Remove sections that were pre-allocated (patch sections). - llvm::erase_if(CodeSections, [](BinarySection *Section) { - return Section->getOutputAddress(); - }); - LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n"; - for (const BinarySection *Section : CodeSections) - dbgs() << Section->getName() << '\n'; - ); - - uint64_t PaddingSize = 0; // size of padding required at the end - - // Allocate sections starting at a given Address. - auto allocateAt = [&](uint64_t Address) { - for (BinarySection *Section : CodeSections) { - Address = alignTo(Address, Section->getAlignment()); - Section->setOutputAddress(Address); - Address += Section->getOutputSize(); - - // Hugify: Additional huge page from right side due to - // weird ASLR mapping addresses (4KB aligned) - if (opts::Hugify && !BC->HasFixedLoadAddress && - Section->getName() == BC->getMainCodeSectionName()) - Address = alignTo(Address, Section->getAlignment()); - } - - // Make sure we allocate enough space for huge pages. - ErrorOr TextSection = - BC->getUniqueSectionByName(BC->getMainCodeSectionName()); - if (opts::HotText && TextSection && TextSection->hasValidSectionID()) { - uint64_t HotTextEnd = - TextSection->getOutputAddress() + TextSection->getOutputSize(); - HotTextEnd = alignTo(HotTextEnd, BC->PageAlign); - if (HotTextEnd > Address) { - PaddingSize = HotTextEnd - Address; - Address = HotTextEnd; - } - } - return Address; - }; - - // Check if we can fit code in the original .text - bool AllocationDone = false; - if (opts::UseOldText) { - const uint64_t CodeSize = - allocateAt(BC->OldTextSectionAddress) - BC->OldTextSectionAddress; - - if (CodeSize <= BC->OldTextSectionSize) { - outs() << "BOLT-INFO: using original .text for new code with 0x" - << Twine::utohexstr(opts::AlignText) << " alignment\n"; - AllocationDone = true; - } else { - errs() << "BOLT-WARNING: original .text too small to fit the new code" - << " using 0x" << Twine::utohexstr(opts::AlignText) - << " alignment. " << CodeSize << " bytes needed, have " - << BC->OldTextSectionSize << " bytes available.\n"; - opts::UseOldText = false; - } - } - - if (!AllocationDone) - NextAvailableAddress = allocateAt(NextAvailableAddress); - - // Do the mapping for ORC layer based on the allocation. - for (BinarySection *Section : CodeSections) { - LLVM_DEBUG( - dbgs() << "BOLT: mapping " << Section->getName() << " at 0x" - << Twine::utohexstr(Section->getAllocAddress()) << " to 0x" - << Twine::utohexstr(Section->getOutputAddress()) << '\n'); - MapSection(*Section, Section->getOutputAddress()); - Section->setOutputFileOffset( - getFileOffsetForAddress(Section->getOutputAddress())); - } + for (const ProgramHeader &Phdr : BC->loadableSegments()) { + copySegment(Phdr, AssignAddress); + } - // Check if we need to insert a padding section for hot text. - if (PaddingSize && !opts::UseOldText) - outs() << "BOLT-INFO: padding code to 0x" - << Twine::utohexstr(NextAvailableAddress) - << " to accommodate hot text\n"; + assert(NextAvailableAddress && "Uninitialized NextAvailableAddress!"); + assert(PHDRTableAddress && PHDRTableOffset && + "Uninitialized PHDRTable location!"); + BC->OutputSegments.insert( + BC->OutputSegments.begin(), + ProgramHeader(ELF::PT_PHDR, ELF::PF_R, PHDRTableOffset, PHDRTableAddress, + PHDRTableAddress, + /* file/mem size will be adjusted in writeELFPHDRTable*/ 0, + 0, 0x8)); - return; + if (!BC->HasRelocations) { + mapFunctionsNonRelocMode(AssignAddress); + } + + // if we instrument without rewriting, we want a single segment at + // PHDRTableAddress that will be created in mapRuntimeLibrary + if (BC->getRuntimeLibrary()) + mapSectionGroup(BC->getAllNewSections(), AssignAddress); + else { + createLoadSegment( + BC->getNewSectionsByFlags(ELF::SHF_ALLOC | ELF::SHF_EXECINSTR, + /*ROwithRX*/ true), + AssignAddress, ELF::PF_R | ELF::PF_X, BC->PageAlign, PHDRTableAddress); + // In case writable sections were emitted, put them in a separate segment. + createLoadSegment( + BC->getNewSectionsByFlags(ELF::SHF_ALLOC | ELF::SHF_WRITE), + AssignAddress, ELF::PF_R | ELF::PF_W, BC->PageAlign); } +} - // Processing in non-relocation mode. +// Processing in non-relocation mode. +void RewriteInstance::mapFunctionsNonRelocMode( + BOLTLinker::SectionMapper AssignAddress) { uint64_t NewTextSectionStartAddress = NextAvailableAddress; for (auto &BFI : BC->getBinaryFunctions()) { @@ -3474,11 +3644,12 @@ ErrorOr FuncSection = Function.getCodeSection(); assert(FuncSection && "cannot find section for function"); FuncSection->setOutputAddress(Function.getAddress()); - LLVM_DEBUG(dbgs() << "BOLT: mapping 0x" - << Twine::utohexstr(FuncSection->getAllocAddress()) - << " to 0x" << Twine::utohexstr(Function.getAddress()) - << '\n'); - MapSection(*FuncSection, Function.getAddress()); + LLVM_DEBUG(dbgs() << formatv( + "BOLT-DEBUG: mapping function {0} at {1:x} to {2:x}\n", + Function.getOneName(), FuncSection->getAllocAddress(), + Function.getAddress())); + + AssignAddress(*FuncSection, Function.getAddress()); Function.setImageAddress(FuncSection->getAllocAddress()); Function.setImageSize(FuncSection->getOutputSize()); if (Function.getImageSize() > Function.getMaxSize()) { @@ -3496,7 +3667,7 @@ LLVM_DEBUG(dbgs() << "BOLT-DEBUG: mapping JT " << Section.getName() << " to 0x" << Twine::utohexstr(JT->getAddress()) << '\n'); - MapSection(Section, JT->getAddress()); + AssignAddress(Section, JT->getAddress()); } } @@ -3524,7 +3695,7 @@ FF.setAddress(NextAvailableAddress); FF.setImageAddress(ColdSection->getAllocAddress()); FF.setImageSize(ColdSection->getOutputSize()); - FF.setFileOffset(getFileOffsetForAddress(NextAvailableAddress)); + FF.setFileOffset(getOutputFileOffsetForAddress(NextAvailableAddress)); ColdSection->setOutputAddress(FF.getAddress()); } @@ -3532,7 +3703,7 @@ dbgs() << formatv( "BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n", FF.getImageAddress(), FF.getAddress(), FF.getImageSize())); - MapSection(*ColdSection, FF.getAddress()); + AssignAddress(*ColdSection, FF.getAddress()); if (TooLarge) BC->deregisterSection(*ColdSection); @@ -3558,218 +3729,296 @@ 16); Section.setOutputAddress(NewTextSectionStartAddress); Section.setOutputFileOffset( - getFileOffsetForAddress(NewTextSectionStartAddress)); + getOutputFileOffsetForAddress(NewTextSectionStartAddress)); } } -void RewriteInstance::mapAllocatableSections( - BOLTLinker::SectionMapper MapSection) { - // Allocate read-only sections first, then writable sections. - enum : uint8_t { ST_READONLY, ST_READWRITE }; - for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) { - const uint64_t LastNextAvailableAddress = NextAvailableAddress; - if (SType == ST_READWRITE) { - // Align R+W segment to regular page size - NextAvailableAddress = alignTo(NextAvailableAddress, BC->RegularPageSize); - NewWritableSegmentAddress = NextAvailableAddress; - } - - for (BinarySection &Section : BC->allocatableSections()) { - if (!Section.hasValidSectionID()) - continue; +void RewriteInstance::createNonLoadSegment( + const std::vector &Sections, const unsigned p_type, + const unsigned p_flags, const unsigned p_align) { + assert(p_type != ELF::PT_LOAD && + "Called createNonLoadSegment with PT_LOAD type"); + if (p_type == ELF::PT_GNU_STACK) { + assert(Sections.empty() && "Unexpected sections in GNU_STACK segment"); + BC->OutputSegments.push_back( + ProgramHeader(ELF::PT_GNU_STACK, p_flags, 0, 0, 0, 0, 0, p_align)); + return; + } + if (Sections.empty()) + return; - if (Section.isWritable() == (SType == ST_READONLY)) - continue; + bool Mapped = all_of(Sections, std::bind(&BinarySection::getOutputAddress, + std::placeholders::_1)) && + all_of(Sections, std::bind(&BinarySection::getOutputFileOffset, + std::placeholders::_1)); + assert(Mapped && "attempt to create non-load segment with unmapped sections"); + (void)Mapped; + const uint64_t p_vaddr = Sections.front()->getOutputAddress(); + const uint64_t p_memsz = Sections.back()->getOutputAddress() + + Sections.back()->getOutputSize() - p_vaddr; + const uint64_t p_filesz = + p_memsz - Sections.back()->isVirtual() * Sections.back()->getOutputSize(); + const uint64_t p_offset = Sections.front()->getOutputFileOffset(); + ProgramHeader Result(p_type, p_flags, p_offset, p_vaddr, p_vaddr, p_filesz, + p_memsz, p_align); + + if (p_type == ELF::PT_INTERP) { + assert(BC->OutputSegments.size() && "PHDR segment expected"); + // Put INTERP after PHDR because it must precede loadable segments + BC->OutputSegments.insert(BC->OutputSegments.begin() + 1, Result); + } else { + BC->OutputSegments.push_back(Result); + } +} - if (Section.getOutputAddress()) { - LLVM_DEBUG({ - dbgs() << "BOLT-DEBUG: section " << Section.getName() - << " is already mapped at 0x" - << Twine::utohexstr(Section.getOutputAddress()) << '\n'; - }); - continue; - } +void RewriteInstance::mapSection(BinarySection &Section) { + assert(!Section.getOutputAddress()); + if (!Section.isTBSS()) + NextAvailableAddress = + alignTo(NextAvailableAddress, Section.getAlignment()); - if (Section.hasSectionRef()) { - LLVM_DEBUG({ - dbgs() << "BOLT-DEBUG: mapping original section " << Section.getName() - << " to 0x" << Twine::utohexstr(Section.getAddress()) << '\n'; - }); - Section.setOutputAddress(Section.getAddress()); - Section.setOutputFileOffset(Section.getInputFileOffset()); - MapSection(Section, Section.getAddress()); - } else { - NextAvailableAddress = - alignTo(NextAvailableAddress, Section.getAlignment()); - LLVM_DEBUG({ - dbgs() << "BOLT: mapping section " << Section.getName() << " (0x" - << Twine::utohexstr(Section.getAllocAddress()) << ") to 0x" - << Twine::utohexstr(NextAvailableAddress) << ":0x" - << Twine::utohexstr(NextAvailableAddress + - Section.getOutputSize()) - << '\n'; - }); + Section.setOutputAddress(NextAvailableAddress); + Section.setOutputFileOffset( + getOutputFileOffsetForAddress(Section.getOutputAddress())); - MapSection(Section, NextAvailableAddress); - Section.setOutputAddress(NextAvailableAddress); - Section.setOutputFileOffset( - getFileOffsetForAddress(NextAvailableAddress)); + if (!Section.isTBSS()) + NextAvailableAddress += Section.getOutputSize(); + Section.setIsFinalized(); + LLVM_DEBUG({ + std::string What = Section.hasSectionRef() ? "original" : "new"; + dbgs() << formatv( + "BOLT-DEBUG: mapping {0} section {1} ({2:x}) to {3:x}:{4:x}\n", What, + Section.getName(), Section.getAllocAddress(), + Section.getOutputAddress(), + Section.getOutputAddress() + Section.getOutputSize()); + }); + // Hugify: Additional huge page from right side due to + // weird ASLR mapping addresses (4KB aligned) + if (opts::Hugify && Section.getOutputName() == BC->getMainCodeSectionName()) + NextAvailableAddress = + alignTo(NextAvailableAddress, Section.getAlignment()); +} - NextAvailableAddress += Section.getOutputSize(); - } +uint64_t +RewriteInstance::mapSectionGroup(const std::vector &Sections, + BOLTLinker::SectionMapper AssignAddress) { + uint64_t NobitsSize = 0; + for (BinarySection *Section : Sections) { + if (Section->getOutputAddress()) { + LLVM_DEBUG({ + dbgs() << formatv( + "BOLT-DEBUG: section {0} is already mapped at {1:x}\n", + Section->getName(), Section->getOutputAddress()); + }); + continue; } - - if (SType == ST_READONLY) { - if (PHDRTableAddress) { - // Segment size includes the size of the PHDR area. - NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; - } else { - // Existing PHDR table would be updated. - NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; - } - } else if (SType == ST_READWRITE) { - NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress; - // Restore NextAvailableAddress if no new writable sections - if (!NewWritableSegmentSize) - NextAvailableAddress = LastNextAvailableAddress; + // .eh_frame_hdr address and size are assigned at mapEhFrames + if (Section->getName() == getEHFrameHeaderSectionName()) + continue; + if (Section->getName().endswith(getEHFrameSectionName())) { + mapEhFrameAndHeader(AssignAddress); + continue; } - } -} -void RewriteInstance::updateOutputValues(const MCAsmLayout &Layout) { - for (BinaryFunction *Function : BC->getAllBinaryFunctions()) - Function->updateOutputValues(Layout); + mapSection(*Section); + if (Section->isVirtual() && !Section->isTLS()) + NobitsSize += Section->getOutputSize(); + if (Section->hasValidSectionID()) + AssignAddress(*Section, Section->getOutputAddress()); + } + return NobitsSize; } +void RewriteInstance::createLoadSegment( + const std::vector &Sections, + BOLTLinker::SectionMapper AssignAddress, const unsigned p_flags, + const unsigned p_align, const std::optional ForceAddress) { -void RewriteInstance::patchELFPHDRTable() { - auto ELF64LEFile = cast(InputFile); - const ELFFile &Obj = ELF64LEFile->getELFFile(); - raw_fd_ostream &OS = Out->os(); - - // Write/re-write program headers. - Phnum = Obj.getHeader().e_phnum; - if (PHDRTableOffset) { - // Writing new pheader table and adding one new entry for R+X segment. - Phnum += 1; - if (NewWritableSegmentSize) { - // Adding one more entry for R+W segment. - Phnum += 1; - } - } else { - assert(!PHDRTableAddress && "unexpected address for program header table"); - PHDRTableOffset = Obj.getHeader().e_phoff; - if (NewWritableSegmentSize) { - errs() << "Unable to add writable segment with UseGnuStack option\n"; - exit(1); + if (Sections.empty()) + return; + if (!ForceAddress) { + NextAvailableAddress = alignTo(NextAvailableAddress, p_align); + if (BC->OutputSegments.size() && BC->OutputSegments.back().isLOAD()) { + const ProgramHeader &PrevSegment = BC->OutputSegments.back(); + NextAvailableAddress = alignTo(NextAvailableAddress, PrevSegment.p_align); + const uint64_t Align = + Sections.size() ? Sections.front()->getAlignment() : 8; + const uint64_t SegmentStartOffset = + alignTo(PrevSegment.p_offset + PrevSegment.p_memsz, Align); + NextAvailableAddress += SegmentStartOffset & (p_align - 1); + BC->OutputAddressToOffsetMap[NextAvailableAddress] = SegmentStartOffset; } } + uint64_t p_vaddr = NextAvailableAddress; + uint64_t p_offset = getOutputFileOffsetForAddress(NextAvailableAddress); + LLVM_DEBUG({ dbgs() << "BOLT-DEBUG: creating LOAD segment\n"; }); + const uint64_t NobitsSize = mapSectionGroup(Sections, AssignAddress); + uint64_t p_memsz = NextAvailableAddress - p_vaddr; + if (!p_memsz) + return; + if (ForceAddress) { + p_memsz += p_vaddr - *ForceAddress; + p_offset = getOutputFileOffsetForAddress(*ForceAddress); + p_vaddr = *ForceAddress; + } + uint64_t p_filesz = p_memsz - NobitsSize; + ProgramHeader Result = ProgramHeader(ELF::PT_LOAD, p_flags, p_offset, p_vaddr, + p_vaddr, p_filesz, p_memsz, p_align); + BC->OutputSegments.push_back(Result); +} - // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate - // last segments size based on the NextAvailableAddress variable. - if (!NewWritableSegmentSize) { - if (PHDRTableAddress) - NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; - else - NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; - } else { - NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress; +std::vector +RewriteInstance::getSectionsForSegment(const ProgramHeader &Phdr, + bool IncludeNew) { + std::vector Result; + std::vector Nobits; + for (BinarySection &Section : BC->allocatableSections()) { + if (Phdr.isTLS() > Section.isTLS()) + continue; + if (Phdr.contains(Section) && !shouldStrip(Section)) { + if (Section.isVirtual() && !Section.isTLS()) + Nobits.push_back(&Section); + else + Result.push_back(&Section); + } } - OS.seek(PHDRTableOffset); + auto GetSectionIt = [&Result](StringRef Name) { + return std::find_if( + Result.begin(), Result.end(), + [Name](BinarySection *Sec) { return Sec->getOutputName() == Name; }); + }; - bool ModdedGnuStack = false; - (void)ModdedGnuStack; - bool AddedSegment = false; - (void)AddedSegment; - - auto createNewTextPhdr = [&]() { - ELF64LEPhdrTy NewPhdr; - NewPhdr.p_type = ELF::PT_LOAD; - if (PHDRTableAddress) { - NewPhdr.p_offset = PHDRTableOffset; - NewPhdr.p_vaddr = PHDRTableAddress; - NewPhdr.p_paddr = PHDRTableAddress; + if (IncludeNew && Phdr.isLOAD()) { + std::vector Extra = BC->getNewSectionsByFlags( + Phdr.getSectionFlags(), /*ROwithRX*/ !HasReadOnlySegment); + // We need to put .text.cold after .text + auto PosForNewSections = Result.end(); + if (Phdr.isExec()) { + PosForNewSections = GetSectionIt(".text"); + if (PosForNewSections != Result.end()) + ++PosForNewSections; + } + Result.insert(PosForNewSections, Extra.begin(), Extra.end()); + } + if (opts::Rewrite && Phdr.isExec()) { + // Put .plt before .text because: + // - it makes clear where PLT will be relative to .text, which simplifies + // stub insertion for AArch64. Without it we can't decide where to insert + // stubs because we don't know the final binary layout before linking. + // - unless --hot-functions-at-end is used, PLT comes just before hot text + // section which increases locality. + auto PltIt = GetSectionIt(".plt"); + if (PltIt != Result.end()) { + BinarySection *Plt = *PltIt; + Result.erase(PltIt); + auto TextIt = GetSectionIt(".text"); + assert(TextIt != Result.end() && "No .text section!"); + Result.insert(TextIt, Plt); + } + } + Result.insert(Result.end(), Nobits.begin(), Nobits.end()); + return Result; +} +void RewriteInstance::mapNonLoadableSegments() { + for (const ProgramHeader &Phdr : BC->nonLoadableSegments()) { + // PHDR is created earlier + if (Phdr.p_type == ELF::PT_PHDR) + continue; + if (!opts::Rewrite && Phdr.p_type == ELF::PT_GNU_EH_FRAME) { + BinarySection *EHFrameHeader = getSection(getEHFrameHeaderSectionName()); + assert(EHFrameHeader && "Cannot find .eh_frame_hdr for PT_GNU_EH_FRAME!"); + createNonLoadSegment({EHFrameHeader}, Phdr.p_type, Phdr.p_flags, + Phdr.p_align); } else { - NewPhdr.p_offset = NewTextSegmentOffset; - NewPhdr.p_vaddr = NewTextSegmentAddress; - NewPhdr.p_paddr = NewTextSegmentAddress; - } - NewPhdr.p_filesz = NewTextSegmentSize; - NewPhdr.p_memsz = NewTextSegmentSize; - NewPhdr.p_flags = ELF::PF_X | ELF::PF_R; - // FIXME: Currently instrumentation is experimental and the runtime data - // is emitted with code, thus everything needs to be writable - if (opts::Instrument) - NewPhdr.p_flags |= ELF::PF_W; - NewPhdr.p_align = BC->PageAlign; - - return NewPhdr; - }; + createNonLoadSegment(getSectionsForSegment(Phdr), Phdr.p_type, + Phdr.p_flags, Phdr.p_align); + } + } +} - auto createNewWritableSectionsPhdr = [&]() { - ELF64LEPhdrTy NewPhdr; - NewPhdr.p_type = ELF::PT_LOAD; - NewPhdr.p_offset = getFileOffsetForAddress(NewWritableSegmentAddress); - NewPhdr.p_vaddr = NewWritableSegmentAddress; - NewPhdr.p_paddr = NewWritableSegmentAddress; - NewPhdr.p_filesz = NewWritableSegmentSize; - NewPhdr.p_memsz = NewWritableSegmentSize; - NewPhdr.p_align = BC->RegularPageSize; - NewPhdr.p_flags = ELF::PF_R | ELF::PF_W; - return NewPhdr; - }; +void RewriteInstance::mapLoadableSegments( + BOLTLinker::SectionMapper AssignAddress) { + BC->deregisterUnusedSections(); - // Copy existing program headers with modifications. - for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) { - ELF64LE::Phdr NewPhdr = Phdr; - if (PHDRTableAddress && Phdr.p_type == ELF::PT_PHDR) { - NewPhdr.p_offset = PHDRTableOffset; - NewPhdr.p_vaddr = PHDRTableAddress; - NewPhdr.p_paddr = PHDRTableAddress; - NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum; - NewPhdr.p_memsz = sizeof(NewPhdr) * Phnum; - } else if (Phdr.p_type == ELF::PT_GNU_EH_FRAME) { - ErrorOr EHFrameHdrSec = - BC->getUniqueSectionByName(getNewSecPrefix() + ".eh_frame_hdr"); - if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() && - EHFrameHdrSec->isFinalized()) { - NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset(); - NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress(); - NewPhdr.p_paddr = EHFrameHdrSec->getOutputAddress(); - NewPhdr.p_filesz = EHFrameHdrSec->getOutputSize(); - NewPhdr.p_memsz = EHFrameHdrSec->getOutputSize(); - } - } else if (opts::UseGnuStack && Phdr.p_type == ELF::PT_GNU_STACK) { - NewPhdr = createNewTextPhdr(); - ModdedGnuStack = true; - } else if (!opts::UseGnuStack && Phdr.p_type == ELF::PT_DYNAMIC) { - // Insert the new header before DYNAMIC. - ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); - OS.write(reinterpret_cast(&NewTextPhdr), - sizeof(NewTextPhdr)); - if (NewWritableSegmentSize) { - ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr(); - OS.write(reinterpret_cast(&NewWritablePhdr), - sizeof(NewWritablePhdr)); - } - AddedSegment = true; - } - OS.write(reinterpret_cast(&NewPhdr), sizeof(NewPhdr)); + PHDRTableOffset = 0x40; + NextAvailableAddress = BaseAddress + PHDRTableOffset; + PHDRTableAddress = NextAvailableAddress; + assert(BC->OutputAddressToOffsetMap.empty() && "Unexpected mapping!"); + + BC->OutputAddressToOffsetMap[BaseAddress] = 0; + // add PHDR and reserve space for RE/RW segments for runtime library + uint64_t Phnum = BC->InputSegments.size() + !HasProgramHeaderSegment + + (!!BC->getRuntimeLibrary()) * 2; + uint64_t PHDRTableSize = Phnum * sizeof(ELF64LE::Phdr); + BC->MaxPHDRSize = PHDRTableSize; + BC->OutputSegments.push_back( + ProgramHeader(ELF::PT_PHDR, ELF::PF_R, PHDRTableOffset, PHDRTableAddress, + PHDRTableAddress, PHDRTableSize, PHDRTableSize, 0x8)); + NextAvailableAddress += PHDRTableSize; + + for (const ProgramHeader &Phdr : BC->loadableSegments()) { + + const uint64_t Align = Phdr.isExec() ? BC->PageAlign : Phdr.p_align; + bool IsFirstLoad = (Phdr.p_offset == 0); + createLoadSegment(getSectionsForSegment(Phdr), AssignAddress, Phdr.p_flags, + Align, + /*ForceAddress=*/ + IsFirstLoad ? std::optional(BaseAddress) : std::nullopt); + } +} +void RewriteInstance::copySegment(const ProgramHeader &Segment, + BOLTLinker::SectionMapper AssignAddress) { + BC->OutputSegments.push_back(Segment); + BC->OutputAddressToOffsetMap[Segment.p_vaddr] = Segment.p_offset; + for (BinarySection *Section : getSectionsForSegment(Segment, false)) { + + Section->setOutputAddress(Section->getAddress()); + Section->setOutputFileOffset(Section->getInputFileOffset()); + if (Section->hasValidSectionID()) + AssignAddress(*Section, Section->getOutputAddress()); + Section->setIsFinalized(); } +} - if (!opts::UseGnuStack && !AddedSegment) { - // Append the new header to the end of the table. - ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); - OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); - if (NewWritableSegmentSize) { - ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr(); - OS.write(reinterpret_cast(&NewWritablePhdr), - sizeof(NewWritablePhdr)); - } +void RewriteInstance::mapRuntimeLibrary( + BOLTLinker::SectionMapper AssignAddress) { + if (opts::Rewrite) { + // if we are rewriting, we can afford two segments with proper flags. + createLoadSegment( + BC->getNewSectionsByFlags(ELF::SHF_ALLOC | ELF::SHF_EXECINSTR, + /*ROwithRX*/ true), + AssignAddress, ELF::PF_R | ELF::PF_X, BC->RegularPageSize); + createLoadSegment( + BC->getNewSectionsByFlags(ELF::SHF_ALLOC | ELF::SHF_WRITE), + AssignAddress, ELF::PF_R | ELF::PF_W, BC->RegularPageSize); + } else { + auto NewSegmentContents = BC->getAllNewSections(); + unsigned Flags = ELF::PF_R | ELF::PF_X | ELF::PF_W; + createLoadSegment(NewSegmentContents, AssignAddress, Flags, BC->PageAlign, + PHDRTableAddress); } +} + +void RewriteInstance::updateOutputValues(const MCAsmLayout &Layout) { + for (BinaryFunction *Function : BC->getAllBinaryFunctions()) + Function->updateOutputValues(Layout); +} - assert((!opts::UseGnuStack || ModdedGnuStack) && - "could not find GNU_STACK program header to modify"); +void RewriteInstance::writeELFPHDRTable() { + raw_fd_ostream &OS = Out->os(); + assert(PHDRTableOffset && "No PHDR table offset!"); + assert(BC->OutputSegments.size() && "Empty PHDR!"); + assert(BC->OutputSegments[0].p_type == ELF::PT_PHDR && "No PHDR segment!"); + OS.seek(PHDRTableOffset); + uint64_t PHDRTableSize = BC->OutputSegments.size() * sizeof(ELF64LE::Phdr); + assert(PHDRTableSize <= BC->MaxPHDRSize && "Exceeded MaxPHDRSize!"); + BC->OutputSegments[0].p_filesz = PHDRTableSize; + BC->OutputSegments[0].p_memsz = PHDRTableSize; + for (const auto &Phdr : BC->OutputSegments) { + ELF64LE::Phdr PhdrToWrite(Phdr); + OS.write(reinterpret_cast(&PhdrToWrite), sizeof(PhdrToWrite)); + } + return; } namespace { @@ -3795,7 +4044,8 @@ const ELFFile &Obj = ELF64LEFile->getELFFile(); raw_fd_ostream &OS = Out->os(); - uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress); + uint64_t NextAvailableOffset = + getOutputFileOffsetForAddress(NextAvailableAddress); assert(NextAvailableOffset >= FirstNonAllocatableOffset && "next available offset calculation failure"); OS.seek(NextAvailableOffset); @@ -3853,7 +4103,7 @@ assert(BSec->getAlignment() <= Section.sh_addralign && "alignment exceeds value in file"); - if (BSec->getAllocAddress()) { + if (BSec->getAllocAddress() && !DataWritten) { assert(!DataWritten && "Writing section twice."); (void)DataWritten; SectionData = BSec->getOutputData(); @@ -3881,7 +4131,8 @@ // Write new note sections. for (BinarySection &Section : BC->nonAllocatableSections()) { - if (Section.getOutputFileOffset() || !Section.getAllocAddress()) + if (Section.getOutputFileOffset() || !Section.getAllocAddress() || + shouldStrip(Section)) continue; assert(!Section.hasPendingRelocations() && "cannot have pending relocs"); @@ -3904,8 +4155,12 @@ void RewriteInstance::finalizeSectionStringTable(ELFObjectFile *File) { // Pre-populate section header string table. for (const BinarySection &Section : BC->sections()) - if (!Section.isAnonymous()) + if (!Section.isAnonymous()) { SHStrTab.add(Section.getOutputName()); + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding " << Section.getOutputName() + << " to shstrtab\n";); + } + SHStrTab.finalize(); const size_t SHStrTabSize = SHStrTab.getSize(); @@ -3978,6 +4233,22 @@ return false; } +bool RewriteInstance::shouldStrip(const BinarySection &Section) { + // Strip non-allocatable relocation sections. + if (!(Section.getELFFlags() & ELF::SHF_ALLOC) && + Section.getELFType() == ELF::SHT_RELA) + return true; + + // Strip debug sections if not updating them. + if (isDebugSection(Section.getName()) && !opts::UpdateDebugSections) + return true; + + // Strip symtab section if needed + if (opts::RemoveSymtab && Section.getELFType() == ELF::SHT_SYMTAB) + return true; + + return false; +} template std::vector::Elf_Shdr> @@ -3985,39 +4256,23 @@ std::vector &NewSectionIndex) { using ELFShdrTy = typename ELFObjectFile::Elf_Shdr; const ELFFile &Obj = File->getELFFile(); - typename ELFT::ShdrRange Sections = cantFail(Obj.sections()); + typename ELFT::ShdrRange InputSections = cantFail(Obj.sections()); - // Keep track of section header entries attached to the corresponding section. - std::vector> OutputSections; - auto addSection = [&](const ELFShdrTy &Section, BinarySection *BinSec) { + // Keep track of section header entries together with their name. + std::vector> OutputSections; + auto addSection = [&](const std::string &Name, const ELFShdrTy &Section) { ELFShdrTy NewSection = Section; - NewSection.sh_name = SHStrTab.getOffset(BinSec->getOutputName()); - OutputSections.emplace_back(BinSec, std::move(NewSection)); + NewSection.sh_name = SHStrTab.getOffset(Name); + OutputSections.emplace_back(Name, std::move(NewSection)); }; - // Copy over entries for original allocatable sections using modified name. - for (const ELFShdrTy &Section : Sections) { - // Always ignore this section. - if (Section.sh_type == ELF::SHT_NULL) { - OutputSections.emplace_back(nullptr, Section); - continue; - } - - if (!(Section.sh_flags & ELF::SHF_ALLOC)) - continue; - - SectionRef SecRef = File->toSectionRef(&Section); - BinarySection *BinSec = BC->getSectionForSectionRef(SecRef); - assert(BinSec && "Matching BinarySection should exist."); - - addSection(Section, BinSec); - } + // NULL Section + OutputSections.emplace_back("", ELFShdrTy{}); - for (BinarySection &Section : BC->allocatableSections()) { - if (!Section.isFinalized()) - continue; + // TODO: Reduce number of for loops + for (const BinarySection &Section : BC->allocatableSections()) { - if (Section.hasSectionRef() || Section.isAnonymous()) { + if (Section.isAnonymous() || !Section.isFinalized()) { if (opts::Verbosity) outs() << "BOLT-INFO: not writing section header for section " << Section.getOutputName() << '\n'; @@ -4025,10 +4280,10 @@ } if (opts::Verbosity >= 1) - outs() << "BOLT-INFO: writing section header for " - << Section.getOutputName() << '\n'; + outs() << "BOLT-INFO: writing section header for " << Section.getName() + << '\n'; ELFShdrTy NewSection; - NewSection.sh_type = ELF::SHT_PROGBITS; + NewSection.sh_type = Section.getELFType(); NewSection.sh_addr = Section.getOutputAddress(); NewSection.sh_offset = Section.getOutputFileOffset(); NewSection.sh_size = Section.getOutputSize(); @@ -4037,17 +4292,19 @@ NewSection.sh_link = 0; NewSection.sh_info = 0; NewSection.sh_addralign = Section.getAlignment(); - addSection(NewSection, &Section); + addSection(std::string(Section.getOutputName()), NewSection); } // Sort all allocatable sections by their offset. - llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) { - return A.second.sh_offset < B.second.sh_offset; - }); + llvm::stable_sort(OutputSections, + [](const std::pair &A, + const std::pair &B) { + return A.second.sh_offset < B.second.sh_offset; + }); // Fix section sizes to prevent overlapping. ELFShdrTy *PrevSection = nullptr; - BinarySection *PrevBinSec = nullptr; + StringRef PrevSectionName; for (auto &SectionKV : OutputSections) { ELFShdrTy &Section = SectionKV.second; @@ -4059,22 +4316,22 @@ if (PrevSection && PrevSection->sh_addr + PrevSection->sh_size > Section.sh_addr) { if (opts::Verbosity > 1) - outs() << "BOLT-INFO: adjusting size for section " - << PrevBinSec->getOutputName() << '\n'; + outs() << "BOLT-INFO: adjusting size for section " << PrevSectionName + << '\n'; PrevSection->sh_size = Section.sh_addr > PrevSection->sh_addr ? Section.sh_addr - PrevSection->sh_addr : 0; } PrevSection = &Section; - PrevBinSec = SectionKV.first; + PrevSectionName = SectionKV.first; } uint64_t LastFileOffset = 0; // Copy over entries for non-allocatable sections performing necessary // adjustments. - for (const ELFShdrTy &Section : Sections) { + for (const ELFShdrTy &Section : InputSections) { if (Section.sh_type == ELF::SHT_NULL) continue; if (Section.sh_flags & ELF::SHF_ALLOC) @@ -4086,20 +4343,19 @@ if (shouldStrip(Section, SectionName)) continue; - SectionRef SecRef = File->toSectionRef(&Section); - BinarySection *BinSec = BC->getSectionForSectionRef(SecRef); - assert(BinSec && "Matching BinarySection should exist."); + ErrorOr BSec = BC->getUniqueSectionByName(SectionName); + assert(BSec && "missing section info for non-allocatable section"); ELFShdrTy NewSection = Section; - NewSection.sh_offset = BinSec->getOutputFileOffset(); - NewSection.sh_size = BinSec->getOutputSize(); + NewSection.sh_offset = BSec->getOutputFileOffset(); + NewSection.sh_size = BSec->getOutputSize(); if (NewSection.sh_type == ELF::SHT_SYMTAB) NewSection.sh_info = NumLocalSymbols; - addSection(NewSection, BinSec); + addSection(std::string(SectionName), NewSection); - LastFileOffset = BinSec->getOutputFileOffset(); + LastFileOffset = BSec->getOutputFileOffset(); } // Create entries for new non-allocatable sections. @@ -4107,9 +4363,12 @@ if (Section.getOutputFileOffset() <= LastFileOffset) continue; + if (shouldStrip(Section)) + continue; + if (opts::Verbosity >= 1) - outs() << "BOLT-INFO: writing section header for " - << Section.getOutputName() << '\n'; + outs() << "BOLT-INFO: writing section header for " << Section.getName() + << '\n'; ELFShdrTy NewSection; NewSection.sh_type = Section.getELFType(); @@ -4122,34 +4381,72 @@ NewSection.sh_info = 0; NewSection.sh_addralign = Section.getAlignment(); - addSection(NewSection, &Section); + addSection(std::string(Section.getName()), NewSection); } // Assign indices to sections. std::unordered_map NameToIndex; - for (uint32_t Index = 1; Index < OutputSections.size(); ++Index) - OutputSections[Index].first->setIndex(Index); + for (uint32_t Index = 1; Index < OutputSections.size(); ++Index) { + const std::string &SectionName = OutputSections[Index].first; + NameToIndex[SectionName] = Index; + if (ErrorOr Section = + BC->getUniqueSectionByName(SectionName)) + Section->setIndex(Index); + } // Update section index mapping NewSectionIndex.clear(); - NewSectionIndex.resize(Sections.size(), 0); - for (const ELFShdrTy &Section : Sections) { + NewSectionIndex.resize(InputSections.size(), 0); + for (const ELFShdrTy &Section : InputSections) { if (Section.sh_type == ELF::SHT_NULL) continue; - size_t OrgIndex = std::distance(Sections.begin(), &Section); - - SectionRef SecRef = File->toSectionRef(&Section); - BinarySection *BinSec = BC->getSectionForSectionRef(SecRef); - assert(BinSec && "BinarySection should exist for an input section."); + size_t OldIndex = std::distance(InputSections.begin(), &Section); + std::string SectionName = + std::string(cantFail(Obj.getSectionName(Section))); // Some sections are stripped - if (!BinSec->hasValidIndex()) + if (!NameToIndex.count(SectionName)) continue; - - NewSectionIndex[OrgIndex] = BinSec->getIndex(); + if (SectionName == ".rodata" || SectionName == ".text") { + std::string OrgSec = (getOrgSecPrefix() + SectionName).str(); + // if we emitted new .rodata or .text, point the indices to the old ones + if (NameToIndex.find(OrgSec) != NameToIndex.end()) { + NewSectionIndex[OldIndex] = NameToIndex[OrgSec]; + continue; + } + } + NewSectionIndex[OldIndex] = NameToIndex[SectionName]; } + for (const ELFShdrTy &Section : InputSections) { + if (Section.sh_type == ELF::SHT_NULL) + continue; + std::string SectionName = + std::string(cantFail(Obj.getSectionName(Section))); + + if (shouldStrip(Section, SectionName)) + continue; + const uint64_t NewIndex = NameToIndex[SectionName]; + if (Section.sh_link) { + auto &LinkSection = InputSections[Section.sh_link]; + std::string LinkSectionName = + std::string(cantFail(Obj.getSectionName(LinkSection))); + OutputSections[NewIndex].second.sh_link = NameToIndex[LinkSectionName]; + } + if (Section.sh_info) { + if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA) { + auto &InfoSection = InputSections[Section.sh_info]; + std::string InfoSectionName = + std::string(cantFail(Obj.getSectionName(InfoSection))); + OutputSections[NewIndex].second.sh_info = NameToIndex[InfoSectionName]; + } else if (SectionName == ".symtab") + OutputSections[NewIndex].second.sh_info = NumLocalSymbols; + else + OutputSections[NewIndex].second.sh_info = Section.sh_info; + } + OutputSections[NewIndex].second.sh_entsize = Section.sh_entsize; + } std::vector SectionsOnly(OutputSections.size()); llvm::copy(llvm::make_second_range(OutputSections), SectionsOnly.begin()); @@ -4186,11 +4483,6 @@ // Write all section header entries while patching section references. for (ELFShdrTy &Section : OutputSections) { - Section.sh_link = NewSectionIndex[Section.sh_link]; - if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA) { - if (Section.sh_info) - Section.sh_info = NewSectionIndex[Section.sh_info]; - } OS.write(reinterpret_cast(&Section), sizeof(Section)); } @@ -4206,7 +4498,7 @@ "cannot find new address for entry point"); } NewEhdr.e_phoff = PHDRTableOffset; - NewEhdr.e_phnum = Phnum; + NewEhdr.e_phnum = BC->OutputSegments.size(); NewEhdr.e_shoff = SHTOffset; NewEhdr.e_shnum = OutputSections.size(); NewEhdr.e_shstrndx = NewSectionIndex[NewEhdr.e_shstrndx]; @@ -4379,7 +4671,7 @@ // For dynamic symbol table, the section index could be wrong on the input, // and its value is ignored by the runtime if it's different from // SHN_UNDEF and SHN_ABS. - if (!IsDynSym && Function && + if (!IsDynSym && Function && Symbol.st_shndx && Symbol.st_shndx != Function->getOriginSection()->getSectionRef().getIndex()) Function = nullptr; @@ -4390,7 +4682,9 @@ if (Function) { // If the symbol matched a function that was not emitted, update the // corresponding section index but otherwise leave it unchanged. - if (Function->isEmitted()) { + if (Function->isPLTFunction()) { + NewSymbol.st_value = Function->getOutputAddress(); + } else if (Function->isEmitted()) { NewSymbol.st_value = Function->getOutputAddress(); NewSymbol.st_size = Function->getOutputSize(); NewSymbol.st_shndx = Function->getCodeSection()->getIndex(); @@ -4443,13 +4737,11 @@ Function->getCodeSection(FF->getFragmentNum())->getIndex(); } else { // Check if the symbol belongs to moved data object and update it. - BinaryData *BD = opts::ReorderData.empty() - ? nullptr - : BC->getBinaryDataAtAddress(Symbol.st_value); + BinaryData *BD = nullptr; + if (!opts::ReorderData.empty() || opts::Rewrite) + BD = BC->getBinaryDataAtAddress(Symbol.st_value); + if (BD && BD->isMoved() && !BD->isJumpTable()) { - assert((!BD->getSize() || !Symbol.st_size || - Symbol.st_size == BD->getSize()) && - "sizes must match"); BinarySection &OutputSection = BD->getOutputSection(); assert(OutputSection.getIndex()); @@ -4495,6 +4787,15 @@ << Twine::utohexstr(NewSymbol.st_value) << '\n'; }; + auto SetToEnd = [&](BinarySection *Section) { + if (Section->getOutputName() == ".bolt.org.text") { + // move .text end symbol with text itself. + Section = &BC->getUniqueSectionByName(".text").get(); + } + NewSymbol.st_value = Section->getOutputEndAddress(); + NewSymbol.st_shndx = Section->getIndex(); + }; + if (opts::HotText && (*SymbolName == "__hot_start" || *SymbolName == "__hot_end")) { updateSymbolValue(*SymbolName); @@ -4507,6 +4808,9 @@ ++NumHotDataSymsUpdated; } + if (auto It = BC->EndSymbols.find(*SymbolName); It != BC->EndSymbols.end()) + SetToEnd(It->second); + if (*SymbolName == "_end") updateSymbolValue(*SymbolName, NextAvailableAddress); @@ -4610,7 +4914,8 @@ // Set pointer at the end of the output file, so we can pwrite old symbol // tables if we need to. - uint64_t NextAvailableOffset = getFileOffsetForAddress(NextAvailableAddress); + uint64_t NextAvailableOffset = + getOutputFileOffsetForAddress(NextAvailableAddress); assert(NextAvailableOffset >= FirstNonAllocatableOffset && "next available offset calculation failure"); Out->os().seek(NextAvailableOffset); @@ -4625,16 +4930,18 @@ } assert((DynSymSection || BC->IsStaticExecutable) && "dynamic symbol table expected"); + if (DynSymSection) { + auto Dynsym = BC->getUniqueSectionByName(".dynsym"); + assert(Dynsym); + const uint64_t NewOffset = Dynsym->getOutputFileOffset(); + assert(NewOffset); updateELFSymbolTable( File, - /*IsDynSym=*/true, - *DynSymSection, - NewSectionIndex, - [&](size_t Offset, const ELFSymTy &Sym) { + /*IsDynSym=*/true, *DynSymSection, NewSectionIndex, + [&, NewOffset](size_t Offset, const ELFSymTy &Sym) { Out->os().pwrite(reinterpret_cast(&Sym), - sizeof(ELFSymTy), - DynSymSection->sh_offset + Offset); + sizeof(ELFSymTy), NewOffset + Offset); }, [](StringRef) -> size_t { return 0; }); } @@ -4708,13 +5015,15 @@ const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize; auto FixAddend = [&](const BinarySection &Section, const Relocation &Rel) { - // Fix relocation symbol value in place if no static relocation found - // on the same address - if (Section.getRelocationAt(Rel.Offset)) - return; + uint64_t Addend = 0; + if (const uint64_t EndSymValue = Section.getNewEndSymbolValue(Rel.Offset)) + Addend = EndSymValue; + + if (!Addend) + Addend = getNewFunctionOrDataAddress(Rel.Addend); // No fixup needed if symbol address was not changed - const uint64_t Addend = getNewFunctionOrDataAddress(Rel.Addend); + if (!Addend) return; @@ -4752,7 +5061,9 @@ BC->getSectionForAddress(*DynamicRelrAddress); assert(Section && "cannot get .relr.dyn section"); assert(Section->isRelr() && "Expected section to be SHT_RELR type"); - uint64_t RelrDynOffset = Section->getInputFileOffset(); + uint64_t RelrDynOffset = Section->getOutputFileOffset(); + assert(RelrDynOffset && "No output offset for .relr.dyn"); + const uint64_t RelrDynEndOffset = RelrDynOffset + Section->getSize(); auto WriteRelr = [&](uint64_t Value) { @@ -4806,8 +5117,8 @@ uint64_t &End) { ErrorOr Section = BC->getSectionForAddress(Address); assert(Section && "cannot get relocation section"); - Start = Section->getInputFileOffset(); - End = Start + Section->getSize(); + Start = Section->getOutputFileOffset(); + End = Start + Section->getOutputSize(); }; if (!DynamicRelocationsAddress && !PLTRelocationsAddress) @@ -4855,7 +5166,14 @@ SymbolIdx = getOutputDynamicSymbolIndex(Symbol); } else { // Usually this case is used for R_*_(I)RELATIVE relocations - const uint64_t Address = getNewFunctionOrDataAddress(Addend); + + // Check if it's end-of-section relocation first because they're + // tricky + uint64_t Address = Section.getNewEndSymbolValue(Rel.Offset); + + if (!Address) + Address = getNewFunctionOrDataAddress(Rel.Addend); + if (Address) Addend = Address; } @@ -4909,41 +5227,6 @@ fillNone(RelPltOffset, RelPltEndOffset); } -template -void RewriteInstance::patchELFGOT(ELFObjectFile *File) { - raw_fd_ostream &OS = Out->os(); - - SectionRef GOTSection; - for (const SectionRef &Section : File->sections()) { - StringRef SectionName = cantFail(Section.getName()); - if (SectionName == ".got") { - GOTSection = Section; - break; - } - } - if (!GOTSection.getObject()) { - if (!BC->IsStaticExecutable) - errs() << "BOLT-INFO: no .got section found\n"; - return; - } - - StringRef GOTContents = cantFail(GOTSection.getContents()); - for (const uint64_t *GOTEntry = - reinterpret_cast(GOTContents.data()); - GOTEntry < reinterpret_cast(GOTContents.data() + - GOTContents.size()); - ++GOTEntry) { - if (uint64_t NewAddress = getNewFunctionAddress(*GOTEntry)) { - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x" - << Twine::utohexstr(*GOTEntry) << " with 0x" - << Twine::utohexstr(NewAddress) << '\n'); - OS.pwrite(reinterpret_cast(&NewAddress), sizeof(NewAddress), - reinterpret_cast(GOTEntry) - - File->getData().data()); - } - } -} - template void RewriteInstance::patchELFDynamic(ELFObjectFile *File) { if (BC->IsStaticExecutable) @@ -4952,15 +5235,12 @@ const ELFFile &Obj = File->getELFFile(); raw_fd_ostream &OS = Out->os(); - using Elf_Phdr = typename ELFFile::Elf_Phdr; using Elf_Dyn = typename ELFFile::Elf_Dyn; // Locate DYNAMIC by looking through program headers. - uint64_t DynamicOffset = 0; - const Elf_Phdr *DynamicPhdr = nullptr; - for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) { + const ProgramHeader *DynamicPhdr = nullptr; + for (const ProgramHeader &Phdr : BC->InputSegments) { if (Phdr.p_type == ELF::PT_DYNAMIC) { - DynamicOffset = Phdr.p_offset; DynamicPhdr = &Phdr; assert(Phdr.p_memsz == Phdr.p_filesz && "dynamic sizes should match"); break; @@ -4970,6 +5250,10 @@ bool ZNowSet = false; + auto DynamicSection = BC->getUniqueSectionByName(".dynamic"); + assert(DynamicSection); + uint64_t NewDynamicOffset = DynamicSection->getOutputFileOffset(); + // Go through all dynamic entries and patch functions addresses with // new ones. typename ELFT::DynRange DynamicEntries = @@ -5020,10 +5304,30 @@ ZNowSet = true; } break; + case ELF::DT_INIT_ARRAY: + case ELF::DT_FINI_ARRAY: + case ELF::DT_GNU_HASH: + case ELF::DT_SYMTAB: + case ELF::DT_STRTAB: + case ELF::DT_PLTGOT: + case ELF::DT_RELA: + case ELF::DT_RELR: + case ELF::DT_JMPREL: + case ELF::DT_VERNEED: + case ELF::DT_VERSYM: { + auto Section = BC->getSectionForAddress(Dyn.d_un.d_ptr); + assert(Section && "Cant'find section for dynamic entry"); + assert(Section->getAddress() == NewDE.d_un.d_ptr && + "Invalid address for dynamic entry"); + assert(Section->getOutputAddress() && + "No output address for section referred in .dynamic"); + NewDE.d_un.d_ptr = Section->getOutputAddress(); + break; + } } if (ShouldPatch) OS.pwrite(reinterpret_cast(&NewDE), sizeof(NewDE), - DynamicOffset + (&Dyn - DTB) * sizeof(Dyn)); + NewDynamicOffset + (&Dyn - DTB) * sizeof(Dyn)); } if (BC->RequiresZNow && !ZNowSet) { @@ -5038,12 +5342,11 @@ Error RewriteInstance::readELFDynamic(ELFObjectFile *File) { const ELFFile &Obj = File->getELFFile(); - using Elf_Phdr = typename ELFFile::Elf_Phdr; using Elf_Dyn = typename ELFFile::Elf_Dyn; // Locate DYNAMIC by looking through program headers. - const Elf_Phdr *DynamicPhdr = nullptr; - for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) { + const ProgramHeader *DynamicPhdr = 0; + for (const ProgramHeader &Phdr : BC->InputSegments) { if (Phdr.p_type == ELF::PT_DYNAMIC) { DynamicPhdr = &Phdr; break; @@ -5147,6 +5450,10 @@ if (BD && BD->isMoved()) return BD->getOutputAddress(); + if (auto Section = BC->getSectionForAddress(OldAddress)) { + assert(Section->getOutputAddress()); + return Section->getOutputAddress() + OldAddress - Section->getAddress(); + } return 0; } @@ -5158,15 +5465,21 @@ raw_fd_ostream &OS = Out->os(); - // Copy allocatable part of the input. - OS << InputFile->getData().substr(0, FirstNonAllocatableOffset); - + if (!opts::Rewrite) { + // Copy allocatable part of the input. + OS << InputFile->getData().substr(0, FirstNonAllocatableOffset); + } + // We obtain an asm-specific writer so that we can emit nops in an + // architecture-specific way at the end of the function. + std::unique_ptr MAB( + BC->TheTarget->createMCAsmBackend(*BC->STI, *BC->MRI, MCTargetOptions())); auto Streamer = BC->createStreamer(OS); // Make sure output stream has enough reserved space, otherwise // pwrite() will fail. - uint64_t Offset = OS.seek(getFileOffsetForAddress(NextAvailableAddress)); + uint64_t Offset = + OS.seek(getOutputFileOffsetForAddress(NextAvailableAddress)); (void)Offset; - assert(Offset == getFileOffsetForAddress(NextAvailableAddress) && + assert(Offset == getOutputFileOffsetForAddress(NextAvailableAddress) && "error resizing output file"); // Overwrite functions with fixed output address. This is mostly used by @@ -5264,7 +5577,7 @@ } } - if (BC->HasRelocations && opts::TrapOldCode) { + if (BC->HasRelocations && opts::TrapOldCode && !opts::Rewrite) { uint64_t SavedPos = OS.tell(); // Overwrite function body to make sure we never execute these instructions. for (auto &BFI : BC->getBinaryFunctions()) { @@ -5278,16 +5591,24 @@ OS.seek(SavedPos); } + auto ShouldWrite = [](BinarySection &Section) { + return Section.isFinalized() && Section.getOutputData() && + !Section.isVirtual() && + (Section.getData() != Section.getOutputData() || + Section.getAddress() != Section.getOutputAddress() || + opts::Rewrite); + }; // Write all allocatable sections - reloc-mode text is written here as well for (BinarySection &Section : BC->allocatableSections()) { - if (!Section.isFinalized() || !Section.getOutputData()) + if (!ShouldWrite(Section)) continue; if (opts::Verbosity >= 1) outs() << "BOLT: writing new section " << Section.getName() << "\n data at 0x" << Twine::utohexstr(Section.getAllocAddress()) - << "\n of size " << Section.getOutputSize() << "\n at offset " - << Section.getOutputFileOffset() << '\n'; + << "\n of size 0x" << Twine::utohexstr(Section.getOutputSize()) + << "\n at offset 0x" + << Twine::utohexstr(Section.getOutputFileOffset()) << '\n'; OS.pwrite(reinterpret_cast(Section.getOutputData()), Section.getOutputSize(), Section.getOutputFileOffset()); } @@ -5306,8 +5627,8 @@ if (opts::EnableBAT) addBATSection(); - // Patch program header table. - patchELFPHDRTable(); + // Write program header table. + writeELFPHDRTable(); // Finalize memory image of section string table. finalizeSectionStringTable(); @@ -5326,7 +5647,6 @@ if (BC->HasRelocations) { patchELFAllocatableRelaSections(); patchELFAllocatableRelrSection(); - patchELFGOT(); } // Patch dynamic section/segment. @@ -5345,20 +5665,67 @@ check_error(EC, "cannot set permissions of output file"); } +// this function is needed to estimate .eh_frame_hdr size when assigning +// addresses to leave enough space for entries, but the actual content of +// .eh_frame_hdr can only we written later after linking +void RewriteInstance::mapEhFrameAndHeader( + BOLTLinker::SectionMapper AssignAddress) { + + auto getEHFrameHeaderEntriesCount = [this](BinarySection *EHFrameSection) { + DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true, + EHFrameSection->getOutputAddress()); + Error E = NewEHFrame.parse(DWARFDataExtractor( + EHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(), + BC->AsmInfo->getCodePointerSize())); + check_error(std::move(E), "failed to parse EH frame"); + const uint64_t NumEntries = + NewEHFrame.entries().end() - NewEHFrame.entries().begin(); + return NumEntries; + }; + + BinarySection *NewEHFrameSection = getSection(getEHFrameSectionName()); + assert(NewEHFrameSection && NewEHFrameSection->hasValidSectionID()); + + mapSection(*NewEHFrameSection); + AssignAddress(*NewEHFrameSection, NewEHFrameSection->getOutputAddress()); + uint64_t NumEntries = getEHFrameHeaderEntriesCount(NewEHFrameSection); + + if (BinarySection *RelocatedEHFrameSection = + getSection(".relocated" + getEHFrameSectionName())) { + NumEntries += getEHFrameHeaderEntriesCount(RelocatedEHFrameSection); + mapSection(*RelocatedEHFrameSection); + AssignAddress(*RelocatedEHFrameSection, + RelocatedEHFrameSection->getOutputAddress()); + } + + const uint64_t Size = NumEntries * 8 + 12; + + const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true, + /*IsText=*/false, + /*IsAllocatable=*/true); + BinarySection &EHFrameHdr = BC->registerOrUpdateSection( + getEHFrameHeaderSectionName(), ELF::SHT_PROGBITS, Flags, nullptr, Size, + /*Alignment=*/1); + + mapSection(EHFrameHdr); +} + void RewriteInstance::writeEHFrameHeader() { - BinarySection *NewEHFrameSection = - getSection(getNewSecPrefix() + getEHFrameSectionName()); + BinarySection *NewEHFrameSection = getSection(getEHFrameSectionName()); // No need to update the header if no new .eh_frame was created. - if (!NewEHFrameSection) + if (!NewEHFrameSection || !NewEHFrameSection->hasValidSectionID()) { return; + } DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true, NewEHFrameSection->getOutputAddress()); + Error E = NewEHFrame.parse(DWARFDataExtractor( NewEHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(), BC->AsmInfo->getCodePointerSize())); check_error(std::move(E), "failed to parse EH frame"); + LLVM_DEBUG(dbgs() << "BOLT: writing a new .eh_frame_hdr\n"); uint64_t RelocatedEHFrameAddress = 0; StringRef RelocatedEHFrameContents; @@ -5375,36 +5742,15 @@ BC->AsmInfo->getCodePointerSize())); check_error(std::move(Er), "failed to parse EH frame"); - LLVM_DEBUG(dbgs() << "BOLT: writing a new .eh_frame_hdr\n"); - - NextAvailableAddress = - appendPadding(Out->os(), NextAvailableAddress, EHFrameHdrAlign); - - const uint64_t EHFrameHdrOutputAddress = NextAvailableAddress; - const uint64_t EHFrameHdrFileOffset = - getFileOffsetForAddress(NextAvailableAddress); - + auto EHFrameHdr = BC->getUniqueSectionByName(getEHFrameHeaderSectionName()); + assert(EHFrameHdr); std::vector NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader( - RelocatedEHFrame, NewEHFrame, EHFrameHdrOutputAddress, FailedAddresses); - - assert(Out->os().tell() == EHFrameHdrFileOffset && "offset mismatch"); - Out->os().write(NewEHFrameHdr.data(), NewEHFrameHdr.size()); + RelocatedEHFrame, NewEHFrame, EHFrameHdr->getOutputAddress(), + FailedAddresses); - const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true, - /*IsText=*/false, - /*IsAllocatable=*/true); - BinarySection *OldEHFrameHdrSection = getSection(".eh_frame_hdr"); - if (OldEHFrameHdrSection) - OldEHFrameHdrSection->setOutputName(getOrgSecPrefix() + ".eh_frame_hdr"); - - BinarySection &EHFrameHdrSec = BC->registerOrUpdateSection( - getNewSecPrefix() + ".eh_frame_hdr", ELF::SHT_PROGBITS, Flags, nullptr, - NewEHFrameHdr.size(), /*Alignment=*/1); - EHFrameHdrSec.setOutputFileOffset(EHFrameHdrFileOffset); - EHFrameHdrSec.setOutputAddress(EHFrameHdrOutputAddress); - EHFrameHdrSec.setOutputName(".eh_frame_hdr"); - - NextAvailableAddress += EHFrameHdrSec.getOutputSize(); + assert(NewEHFrameHdr.size() <= EHFrameHdr->getOutputSize()); + Out->os().pwrite(NewEHFrameHdr.data(), NewEHFrameHdr.size(), + EHFrameHdr->getOutputFileOffset()); // Merge new .eh_frame with the relocated original so that gdb can locate all // FDEs. @@ -5413,13 +5759,12 @@ RelocatedEHFrameSection->getOutputAddress() + RelocatedEHFrameSection->getOutputSize() - NewEHFrameSection->getOutputAddress(); - NewEHFrameSection->updateContents(NewEHFrameSection->getOutputData(), - NewEHFrameSectionSize); + NewEHFrameSection->updateContents(nullptr, NewEHFrameSectionSize); BC->deregisterSection(*RelocatedEHFrameSection); } - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of .eh_frame after merge is " - << NewEHFrameSection->getOutputSize() << '\n'); + LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of new .eh_frame is " + << EHFrameSection->getOutputSize() << '\n'); } uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) { @@ -5435,22 +5780,32 @@ return BD->getAddress(); } -uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const { - // Check if it's possibly part of the new segment. - if (Address >= NewTextSegmentAddress) - return Address - NewTextSegmentAddress + NewTextSegmentOffset; - +uint64_t +RewriteInstance::getFileOffsetForAddress(const uint64_t Address) const { // Find an existing segment that matches the address. const auto SegmentInfoI = BC->SegmentMapInfo.upper_bound(Address); if (SegmentInfoI == BC->SegmentMapInfo.begin()) return 0; - const SegmentInfo &SegmentInfo = std::prev(SegmentInfoI)->second; - if (Address < SegmentInfo.Address || - Address >= SegmentInfo.Address + SegmentInfo.FileSize) + const ProgramHeader &SegmentInfo = std::prev(SegmentInfoI)->second; + if (Address < SegmentInfo.p_vaddr || + Address >= SegmentInfo.p_vaddr + SegmentInfo.p_filesz) return 0; - return SegmentInfo.FileOffset + Address - SegmentInfo.Address; + return SegmentInfo.p_offset + Address - SegmentInfo.p_vaddr; +} + +uint64_t +RewriteInstance::getOutputFileOffsetForAddress(const uint64_t Address) const { + auto LB = BC->OutputAddressToOffsetMap.lower_bound(Address); + if (LB == BC->OutputAddressToOffsetMap.end() || LB->first > Address) { + assert(LB != BC->OutputAddressToOffsetMap.begin()); + --LB; + } + const uint64_t SegmentStartAddress = LB->first; + const uint64_t SegmentStartOffset = LB->second; + const uint64_t OffsetInSegment = Address - SegmentStartAddress; + return SegmentStartOffset + OffsetInSegment; } bool RewriteInstance::willOverwriteSection(StringRef SectionName) { Index: bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp =================================================================== --- bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -20,6 +20,7 @@ #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" +#include #define DEBUG_TYPE "mcplus" @@ -97,7 +98,7 @@ Inst.getOpcode() == AArch64::MOVZWi); } - bool isADD(const MCInst &Inst) const { + bool isADD(const MCInst &Inst) const override { return (Inst.getOpcode() == AArch64::ADDSWri || Inst.getOpcode() == AArch64::ADDSWrr || Inst.getOpcode() == AArch64::ADDSWrs || @@ -688,11 +689,10 @@ /// add x16, x16, #0xbe0 /// br x17 /// - uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin, - InstructionIterator End, + uint64_t analyzePLTEntry(InstructionIterator Begin, InstructionIterator End, uint64_t BeginPC) const override { // Check branch instruction - MCInst *Branch = &Instruction; + MCInst *Branch = &*(std::prev(End)); assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode"); DenseMap> UDChain = @@ -1042,17 +1042,21 @@ return 3; } - bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override { - if (!isADRP(Adrp) || !isAddXri(Add)) + bool matchAdrpPair(const MCInst &Adrp, + const MCInst &AddOrLdr) const override { + if (!isADRP(Adrp)) + return false; + if (!isAddXri(AddOrLdr) && !isLDRX(AddOrLdr)) return false; - assert(Adrp.getOperand(0).isReg() && "Unexpected operand in ADRP instruction"); MCPhysReg AdrpReg = Adrp.getOperand(0).getReg(); - assert(Add.getOperand(1).isReg() && - "Unexpected operand in ADDXri instruction"); - MCPhysReg AddReg = Add.getOperand(1).getReg(); - return AdrpReg == AddReg; + assert(AddOrLdr.getOperand(1).isReg() && + "Unexpected operand in ADDXri/LDRX instruction"); + MCPhysReg AddReg = AddOrLdr.getOperand(1).getReg(); + if (AdrpReg != AddReg) + return false; + return true; } bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, @@ -1075,6 +1079,29 @@ return true; } + bool patchPLTInstructions(InstructionIterator Begin, InstructionIterator End, + const MCSymbol *Target, MCContext *Ctx, + const MCSymbol *BFSymbol) const override { + int64_t Val; + int Count = 0; + for (auto I = Begin; I != End; ++I) { + if (isADRP(*I)) + Count += replaceImmWithSymbolRef(*I, Target, 0, Ctx, Val, + ELF::R_AARCH64_ADR_PREL_PG_HI21); + else if (isADD(*I) || isLoad(*I)) + Count += replaceImmWithSymbolRef(*I, Target, 0, Ctx, Val, + ELF::R_AARCH64_LDST64_ABS_LO12_NC); + } + return Count == 3; + } + + bool relaxLdrToAdd(MCInst &Inst) const override { + assert(Inst.getNumOperands() == 3); + assert(Inst.getOpcode() == AArch64::LDRXui); + Inst.setOpcode(AArch64::ADDXri); + Inst.addOperand(MCOperand::createImm(0)); // zero shift + return true; + } bool createUncondBranch(MCInst &Inst, const MCSymbol *TBB, MCContext *Ctx) const override { Index: bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp =================================================================== --- bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -63,6 +63,12 @@ return isNop(Inst) || isCNop(Inst); } + bool isAuipc(const MCInst &Inst) const { + return Inst.getOpcode() == RISCV::AUIPC; + } + + bool isLD(const MCInst &Inst) const { return Inst.getOpcode() == RISCV::LD; } + bool hasPCRelOperand(const MCInst &Inst) const override { switch (Inst.getOpcode()) { default: @@ -281,12 +287,15 @@ return true; } - uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin, - InstructionIterator End, + uint64_t analyzePLTEntry(InstructionIterator Begin, InstructionIterator End, uint64_t BeginPC) const override { - auto I = Begin; - assert(I != End); + assert(Begin != End && std::next(Begin) != End); + + if (std::next(Begin)->getOpcode() == RISCV::SUB) + return analyzePLTHeader(Begin, End, BeginPC); + + auto I = Begin; auto &AUIPC = *I++; assert(AUIPC.getOpcode() == RISCV::AUIPC); assert(AUIPC.getOperand(0).getReg() == RISCV::X28); @@ -316,6 +325,66 @@ return BeginPC + AUIPCOffset + LDOffset; } + uint64_t analyzePLTHeader(InstructionIterator Begin, InstructionIterator End, + uint64_t BeginPC) const { + auto I = Begin; + + assert(I != End); + auto &AUIPC = *I++; + assert(AUIPC.getOpcode() == RISCV::AUIPC); + assert(AUIPC.getOperand(0).getReg() == RISCV::X7); + + assert(I != End); + auto &SUB = *I++; + assert(SUB.getOpcode() == RISCV::SUB); + assert(SUB.getOperand(0).getReg() == RISCV::X6); + assert(SUB.getOperand(1).getReg() == RISCV::X6); + assert(SUB.getOperand(2).getReg() == RISCV::X28); + + assert(I != End); + auto &LD = *I++; + assert(LD.getOpcode() == RISCV::LD); + assert(LD.getOperand(0).getReg() == RISCV::X28); + assert(LD.getOperand(1).getReg() == RISCV::X7); + + assert(I != End); + auto &ADDI1 = *I++; + assert(ADDI1.getOpcode() == RISCV::ADDI); + assert(ADDI1.getOperand(0).getReg() == RISCV::X6); + assert(ADDI1.getOperand(1).getReg() == RISCV::X6); + + assert(I != End); + auto &ADDI2 = *I++; + assert(ADDI2.getOpcode() == RISCV::ADDI); + assert(ADDI2.getOperand(0).getReg() == RISCV::X5); + assert(ADDI2.getOperand(1).getReg() == RISCV::X7); + + assert(I != End); + auto &SRLI = *I++; + assert(SRLI.getOpcode() == RISCV::SRLI); + assert(SRLI.getOperand(0).getReg() == RISCV::X6); + assert(SRLI.getOperand(1).getReg() == RISCV::X6); + + assert(I != End); + auto &LD2 = *I++; + assert(LD2.getOpcode() == RISCV::LD); + assert(LD2.getOperand(0).getReg() == RISCV::X5); + assert(LD2.getOperand(1).getReg() == RISCV::X5); + assert(LD2.getOperand(2).getImm() == 8); + + assert(I != End); + auto &JR = *I++; + (void)JR; + assert(JR.getOpcode() == RISCV::JALR); + assert(JR.getOperand(0).getReg() == RISCV::X0); + assert(JR.getOperand(1).getReg() == RISCV::X28); + assert(I == End); + + auto AUIPCOffset = AUIPC.getOperand(1).getImm() << 12; + auto LDOffset = LD.getOperand(2).getImm(); + return BeginPC + AUIPCOffset + LDOffset; + } + bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol, int64_t Addend, MCContext *Ctx, int64_t &Value, uint64_t RelType) const override { @@ -383,6 +452,24 @@ } } + bool patchPLTInstructions(InstructionIterator Begin, InstructionIterator End, + const MCSymbol *Target, MCContext *Ctx, + const MCSymbol *BFSymbol) const override { + int64_t Val; + int Count = 0; + + for (auto I = Begin; I != End; ++I) { + if (isAuipc(*I)) + Count += replaceImmWithSymbolRef(*I, Target, 0, Ctx, Val, + ELF::R_RISCV_PCREL_HI20); + else if (isLD(*I) && (I->getOperand(1).getReg() == RISCV::X7 || + I->getOperand(1).getReg() == RISCV::X28)) + Count += replaceImmWithSymbolRef(*I, BFSymbol, 0, Ctx, Val, + ELF::R_RISCV_PCREL_LO12_I); + } + return Count == 2; + } + bool isRISCVCall(const MCInst &First, const MCInst &Second) const override { if (!isCallAuipc(First)) return false; Index: bolt/lib/Target/X86/X86MCPlusBuilder.cpp =================================================================== --- bolt/lib/Target/X86/X86MCPlusBuilder.cpp +++ bolt/lib/Target/X86/X86MCPlusBuilder.cpp @@ -308,6 +308,10 @@ return Inst.getOpcode() == X86::LEAVE || Inst.getOpcode() == X86::LEAVE64; } + virtual bool isCall64m(const MCInst &Inst) const override { + return Inst.getOpcode() == X86::CALL64m; + } + bool isMoveMem2Reg(const MCInst &Inst) const override { switch (Inst.getOpcode()) { case X86::MOV16rm: Index: bolt/lib/Utils/CommandLineOpts.cpp =================================================================== --- bolt/lib/Utils/CommandLineOpts.cpp +++ bolt/lib/Utils/CommandLineOpts.cpp @@ -42,7 +42,7 @@ cl::opt AlignText("align-text", cl::desc("alignment of .text section"), cl::Hidden, - cl::cat(BoltCategory)); + cl::init(64), cl::cat(BoltCategory)); cl::opt AlignFunctions( "align-functions", @@ -176,10 +176,16 @@ cl::desc("print time spent in each optimization"), cl::cat(BoltOptCategory)); -cl::opt UseOldText( - "use-old-text", - cl::desc("re-use space in old .text if possible (relocation mode)"), - cl::cat(BoltCategory)); +cl::opt UseOldText("use-old-text", + cl::desc("re-use space in old .text. Deprecated - use " + "-rewrite instead"), + cl::cat(BoltCategory)); + +cl::opt + Rewrite("rewrite", + cl::desc("Rewrite entire binary - relocate all sections and " + "create new program header table. Requires relocations"), + cl::cat(BoltCategory)); cl::opt UpdateDebugSections( "update-debug-sections", @@ -195,7 +201,7 @@ if (opts::AggregateOnly) return false; - if (UseOldText || StrictMode) + if (Rewrite || StrictMode) return true; return false; Index: bolt/test/AArch64/Inputs/array_end.c =================================================================== --- /dev/null +++ bolt/test/AArch64/Inputs/array_end.c @@ -0,0 +1,17 @@ + +__attribute__((destructor)) void destr() {} + +__attribute__((noinline)) void callFini() { + extern void (*__fini_array_start[])(); + extern void (*__fini_array_end[])(); + unsigned long Count = __fini_array_end - __fini_array_start; + for (unsigned long I = 0; I < Count; ++I) + (*__fini_array_start[I])(); +} + +void _start() { + callFini(); + __asm__("mov x0, #0\n" + "mov w8, #93\n" + "svc #0\n"); +} Index: bolt/test/AArch64/Inputs/array_end.lld_script =================================================================== --- bolt/test/AArch64/Inputs/array_end.lld_script +++ bolt/test/AArch64/Inputs/array_end.lld_script @@ -1,12 +1,28 @@ + +PHDRS +{ + text PT_LOAD FILEHDR PHDRS; +} + SECTIONS { + . = SIZEOF_HEADERS; .fini_array : { PROVIDE_HIDDEN (__fini_array_start = .); - KEEP (*(SORT_BY_INIT_PRIORITY(.fini_array.*) SORT_BY_INIT_PRIORITY(.dtors.*))) + KEEP (*(SORT_BY_INIT_PRIORITY(.fini_array) SORT_BY_INIT_PRIORITY(.dtors.*))) PROVIDE_HIDDEN (__fini_array_end = .); } . = . + 128; .text : { *(.text) } + .text : { *(.text) } :text + /DISCARD/ : { + *(.eh_frame); + *(.comment); + *(.got); + *(.got.plt); + *(.data); + *(.bss) + } } Index: bolt/test/AArch64/Inputs/dw_cfa_gnu_window_save.yaml =================================================================== --- bolt/test/AArch64/Inputs/dw_cfa_gnu_window_save.yaml +++ bolt/test/AArch64/Inputs/dw_cfa_gnu_window_save.yaml @@ -6,13 +6,167 @@ Machine: EM_AARCH64 Entry: 0x4100C0 ProgramHeaders: + - Type: PT_PHDR + Flags: [ PF_R ] + VAddr: 0x400040 + Align: 0x8 + - Type: PT_INTERP + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .interp + VAddr: 0x4002A8 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .interp + LastSec: .rela.plt + VAddr: 0x400000 + Offset: 0x0 + Align: 0x10000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] FirstSec: .init LastSec: .fini VAddr: 0x410000 Align: 0x10000 + - Type: PT_LOAD + Flags: [ PF_R ] + FirstSec: .rodata + LastSec: .eh_frame + VAddr: 0x420000 + Align: 0x10000 + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] + FirstSec: .init_array + LastSec: .bss + VAddr: 0x43FDC8 + Align: 0x10000 + - Type: PT_DYNAMIC + Flags: [ PF_W, PF_R ] + FirstSec: .dynamic + LastSec: .dynamic + VAddr: 0x43FDD8 + Align: 0x8 + - Type: PT_NOTE + Flags: [ PF_R ] + FirstSec: .note.gnu.build-id + LastSec: .note.ABI-tag + VAddr: 0x4002C4 + Align: 0x4 + - Type: PT_GNU_EH_FRAME + Flags: [ PF_R ] + FirstSec: .eh_frame_hdr + LastSec: .eh_frame_hdr + VAddr: 0x420020 + Align: 0x4 + - Type: PT_GNU_STACK + Flags: [ PF_W, PF_R ] + Align: 0x10 + - Type: PT_GNU_RELRO + Flags: [ PF_R ] + FirstSec: .init_array + LastSec: .got + VAddr: 0x43FDC8 Sections: + - Name: .interp + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x4002A8 + AddressAlign: 0x1 + Content: 2F6C69622F6C642D6C696E75782D616172636836342E736F2E3100 + - Name: .note.gnu.build-id + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002C4 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: EFF4CEC279C936EA2FB1125B9333F1EBB8FB169C + Type: NT_PRPSINFO + - Name: .note.ABI-tag + Type: SHT_NOTE + Flags: [ SHF_ALLOC ] + Address: 0x4002E8 + AddressAlign: 0x4 + Notes: + - Name: GNU + Desc: '00000000030000000700000000000000' + Type: NT_VERSION + - Name: .gnu.hash + Type: SHT_GNU_HASH + Flags: [ SHF_ALLOC ] + Address: 0x400308 + Link: .dynsym + AddressAlign: 0x8 + Header: + SymNdx: 0x1 + Shift2: 0x0 + BloomFilter: [ 0x0 ] + HashBuckets: [ 0x0 ] + HashValues: [ ] + - Name: .dynsym + Type: SHT_DYNSYM + Flags: [ SHF_ALLOC ] + Address: 0x400328 + Link: .dynstr + AddressAlign: 0x8 + - Name: .dynstr + Type: SHT_STRTAB + Flags: [ SHF_ALLOC ] + Address: 0x4003B8 + AddressAlign: 0x1 + - Name: .gnu.version + Type: SHT_GNU_versym + Flags: [ SHF_ALLOC ] + Address: 0x40045E + Offset: 0x45E + Link: .dynsym + AddressAlign: 0x2 + Entries: [ 0, 2, 3, 1, 1, 1 ] + - Name: .gnu.version_r + Type: SHT_GNU_verneed + Flags: [ SHF_ALLOC ] + Address: 0x400470 + Link: .dynstr + AddressAlign: 0x8 + Dependencies: + - Version: 1 + File: libc.so.6 + Entries: + - Name: GLIBC_2.17 + Hash: 110530967 + Flags: 0 + Other: 3 + - Name: GLIBC_2.34 + Hash: 110530996 + Flags: 0 + Other: 2 + - Name: .rela.dyn + Type: SHT_RELA + Flags: [ SHF_ALLOC ] + Address: 0x4004A0 + Link: .dynsym + AddressAlign: 0x8 + Relocations: + - Offset: 0x43FFE0 + Symbol: __gmon_start__ + Type: R_AARCH64_GLOB_DAT + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Address: 0x4004B8 + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x440000 + Symbol: __libc_start_main + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x440008 + Symbol: abort + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x440010 + Symbol: __gmon_start__ + Type: R_AARCH64_JUMP_SLOT - Name: .init Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] @@ -38,25 +192,751 @@ Address: 0x4101CC AddressAlign: 0x4 Content: 3F2303D5FD7BBFA9FD030091FD7BC1A8BF2303D5C0035FD6 + - Name: .rodata + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x420000 + AddressAlign: 0x8 + Offset: 0x20000 + Content: '0100020000000000000000000000000000000000000000000000000000000000' + - Name: .eh_frame_hdr + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x420020 + AddressAlign: 0x4 + Content: 011B033B44000000070000006000FFFFF0000000A000FFFF5C000000E000FFFF700000000001FFFF880000003001FFFF9C0000006C01FFFFB0000000A401FFFFD8000000 - Name: .eh_frame Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x420068 AddressAlign: 0x8 Content: 1000000000000000017A520004781E011B0C1F0010000000180000003C00FFFF3C0000000041071E140000002C0000006800FFFF08000000000000000000000010000000440000007000FFFF300000000000000010000000580000008C00FFFF3C00000000000000240000006C000000B400FFFF3800000000412D410E209D049E0342930248DEDDD30E00412D0000001400000094000000C400FFFF08000000000000000000000010000000AC00000068FFFEFF080000000000000000000000 + - Name: .init_array + Type: SHT_INIT_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x43FDC8 + AddressAlign: 0x8 + EntSize: 0x8 + Offset: 0x2FDC8 + Content: C401410000000000 + - Name: .fini_array + Type: SHT_FINI_ARRAY + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x43FDD0 + AddressAlign: 0x8 + EntSize: 0x8 + Content: 8C01410000000000 + - Name: .dynamic + Type: SHT_DYNAMIC + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x43FDD8 + Link: .dynstr + AddressAlign: 0x8 + Entries: + - Tag: DT_NEEDED + Value: 0x5E + - Tag: DT_NEEDED + Value: 0x6D + - Tag: DT_NEEDED + Value: 0x77 + - Tag: DT_NEEDED + Value: 0x85 + - Tag: DT_INIT + Value: 0x410000 + - Tag: DT_FINI + Value: 0x4101CC + - Tag: DT_INIT_ARRAY + Value: 0x43FDC8 + - Tag: DT_INIT_ARRAYSZ + Value: 0x8 + - Tag: DT_FINI_ARRAY + Value: 0x43FDD0 + - Tag: DT_FINI_ARRAYSZ + Value: 0x8 + - Tag: DT_GNU_HASH + Value: 0x400308 + - Tag: DT_STRTAB + Value: 0x4003B8 + - Tag: DT_SYMTAB + Value: 0x400328 + - Tag: DT_STRSZ + Value: 0xA5 + - Tag: DT_SYMENT + Value: 0x18 + - Tag: DT_DEBUG + Value: 0x0 + - Tag: DT_PLTGOT + Value: 0x43FFE8 + - Tag: DT_PLTRELSZ + Value: 0x48 + - Tag: DT_PLTREL + Value: 0x7 + - Tag: DT_JMPREL + Value: 0x4004B8 + - Tag: DT_RELA + Value: 0x4004A0 + - Tag: DT_RELASZ + Value: 0x18 + - Tag: DT_RELAENT + Value: 0x18 + - Tag: DT_VERNEED + Value: 0x400470 + - Tag: DT_VERNEEDNUM + Value: 0x1 + - Tag: DT_VERSYM + Value: 0x40045E + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Name: .got + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x43FFD8 + AddressAlign: 0x8 + EntSize: 0x8 + Content: D8FD4300000000000000000000000000 + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x43FFE8 + AddressAlign: 0x8 + EntSize: 0x8 + Content: '000000000000000000000000000000000000000000000000200041000000000020004100000000002000410000000000' + - Name: .data + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x440018 + AddressAlign: 0x1 + Content: '00000000' + - Name: .tm_clone_table + Type: SHT_PROGBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x440020 + AddressAlign: 0x8 + - Name: .bss + Type: SHT_NOBITS + Flags: [ SHF_WRITE, SHF_ALLOC ] + Address: 0x440020 + AddressAlign: 0x1 + Size: 0x8 + - Name: .comment + Type: SHT_PROGBITS + Flags: [ SHF_MERGE, SHF_STRINGS ] + AddressAlign: 0x1 + EntSize: 0x1 + Content: 4743433A2028474E55292031312E332E312032303232303432312028526564204861742031312E332E312D3229004743433A2028474E55292031312E332E312032303232313132312028526564204861742031312E332E312D342900 + - Name: .rela.init + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init + Relocations: + - Offset: 0x41000C + Symbol: .text + Type: R_AARCH64_CALL26 + Addend: 136 - Name: .rela.text Type: SHT_RELA Flags: [ SHF_INFO_LINK ] Link: .symtab AddressAlign: 0x8 Info: .text + Relocations: + - Offset: 0x4100DC + Symbol: .text + Type: R_AARCH64_ADR_PREL_PG_HI21 + Addend: 116 + - Offset: 0x4100E0 + Symbol: .text + Type: R_AARCH64_ADD_ABS_LO12_NC + Addend: 116 + - Offset: 0x4100EC + Symbol: '__libc_start_main@GLIBC_2.34' + Type: R_AARCH64_CALL26 + - Offset: 0x4100F0 + Symbol: 'abort@GLIBC_2.17' + Type: R_AARCH64_CALL26 + - Offset: 0x4100F8 + Symbol: main + Type: R_AARCH64_JUMP26 + - Offset: 0x410108 + Symbol: __gmon_start__ + Type: R_AARCH64_ADR_GOT_PAGE + - Offset: 0x41010C + Symbol: __gmon_start__ + Type: R_AARCH64_LD64_GOT_LO12_NC + - Offset: 0x410114 + Symbol: __gmon_start__ + Type: R_AARCH64_JUMP26 + - Offset: 0x410120 + Symbol: .tm_clone_table + Type: R_AARCH64_ADR_PREL_PG_HI21 + - Offset: 0x410124 + Symbol: __TMC_END__ + Type: R_AARCH64_ADR_PREL_PG_HI21 + - Offset: 0x410128 + Symbol: .tm_clone_table + Type: R_AARCH64_ADD_ABS_LO12_NC + - Offset: 0x41012C + Symbol: __TMC_END__ + Type: R_AARCH64_ADD_ABS_LO12_NC + - Offset: 0x410138 + Symbol: .rodata + Type: R_AARCH64_ADR_PREL_PG_HI21 + Addend: 16 + - Offset: 0x41013C + Symbol: .rodata + Type: R_AARCH64_LDST64_ABS_LO12_NC + Addend: 16 + - Offset: 0x410150 + Symbol: .tm_clone_table + Type: R_AARCH64_ADR_PREL_PG_HI21 + - Offset: 0x410154 + Symbol: __TMC_END__ + Type: R_AARCH64_ADR_PREL_PG_HI21 + - Offset: 0x410158 + Symbol: .tm_clone_table + Type: R_AARCH64_ADD_ABS_LO12_NC + - Offset: 0x41015C + Symbol: __TMC_END__ + Type: R_AARCH64_ADD_ABS_LO12_NC + - Offset: 0x410174 + Symbol: .rodata + Type: R_AARCH64_ADR_PREL_PG_HI21 + Addend: 24 + - Offset: 0x410178 + Symbol: .rodata + Type: R_AARCH64_LDST64_ABS_LO12_NC + Addend: 24 + - Offset: 0x41019C + Symbol: .bss + Type: R_AARCH64_ADR_PREL_PG_HI21 + - Offset: 0x4101A0 + Symbol: .bss + Type: R_AARCH64_LDST8_ABS_LO12_NC + - Offset: 0x4101B0 + Symbol: .bss + Type: R_AARCH64_LDST8_ABS_LO12_NC + - Name: .rela.rodata + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .rodata + Relocations: + - Offset: 0x420010 + Symbol: _ITM_deregisterTMCloneTable + Type: R_AARCH64_ABS64 + - Offset: 0x420018 + Symbol: _ITM_registerTMCloneTable + Type: R_AARCH64_ABS64 + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x420084 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 64 + - Offset: 0x420098 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 128 + - Offset: 0x4200B0 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 160 + - Offset: 0x4200C4 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 208 + - Offset: 0x4200D8 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 268 + - Offset: 0x420100 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 324 + - Offset: 0x420118 + Symbol: .text + Type: R_AARCH64_PREL32 + - Name: .rela.init_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .init_array + Relocations: + - Offset: 0x43FDC8 + Symbol: .text + Type: R_AARCH64_ABS64 + Addend: 324 + - Name: .rela.fini_array + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .fini_array + Relocations: + - Offset: 0x43FDD0 + Symbol: .text + Type: R_AARCH64_ABS64 + Addend: 268 + - Type: SectionHeaderTable + Sections: + - Name: .interp + - Name: .note.gnu.build-id + - Name: .note.ABI-tag + - Name: .gnu.hash + - Name: .dynsym + - Name: .dynstr + - Name: .gnu.version + - Name: .gnu.version_r + - Name: .rela.dyn + - Name: .rela.plt + - Name: .init + - Name: .rela.init + - Name: .plt + - Name: .text + - Name: .rela.text + - Name: .fini + - Name: .rodata + - Name: .rela.rodata + - Name: .eh_frame_hdr + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .init_array + - Name: .rela.init_array + - Name: .fini_array + - Name: .rela.fini_array + - Name: .dynamic + - Name: .got + - Name: .got.plt + - Name: .data + - Name: .tm_clone_table + - Name: .bss + - Name: .comment + - Name: .symtab + - Name: .strtab + - Name: .shstrtab Symbols: - Name: .text Type: STT_SECTION Section: .text Value: 0x410080 + - Name: .rodata + Type: STT_SECTION + Section: .rodata + Value: 0x420000 + - Name: .eh_frame_hdr + Type: STT_SECTION + Section: .eh_frame_hdr + Value: 0x420020 + - Name: .eh_frame + Type: STT_SECTION + Section: .eh_frame + Value: 0x420068 + - Name: .data + Type: STT_SECTION + Section: .data + Value: 0x440018 + - Name: .tm_clone_table + Type: STT_SECTION + Section: .tm_clone_table + Value: 0x440020 + - Name: .bss + Type: STT_SECTION + Section: .bss + Value: 0x440020 + - Name: crt1.o + Type: STT_FILE + Index: SHN_ABS + - Name: '$x' + Section: .text + Value: 0x4100C0 + - Name: __wrap_main + Section: .text + Value: 0x4100F4 + - Name: '$d' + Section: .eh_frame + Value: 0x42007C + - Name: .annobin_abi_note.c + Section: .text + Value: 0x4100FC + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c_end.hot + Section: .text + Value: 0x410088 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c_end.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c_end.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_abi_note.c_end.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: '$d (6)' + Section: .note.ABI-tag + Value: 0x4002E8 + - Name: __abi_tag + Type: STT_OBJECT + Section: .note.ABI-tag + Value: 0x4002E8 + Size: 0x20 + - Name: .annobin_init.c + Section: .text + Value: 0x4100FC + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c_end + Section: .text + Value: 0x4100FC + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c.hot + Section: .text + Value: 0x410088 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c_end.hot + Section: .text + Value: 0x410088 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c_end.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c_end.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_init.c_end.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: '$d (12)' + Section: .rodata + Value: 0x420000 + - Name: .annobin_static_reloc.c + Section: .text + Value: 0x410100 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c_end + Section: .text + Value: 0x4100FC + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c.hot + Section: .text + Value: 0x410088 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c_end.hot + Section: .text + Value: 0x410088 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c_end.unlikely + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c_end.startup + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin_static_reloc.c_end.exit + Section: .text + Value: 0x410080 + Other: [ STV_HIDDEN ] + - Name: .annobin__dl_relocate_static_pie.start + Section: .text + Value: 0x410100 + Other: [ STV_HIDDEN ] + - Name: .annobin__dl_relocate_static_pie.end + Section: .text + Value: 0x410108 + Other: [ STV_HIDDEN ] + - Name: '$x (1)' + Section: .text + Value: 0x410100 + - Name: '$d (18)' + Section: .eh_frame + Value: 0x420090 + - Name: crti.o + Type: STT_FILE + Index: SHN_ABS + - Name: '$x (2)' + Section: .text + Value: 0x410108 + - Name: call_weak_fn + Type: STT_FUNC + Section: .text + Value: 0x410108 + Size: 0x14 + - Name: '$x (3)' + Section: .init + Value: 0x410000 + - Name: '$x (4)' + Section: .fini + Value: 0x4101CC + - Name: crtn.o + Type: STT_FILE + Index: SHN_ABS + - Name: '$x (5)' + Section: .init + Value: 0x410010 + - Name: '$x (6)' + Section: .fini + Value: 0x4101D8 + - Name: a.cc + Type: STT_FILE + Index: SHN_ABS + - Name: '$x (7)' + Section: .text + Value: 0x410080 + - Name: '$d (19)' + Section: .eh_frame + Value: 0x420110 + - Name: crtstuff.c + Type: STT_FILE + Index: SHN_ABS + - Name: __TMC_LIST__ + Type: STT_OBJECT + Section: .tm_clone_table + Value: 0x440020 + - Name: '$x (8)' + Section: .text + Value: 0x410120 + - Name: deregister_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x410120 + - Name: register_tm_clones + Type: STT_FUNC + Section: .text + Value: 0x410150 + - Name: '$d (20)' + Section: .rodata + Value: 0x420008 + - Name: __do_global_dtors_aux + Type: STT_FUNC + Section: .text + Value: 0x41018C + - Name: completed.0 + Type: STT_OBJECT + Section: .bss + Value: 0x440020 + Size: 0x1 + - Name: '$d (21)' + Section: .fini_array + Value: 0x43FDD0 + - Name: __do_global_dtors_aux_fini_array_entry + Type: STT_OBJECT + Section: .fini_array + Value: 0x43FDD0 + - Name: frame_dummy + Type: STT_FUNC + Section: .text + Value: 0x4101C4 + - Name: '$d (22)' + Section: .init_array + Value: 0x43FDC8 + - Name: __frame_dummy_init_array_entry + Type: STT_OBJECT + Section: .init_array + Value: 0x43FDC8 + - Name: '$d (23)' + Section: .rodata + Value: 0x420010 + - Name: '$d (24)' + Section: .eh_frame + Value: 0x4200A8 + - Name: '$d (25)' + Section: .bss + Value: 0x440020 + - Name: 'crtstuff.c (1)' + Type: STT_FILE + Index: SHN_ABS + - Name: '$d (26)' + Section: .eh_frame + Value: 0x420124 + - Name: __FRAME_END__ + Type: STT_OBJECT + Section: .eh_frame + Value: 0x420124 + - Type: STT_FILE + Index: SHN_ABS + - Name: __GNU_EH_FRAME_HDR + Section: .eh_frame_hdr + Value: 0x420020 + - Name: _DYNAMIC + Type: STT_OBJECT + Section: .dynamic + Value: 0x43FDD8 + - Name: _GLOBAL_OFFSET_TABLE_ + Type: STT_OBJECT + Section: .got + Value: 0x43FFD8 + - Name: '$x (9)' + Section: .plt + Value: 0x410020 + - Name: _edata + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x440020 + - Name: data_start + Section: .data + Binding: STB_WEAK + Value: 0x440018 + - Name: _IO_stdin_used + Type: STT_OBJECT + Section: .rodata + Binding: STB_GLOBAL + Value: 0x420000 + Size: 0x4 + - Name: main + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x410080 + Size: 0x8 + - Name: __end__ + Section: .bss + Binding: STB_GLOBAL + Value: 0x440028 + - Name: __dso_handle + Type: STT_OBJECT + Section: .rodata + Binding: STB_GLOBAL + Value: 0x420008 + Other: [ STV_HIDDEN ] + - Name: _fini + Type: STT_FUNC + Section: .fini + Binding: STB_GLOBAL + Value: 0x4101CC + Other: [ STV_HIDDEN ] + - Name: '__libc_start_main@GLIBC_2.34' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _dl_relocate_static_pie + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x410100 + Size: 0x8 + Other: [ STV_HIDDEN ] + - Name: __bss_end__ + Section: .bss + Binding: STB_GLOBAL + Value: 0x440028 + - Name: _bss_end__ + Section: .bss + Binding: STB_GLOBAL + Value: 0x440028 + - Name: _start + Type: STT_FUNC + Section: .text + Binding: STB_GLOBAL + Value: 0x4100C0 + Size: 0x3C + - Name: _init + Type: STT_FUNC + Section: .init + Binding: STB_GLOBAL + Value: 0x410000 + Other: [ STV_HIDDEN ] + - Name: __TMC_END__ + Type: STT_OBJECT + Section: .tm_clone_table + Binding: STB_GLOBAL + Value: 0x440020 + Other: [ STV_HIDDEN ] + - Name: __bss_start__ + Section: .bss + Binding: STB_GLOBAL + Value: 0x440020 + - Name: __data_start + Section: .data + Binding: STB_GLOBAL + Value: 0x440018 + - Name: _end + Section: .bss + Binding: STB_GLOBAL + Value: 0x440028 + - Name: __bss_start + Section: .bss + Binding: STB_GLOBAL + Value: 0x440020 + - Name: 'abort@GLIBC_2.17' + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: _ITM_deregisterTMCloneTable + Binding: STB_WEAK + - Name: __gmon_start__ + Binding: STB_WEAK + - Name: _ITM_registerTMCloneTable + Binding: STB_WEAK +DynamicSymbols: + - Name: __libc_start_main + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: abort + Type: STT_FUNC + Binding: STB_GLOBAL - Name: _ITM_deregisterTMCloneTable Binding: STB_WEAK + - Name: __gmon_start__ + Binding: STB_WEAK - Name: _ITM_registerTMCloneTable Binding: STB_WEAK ... Index: bolt/test/AArch64/Inputs/go_dwarf.yaml =================================================================== --- bolt/test/AArch64/Inputs/go_dwarf.yaml +++ bolt/test/AArch64/Inputs/go_dwarf.yaml @@ -6,26 +6,32 @@ Machine: EM_AARCH64 Entry: 0x684 ProgramHeaders: - - Type: PT_PHDR - Flags: [ PF_R ] - VAddr: 0x200 - Align: 0x8 - - Type: PT_INTERP - Flags: [ PF_R ] - FirstSec: .plt - LastSec: .plt - VAddr: 0x238 - Type: PT_LOAD Flags: [ PF_X, PF_R ] FirstSec: .plt LastSec: .text Align: 0x10000 - VAddr: 0x5e8 + VAddr: 0x0 + Offset: 0x0 + - Type: PT_LOAD + Flags: [ PF_X, PF_R ] + VAddr: 0x10F40 + Offset: 0xF40 + FirstSec: .dynamic + LastSec: .dynstr + Align: 0x10000 + - Type: PT_DYNAMIC + Flags: [ PF_W, PF_R ] + FirstSec: .dynamic + LastSec: .dynamic + VAddr: 0x10F40 + Align: 0x8 Sections: - Name: .plt Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x5F0 + Offset: 0x5F0 AddressAlign: 0x10 EntSize: 0x10 Content: F07BBFA99000009011CA47F910423E9120021FD61F2003D51F2003D51F2003D59000009011CE47F910623E9120021FD69000009011D247F910823E9120021FD69000009011D647F910A23E9120021FD69000009011DA47F910C23E9120021FD69000009011DE47F910E23E9120021FD6 @@ -33,8 +39,50 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x660 + Offset: 0x660 AddressAlign: 0x8 Content: FD7BBFA9A1D6811200000090FD03009100C02091F7FFFF9700008052FD7BC1A8C0035FD61D0080D21E0080D2E50300AAE10340F9E2230091E60300918000009000F847F98300009063F447F98400009084E447F9DBFFFF97E2FFFF978000009000F047F9400000B4DAFFFF17C0035FD6800000B000400091810000B0214000913F0000EBC00000548100009021E847F9610000B4F00301AA00021FD6C0035FD6800000B000400091810000B021400091210000CB22FC7FD3410C818BFF0781EB21FC4193C00000548200009042FC47F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F9930000B060424039400100358000009000EC47F9800000B4800000B0000440F9A9FFFF97D8FFFF972000805260420039F30B40F9FD7BC2A8C0035FD6DEFFFF171F2003D5FD7BBCA9FD030091F35301A99400009094223691F55B02A995000090B5023691940215CBF603002AF76303A9F70301AAF80302AA83FFFF97FF0F94EB6001005494FE4393130080D2A37A73F8E20318AA73060091E10317AAE003162A60003FD69F0213EB21FFFF54F35341A9F55B42A9F76343A9FD7BC4A8C0035FD61F2003D5C0035FD6 + - Name: .dynamic + Address: 0x10F40 + Offset: 0xF40 + Type: SHT_DYNAMIC + Flags: [ SHF_WRITE, SHF_ALLOC ] + Entries: + - Tag: DT_PLTRELSZ + Value: 0x78 + - Tag: DT_JMPREL + Value: 0x10FB8 + - Tag: DT_NULL + Value: 0x0 + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_WRITE ] + Address: 0x10F80 + Offset: 0xF80 + AddressAlign: 0x8 + Size: 0x38 + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x10f98 + Symbol: pltSym1 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fa0 + Symbol: pltSym2 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fa8 + Symbol: pltSym3 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fb0 + Symbol: pltSym4 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fb8 + Symbol: pltSym5 + Type: R_AARCH64_JUMP_SLOT - Name: .debug_info Type: SHT_PROGBITS AddressAlign: 0x1 @@ -54,6 +102,22 @@ Binding: STB_GLOBAL Value: 0x660 Size: 0x24 +DynamicSymbols: + - Name: pltSym1 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym2 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym3 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym4 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym5 + Type: STT_FUNC + Binding: STB_GLOBAL DWARF: debug_str: - te.c Index: bolt/test/AArch64/Inputs/got-ld64-relaxation.yaml =================================================================== --- bolt/test/AArch64/Inputs/got-ld64-relaxation.yaml +++ bolt/test/AArch64/Inputs/got-ld64-relaxation.yaml @@ -2,133 +2,156 @@ FileHeader: Class: ELFCLASS64 Data: ELFDATA2LSB - Type: ET_EXEC + Type: ET_DYN Machine: EM_AARCH64 - Entry: 0x210648 + Entry: 0x264 ProgramHeaders: - Type: PT_PHDR Flags: [ PF_R ] - VAddr: 0x200040 + VAddr: 0x40 Align: 0x8 + FileSize: 0x1C0 + MemSize: 0x1C0 + Offset: 0x40 - Type: PT_INTERP Flags: [ PF_R ] FirstSec: .interp LastSec: .interp - VAddr: 0x2002A8 - - Type: PT_LOAD - Flags: [ PF_R ] - FirstSec: .interp - LastSec: .rodata - VAddr: 0x200000 - Align: 0x10000 + VAddr: 0x200 + Offset: 0x200 - Type: PT_LOAD Flags: [ PF_X, PF_R ] - FirstSec: .text - LastSec: .plt - VAddr: 0x210648 - Align: 0x10000 - - Type: PT_LOAD - Flags: [ PF_W, PF_R ] - FirstSec: .text - LastSec: .got - VAddr: 0x322580 + FirstSec: .interp + LastSec: .eh_frame Align: 0x10000 + Offset: 0x0 - Type: PT_LOAD Flags: [ PF_W, PF_R ] - FirstSec: .data - LastSec: .bss - VAddr: 0x332720 + FirstSec: .dynamic + LastSec: .got.plt + VAddr: 0x10F10 Align: 0x10000 + Offset: 0xF10 - Type: PT_DYNAMIC Flags: [ PF_W, PF_R ] FirstSec: .dynamic LastSec: .dynamic - VAddr: 0x322590 + VAddr: 0x10F10 Align: 0x8 + Offset: 0xF10 + - Type: PT_GNU_EH_FRAME + Flags: [ PF_R ] + FirstSec: .eh_frame_hdr + LastSec: .eh_frame_hdr + VAddr: 0x298 + Align: 0x4 + Offset: 0x298 + - Type: PT_GNU_STACK + Flags: [ PF_W, PF_R ] + Align: 0x10 + Offset: 0x0 + - Type: PT_GNU_RELRO + Flags: [ PF_R ] + FirstSec: .dynamic + LastSec: .got.plt + VAddr: 0x10F10 + Offset: 0xF10 Sections: - Name: .interp Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] - Address: 0x2002A8 + Address: 0x200 AddressAlign: 0x1 Content: 2F6C69622F6C642D6C696E75782D616172636836342E736F2E3100 + - Name: .gnu.hash + Type: SHT_GNU_HASH + Flags: [ SHF_ALLOC ] + Address: 0x220 + Link: .dynsym + AddressAlign: 0x8 + Header: + SymNdx: 0x1 + Shift2: 0x0 + BloomFilter: [ 0x0 ] + HashBuckets: [ 0x0 ] + HashValues: [ ] - Name: .dynsym Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] - Address: 0x200300 + Address: 0x240 Link: .dynstr AddressAlign: 0x8 - Name: .dynstr Type: SHT_STRTAB Flags: [ SHF_ALLOC ] - Address: 0x2003F4 + Address: 0x258 AddressAlign: 0x1 - - Name: .rodata - Type: SHT_PROGBITS - Flags: [ SHF_ALLOC, SHF_MERGE, SHF_STRINGS ] - Address: 0x2004E8 - AddressAlign: 0x8 - Content: 010002000000000000000000000000000000000000000000486572650A00 - Name: .text Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] - Address: 0x210648 - AddressAlign: 0x8 - Content: 1D0080D21E0080D2E50300AAE10340F9E2230091E60300910000009000001A91030800D063E01191040800D084E01391B6070494B10704942F000014800800D0008843F9400000B4B4070414C0035FD6000900D000C01C91010900D021C01C913F0000EBC000005481FFFF90217842F9610000B4F00301AA00021FD6C0035FD6000900D000C01C91010900D021C01C91210000CB22FC7FD3410C818BFF0781EB21FC4193C000005482FFFF90427C42F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F9130900D060A25D3980000035DEFFFF972000805260A21D39F30B40F9FD7BC2A8C0035FD6E4FFFF17FF4300D1000800D000601191E00700F90004009100008052FF430091C0035FD6FD7BBFA9FD03009160F7FFD00000149142000094FD7BC1A8C0035FD600000000FD7BBCA9FD030091F35301A91F2003D554080810F55B02A91F2003D5B5070810940215CBF603002AF76303A9F70301AAF80302AA14000094FF0F94EB6001005494FE4393130080D2A37A73F8E20318AA73060091E10317AAE003162A60003FD69F0213EB21FFFF54F35341A9F55B42A9F76343A9FD7BC4A8C0035FD61F2003D5C0035FD6 - - Name: .plt + Address: 0x25C + AddressAlign: 0x4 + Content: E0031F2AC0035FD6FF8300D1FD7B01A9FD430091E8031F2AE80B00B9BFC31FB80100009021700991E00B40B9FD7B41A9FF830091A80B8052010000D4 + - Name: .eh_frame_hdr Type: SHT_PROGBITS - Flags: [ SHF_ALLOC, SHF_EXECINSTR ] - Address: 0x312520 - AddressAlign: 0x10 - Content: F07BBFA91001009011A243F910021D9120021FD61F2003D51F2003D51F2003D51001009011A643F910221D9120021FD61001009011AA43F910421D9120021FD61001009011AE43F910621D9120021FD61001009011B243F910821D9120021FD6 + Flags: [ SHF_ALLOC ] + Address: 0x298 + AddressAlign: 0x4 + Content: 011B033B1C00000002000000C4FFFFFF34000000CCFFFFFF48000000 + - Name: .eh_frame + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x2B8 + AddressAlign: 0x8 + Content: 1000000000000000017A5200017C1E011B0C1F00100000001800000088FFFFFF0800000000000000240000002C0000007CFFFFFF3400000000440E20480C1D109E029D04580C1F20500E00DEDD000000 - Name: .dynamic Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x322590 + Address: 0x10F10 Link: .dynstr AddressAlign: 0x8 + Offset: 0xF10 Entries: - - Tag: DT_NEEDED - Value: 0x65 + - Tag: DT_GNU_HASH + Value: 0x220 + - Tag: DT_STRTAB + Value: 0x258 - Tag: DT_SYMTAB - Value: 0x200300 + Value: 0x240 + - Tag: DT_STRSZ + Value: 0x1 - Tag: DT_SYMENT Value: 0x18 - - Tag: DT_STRTAB - Value: 0x2003F4 - - Tag: DT_STRSZ - Value: 0x7A + - Tag: DT_DEBUG + Value: 0x0 + - Tag: DT_FLAGS_1 + Value: 0x8000000 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 + - Tag: DT_NULL + Value: 0x0 - Tag: DT_NULL Value: 0x0 - Name: .got Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x322710 - AddressAlign: 0x8 - Content: '00000000000000005824310000000000' - - Name: .data - Type: SHT_PROGBITS - Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x332720 - AddressAlign: 0x8 - Content: '00000000000000000000000000000000' - - Name: .tm_clone_table - Type: SHT_PROGBITS - Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x332730 + Address: 0x10FE0 AddressAlign: 0x8 + EntSize: 0x8 + Content: 100F010000000000 - Name: .got.plt Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x332730 + Address: 0x10FE8 AddressAlign: 0x8 - Content: '0000000000000000000000000000000000000000000000002025310000000000202531000000000020253100000000002025310000000000' - - Name: .bss - Type: SHT_NOBITS - Flags: [ SHF_WRITE, SHF_ALLOC ] - Address: 0x332768 - AddressAlign: 0x1 - Size: 0x1 + EntSize: 0x8 + Content: '000000000000000000000000000000000000000000000000' - Name: .rela.text Type: SHT_RELA Flags: [ SHF_INFO_LINK ] @@ -136,67 +159,144 @@ AddressAlign: 0x8 Info: .text Relocations: - - Offset: 0x210740 + - Offset: 0x27C Symbol: foo Type: R_AARCH64_ADR_GOT_PAGE - - Offset: 0x210744 + - Offset: 0x280 Symbol: foo Type: R_AARCH64_LD64_GOT_LO12_NC + - Name: .rela.eh_frame + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .eh_frame + Relocations: + - Offset: 0x2D4 + Symbol: .text + Type: R_AARCH64_PREL32 + - Offset: 0x2E8 + Symbol: .text + Type: R_AARCH64_PREL32 + Addend: 8 + - Type: SectionHeaderTable + Sections: + - Name: .interp + - Name: .gnu.hash + - Name: .dynsym + - Name: .dynstr + - Name: .text + - Name: .rela.text + - Name: .eh_frame_hdr + - Name: .eh_frame + - Name: .rela.eh_frame + - Name: .dynamic + - Name: .got + - Name: .got.plt + - Name: .symtab + - Name: .strtab + - Name: .shstrtab Symbols: - - Name: '$x' - Section: .text - Value: 0x210648 + - Name: .interp + Type: STT_SECTION + Section: .interp + Value: 0x200 + - Name: .gnu.hash + Type: STT_SECTION + Section: .gnu.hash + Value: 0x220 + - Name: .dynsym + Type: STT_SECTION + Section: .dynsym + Value: 0x240 + - Name: .dynstr + Type: STT_SECTION + Section: .dynstr + Value: 0x258 - Name: .text Type: STT_SECTION Section: .text - Value: 0x210648 - - Name: .data + Value: 0x25C + - Name: .eh_frame_hdr Type: STT_SECTION - Section: .data - Value: 0x332720 - - Name: .bss + Section: .eh_frame_hdr + Value: 0x298 + - Name: .eh_frame Type: STT_SECTION - Section: .bss - Value: 0x332768 - - Name: .rodata - Type: STT_SECTION - Section: .rodata - Value: 0x2004E8 - - Name: .interp + Section: .eh_frame + Value: 0x2B8 + - Name: .dynamic Type: STT_SECTION - Section: .interp - Value: 0x2002A8 - - Name: _DYNAMIC Section: .dynamic - Value: 0x322590 - Other: [ STV_HIDDEN ] - - Name: _start - Type: STT_FUNC + Value: 0x10F10 + - Name: .got + Type: STT_SECTION + Section: .got + Value: 0x10FE0 + - Name: .got.plt + Type: STT_SECTION + Section: .got.plt + Value: 0x10FE8 + - Name: tmp.c + Type: STT_FILE + Index: SHN_ABS + - Name: '$x.0' Section: .text + Value: 0x25C + - Name: '$d.1' + Section: .eh_frame + Value: 0x2B8 + - Type: STT_FILE + Index: SHN_ABS + - Name: _DYNAMIC + Type: STT_OBJECT + Index: SHN_ABS + Value: 0x10F10 + - Name: __GNU_EH_FRAME_HDR + Section: .eh_frame_hdr + Value: 0x298 + - Name: _GLOBAL_OFFSET_TABLE_ + Type: STT_OBJECT + Index: SHN_ABS + Value: 0x10FE0 + - Name: _bss_end__ + Section: .got.plt Binding: STB_GLOBAL - Value: 0x210648 - - Name: main - Type: STT_FUNC + Value: 0x11000 + - Name: __bss_start__ + Section: .got.plt + Binding: STB_GLOBAL + Value: 0x11000 + - Name: __bss_end__ + Section: .got.plt + Binding: STB_GLOBAL + Value: 0x11000 + - Name: _start Section: .text Binding: STB_GLOBAL - Value: 0x21073C - Size: 0x20 - - Name: data_start - Section: .data - Binding: STB_WEAK - Value: 0x332720 - - Name: _IO_stdin_used - Type: STT_OBJECT - Section: .rodata + Value: 0x264 + Size: 0x34 + - Name: __bss_start + Section: .got.plt Binding: STB_GLOBAL - Value: 0x2004E8 - Size: 0x4 - - Name: __libc_start_main - Type: STT_FUNC + Value: 0x11000 + - Name: __end__ + Section: .got.plt Binding: STB_GLOBAL + Value: 0x11000 - Name: foo Type: STT_FUNC Section: .text Binding: STB_GLOBAL - Value: 0x21075c - Size: 0xF + Value: 0x25C + Size: 0x8 + - Name: _edata + Section: .got.plt + Binding: STB_GLOBAL + Value: 0x11000 + - Name: _end + Section: .got.plt + Binding: STB_GLOBAL + Value: 0x11000 +DynamicSymbols: [] +... Index: bolt/test/AArch64/Inputs/plt-gnu-ld.yaml =================================================================== --- bolt/test/AArch64/Inputs/plt-gnu-ld.yaml +++ bolt/test/AArch64/Inputs/plt-gnu-ld.yaml @@ -9,17 +9,29 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x400040 + Offset: 0x000040 + MemSize: 0x118 + FileSize: 0x118 Align: 0x8 - Type: PT_INTERP Flags: [ PF_R ] FirstSec: .interp LastSec: .interp VAddr: 0x400238 + Offset: 0x000238 - Type: PT_LOAD Flags: [ PF_X, PF_R ] FirstSec: .interp - LastSec: .bss + LastSec: .rodata VAddr: 0x400000 + Offset: 0x0 + Align: 0x10000 + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] + VAddr: 0x410e08 + Offset: 0x000e08 + FirstSec: .dynamic + LastSec: .bss Align: 0x10000 - Type: PT_DYNAMIC Flags: [ PF_W, PF_R ] @@ -32,12 +44,14 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x400238 + Offset: 0x000238 AddressAlign: 0x1 Content: 2F6C69622F6C642D6C696E75782D616172636836342E736F2E3100 - Name: .dynsym Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] Address: 0x4002A0 + Offset: 0x0002A0 Link: .dynstr AddressAlign: 0x8 - Name: .dynstr @@ -49,6 +63,7 @@ Type: SHT_RELA Flags: [ SHF_ALLOC ] Address: 0x4003C8 + Offset: 0x0003C8 Link: .dynsym AddressAlign: 0x8 Relocations: @@ -59,6 +74,7 @@ Type: SHT_RELA Flags: [ SHF_ALLOC, SHF_INFO_LINK ] Address: 0x4003E0 + Offset: 0x0003E0 Link: .dynsym AddressAlign: 0x8 Info: .got.plt @@ -85,6 +101,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x400490 + Offset: 0x000490 AddressAlign: 0x10 EntSize: 0x10 Content: F07BBFA99000009011FE47F910E23F9120021FD61F2003D51F2003D51F2003D5900000B0110240F91002009120021FD6900000B0110640F91022009120021FD6900000B0110A40F91042009120021FD6900000B0110E40F91062009120021FD6900000B0111240F91082009120021FD6900000B0111640F910A2009120021FD6 @@ -98,12 +115,14 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x400760 + Offset: 0x000760 AddressAlign: 0x8 Content: 0100020000000000000000000000000000000000000000005465737420636F6D706C657465640A00 - Name: .dynamic Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x410E08 + Offset: 0x000E08 Link: .dynstr AddressAlign: 0x8 Entries: @@ -139,6 +158,7 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x410FD8 + Offset: 0x000fd8 AddressAlign: 0x8 EntSize: 0x8 Content: '080E4100000000000000000000000000' @@ -153,11 +173,13 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x411040 + Offset: 0x001040 AddressAlign: 0x8 - Name: .bss Type: SHT_NOBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x411040 + Offset: 0x001040 AddressAlign: 0x8 Size: 0x18 - Name: .rela.text Index: bolt/test/AArch64/Inputs/rels-exe.yaml =================================================================== --- bolt/test/AArch64/Inputs/rels-exe.yaml +++ bolt/test/AArch64/Inputs/rels-exe.yaml @@ -9,6 +9,9 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x200040 + Offset: 0x40 + MemSize: 0x188 + FileSize: 0x188 Align: 0x8 - Type: PT_INTERP Flags: [ PF_R ] @@ -20,6 +23,7 @@ FirstSec: .interp LastSec: .rodata VAddr: 0x200000 + Offset: 0x0 Align: 0x10000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] @@ -50,23 +54,27 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x2002A8 + Offset: 0x2A8 AddressAlign: 0x1 Content: 2F6C69622F6C642D6C696E75782D616172636836342E736F2E3100 - Name: .dynsym Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] Address: 0x2002E8 + Offset: 0x2E8 Link: .dynstr AddressAlign: 0x8 - Name: .dynstr Type: SHT_STRTAB Flags: [ SHF_ALLOC ] Address: 0x200418 + Offset: 0x418 AddressAlign: 0x1 - Name: .rela.dyn Type: SHT_RELA Flags: [ SHF_ALLOC, SHF_INFO_LINK ] Address: 0x2004A0 + Offset: 0x4A0 Link: .dynsym AddressAlign: 0x8 Info: .got.plt @@ -84,6 +92,7 @@ Type: SHT_RELA Flags: [ SHF_ALLOC, SHF_INFO_LINK ] Address: 0x200500 + Offset: 0x500 Link: .dynsym AddressAlign: 0x8 Info: .got.plt @@ -101,18 +110,21 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_MERGE ] Address: 0x200560 + Offset: 0x560 AddressAlign: 0x8 Content: '010002000000000000000000000000000000000000000000' - Name: .text Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x210710 + Offset: 0x710 AddressAlign: 0x8 Content: 1D0080D21E0080D2E50300AAE10340F9E2230091E60300910000009000201D911F2003D5230A00101F2003D5E40D0010840000947F0000943800001480000090009845F9400000B482000014C0035FD61F2003D5601F10101F2003D5211F10103F0000EBC000005481FFFF9021B442F9610000B4F00301AA00021FD6C0035FD61F2003D5E01D10101F2003D5A11D1010210000CB22FC7FD3410C818BFF0781EB21FC4193C000005482FFFF9042B842F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F91301009060426E3980000035DEFFFF972000805260422E39F30B40F9FD7BC2A8C0035FD6E4FFFF17FF4300D1E00F00B9E80F40B900050071FF430091C0035FD61F2003D520FFFF10C0035FD6FF8300D1FD7B01A9FD4300918A0000904A9D45F949D03BD528696AB80805001128692AB808010090E80700F900994BB94A000094E80740F900994BB943000094E0031F2AFD7B41A9FF830091C0035FD6FD7BBCA9FD030091F35301A91F2003D5D4080810F55B02A91F2003D535080810940215CBF603002AF76303A9F70301AAF80302AA14000094FF0F94EB6001005494FE4393130080D2A37A73F8E20318AA73060091E10317AAE003162A60003FD69F0213EB21FFFF54F35341A9F55B42A9F76343A9FD7BC4A8C0035FD61F2003D5C0035FD6 - Name: .plt Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x210920 + Offset: 0x920 AddressAlign: 0x10 Content: F07BBFA91001009011B245F910822D9120021FD61F2003D51F2003D51F2003D51001009011B645F910A22D9120021FD61001009011BA45F910C22D9120021FD61001009011BE45F910E22D9120021FD61001009011C245F910022E9120021FD6 - Name: .iplt @@ -125,6 +137,7 @@ Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x2209A0 + Offset: 0x9A0 Link: .dynstr AddressAlign: 0x8 Entries: @@ -162,6 +175,7 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x220B30 + Offset: 0xB30 AddressAlign: 0x8 Content: '00000000000000000000000000000000' - Name: .data Index: bolt/test/AArch64/Inputs/rels-so.yaml =================================================================== --- bolt/test/AArch64/Inputs/rels-so.yaml +++ bolt/test/AArch64/Inputs/rels-so.yaml @@ -8,11 +8,16 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x40 + Offset: 0x40 + MemSize: 0x1C0 + FileSize: 0x1C0 Align: 0x8 - Type: PT_LOAD Flags: [ PF_R ] FirstSec: .dynsym - LastSec: .dynamic + LastSec: .rela.plt + VAddr: 0x00000 + Offset: 0x00000 Align: 0x10000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] @@ -22,9 +27,10 @@ Align: 0x10000 - Type: PT_LOAD Flags: [ PF_W, PF_R ] - FirstSec: .tbss + FirstSec: .dynamic LastSec: .got VAddr: 0x207F0 + Offset: 0x007F0 Align: 0x10000 - Type: PT_LOAD Flags: [ PF_W, PF_R ] @@ -43,6 +49,7 @@ FirstSec: .dynamic LastSec: .dynamic VAddr: 0x20800 + Offset: 0x00800 Align: 0x8 - Type: PT_GNU_RELRO Flags: [ PF_R ] @@ -54,17 +61,27 @@ Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] Address: 0x270 + Offset: 0x270 Link: .dynstr AddressAlign: 0x8 + - Name: .gnu.hash + Type: 0x5 + Flags: [ SHF_ALLOC ] + Address: 0x380 + Offset: 0x380 + Size: 0x1c + AddressAlign: 0x8 - Name: .dynstr Type: SHT_STRTAB Flags: [ SHF_ALLOC ] Address: 0x3AC + Offset: 0x3AC AddressAlign: 0x1 - Name: .rela.dyn Type: SHT_RELA Flags: [ SHF_ALLOC ] Address: 0x428 + Offset: 0x428 Link: .dynsym AddressAlign: 0x8 Relocations: @@ -81,6 +98,7 @@ Type: SHT_RELA Flags: [ SHF_ALLOC, SHF_INFO_LINK ] Address: 0x530 + Offset: 0x530 Link: .dynsym AddressAlign: 0x8 Info: .got.plt @@ -95,12 +113,14 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x10658 + Offset: 0x00658 AddressAlign: 0x8 Content: 8000009000C844F9400000B45B000014C0035FD6000000001F2003D5A01B10101F2003D5611B10103F0000EBC00000548100009021CC44F9610000B4F00301AA00021FD6C0035FD61F2003D5201A10101F2003D5E1191010210000CB22FC7FD3410C818BFF0781EB21FC4193C00000548200009042D044F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F91301009060426839400100358000009000D444F9800000B40001009000E444F935000094D8FFFF972000805260422839F30B40F9FD7BC2A8C0035FD6DEFFFF17FF8300D1FD7B01A9FD430091A0C31FB88900009029D944F9280140B908050011280100B98000009001DC44F900E0269120003FD649D03BD5286960B808050011286920B8A8C35FB800050011FD7B41A9FF830091C0035FD6 - Name: .plt Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x107B0 + Offset: 0x007B0 AddressAlign: 0x10 Content: F07BBFA91001009011FE44F910E2279120021FD61F2003D51F2003D51F2003D510010090110245F91002289120021FD610010090110645F91022289120021FD6 - Name: .tbss @@ -113,6 +133,7 @@ Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x20800 + Offset: 0x00800 Link: .dynstr AddressAlign: 0x8 Entries: @@ -152,6 +173,7 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x20990 + Offset: 0x990 AddressAlign: 0x8 Content: '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' - Name: .data Index: bolt/test/AArch64/Inputs/skip-got-rel.yaml =================================================================== --- bolt/test/AArch64/Inputs/skip-got-rel.yaml +++ bolt/test/AArch64/Inputs/skip-got-rel.yaml @@ -10,6 +10,8 @@ Flags: [ PF_R ] VAddr: 0x40 Align: 0x8 + FileSize: 0x1C0 + MemSize: 0x1C0 - Type: PT_INTERP Flags: [ PF_R ] FirstSec: .interp @@ -18,7 +20,8 @@ - Type: PT_LOAD Flags: [ PF_R ] FirstSec: .interp - LastSec: .dynamic + LastSec: .interp + Offset: 0x0 Align: 0x10000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] @@ -51,6 +54,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x238 + Offset: 0x238 AddressAlign: 0x1 Content: 2F6C69622F6C642D6C696E75782D616172636836342E736F2E3100 - Name: .dynsym @@ -78,6 +82,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x10348 + Offset: 0x348 AddressAlign: 0x4 Content: FF4300D1E00700F9E80740F908014092E003082AFF430091C0035FD6FD7BBFA9FD0300911F2003D580000010F5FFFF97FD7BC1A8C0035FD6C0035FD6 - Name: .dynamic @@ -113,6 +118,7 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x20448 + Offset: 0x448 AddressAlign: 0x8 Content: '0000000000000000' - Name: .rela.text Index: bolt/test/AArch64/Inputs/tls-ld.yaml =================================================================== --- bolt/test/AArch64/Inputs/tls-ld.yaml +++ bolt/test/AArch64/Inputs/tls-ld.yaml @@ -7,9 +7,17 @@ Entry: 0x590 ProgramHeaders: - Type: PT_LOAD - Flags: [ PF_X, PF_R, PF_W ] + Flags: [ PF_X, PF_R ] FirstSec: .dynsym - LastSec: .got + LastSec: .rela.plt + Offset: 0x0 + VAddr: 0x0 + Align: 0x10000 + - Type: PT_LOAD + Flags: [ PF_R, PF_W ] + FirstSec: .tbss + LastSec: .got.plt + VAddr: 0x10DE0 Align: 0x10000 - Type: PT_DYNAMIC Flags: [ PF_W, PF_R ] @@ -34,17 +42,20 @@ Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] Address: 0x250 + Offset: 0x250 Link: .dynstr AddressAlign: 0x8 - Name: .dynstr Type: SHT_STRTAB Flags: [ SHF_ALLOC ] Address: 0x340 + Offset: 0x340 AddressAlign: 0x1 - Name: .plt Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x540 + Offset: 0x540 AddressAlign: 0x10 EntSize: 0x10 Content: F07BBFA99000009011FE47F910E23F9120021FD61F2003D51F2003D51F2003D5900000B0110240F91002009120021FD6900000B0110640F91022009120021FD6900000B0110A40F91042009120021FD6 @@ -52,22 +63,14 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x590 + Offset: 0x590 AddressAlign: 0x8 Content: 8000009000F047F9400000B4F9FFFF17C0035FD61F2003D5800000B000800091810000B0218000913F0000EBC00000548100009021E447F9610000B4F00301AA00021FD6C0035FD6800000B000800091810000B021800091210000CB22FC7FD3410C818BFF0781EB21FC4193C00000548200009042E047F9620000B4F00302AA00021FD6C0035FD6FD7BBEA9FD030091F30B00F9930000B060824039400100358000009000DC47F9800000B4800000B0000C40F9C7FFFF97D8FFFF972000805260820039F30B40F9FD7BC2A8C0035FD6DEFFFF171F2003D5FD7BBEA9FD030091F30B00F9F303002A8000009000403F91BCFFFF971F2003D5E10300AA60060011F30B40F9220040B942040011220000B9FD7BC2A8C0035FD6 - - Name: .rela.text - Type: SHT_RELA - Flags: [ SHF_INFO_LINK ] - Link: .symtab - AddressAlign: 0x8 - Info: .text - Relocations: - - Offset: 0x5C4 - Symbol: t1 - Type: R_AARCH64_TLSDESC_LD64_LO12 - Name: .eh_frame_hdr Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x6B8 + Offset: 0x6B8 AddressAlign: 0x4 Content: 011B033B3400000005000000F0FEFFFF4C00000020FFFFFF6000000060FFFFFF74000000A8FFFFFF98000000B0FFFFFFB0000000 - Name: .eh_frame @@ -76,6 +79,28 @@ Address: 0x6F0 AddressAlign: 0x8 Content: 1000000000000000017A520004781E011B0C1F0010000000180000009CFEFFFF3000000000000000100000002C000000B8FEFFFF40000000000000002000000040000000E4FEFFFF4800000000410E209D049E034293024EDEDDD30E00000000140000006400000008FFFFFF040000000000000000000000200000007C000000F8FEFFFF4000000000410E209D049E034293024CDEDDD30E0000000000000000 + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x10FF8 + Symbol: pltSym1 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11000 + Symbol: pltSym2 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11008 + Symbol: pltSym3 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11010 + Symbol: pltSym4 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fb0 + Symbol: t1 + Type: R_AARCH64_TLSDESC - Name: .tbss Type: SHT_NOBITS Flags: [ SHF_WRITE, SHF_ALLOC, SHF_TLS ] @@ -87,6 +112,7 @@ Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x10DF0 + Offset: 0xDF0 Link: .dynstr AddressAlign: 0x8 Entries: @@ -95,47 +121,58 @@ - Tag: DT_SYMTAB Value: 0x250 - Tag: DT_PLTRELSZ - Value: 0x18 + Value: 0x78 - Tag: DT_PLTREL Value: 0x7 - Tag: DT_JMPREL - Value: 0x418 + Value: 0x790 - Tag: DT_NULL Value: 0x0 - Name: .got Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x10FB0 + Offset: 0xFB0 AddressAlign: 0x8 EntSize: 0x8 Content: F00D010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - Name: .rela.plt - Type: SHT_RELA - Flags: [ SHF_ALLOC, SHF_INFO_LINK ] - Address: 0x418 - Link: .dynsym + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_WRITE ] + Address: 0x10FE8 + Offset: 0xFE8 AddressAlign: 0x8 - Info: .got - Relocations: - - Offset: 0x10fb0 - Symbol: t1 - Type: R_AARCH64_TLSDESC + Size: 0x30 - Type: SectionHeaderTable Sections: - Name: .dynsym - Name: .dynstr - - Name: .rela.plt - Name: .plt - Name: .text - Name: .rela.text - Name: .eh_frame_hdr - Name: .eh_frame + - Name: .rela.plt - Name: .tbss - Name: .dynamic - Name: .got + - Name: .got.plt - Name: .symtab - Name: .strtab - Name: .shstrtab + - Name: .rela.text + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x8 + Info: .text + Relocations: + - Offset: 0x5C0 + Symbol: t1 + Type: R_AARCH64_TLSDESC_ADR_PAGE21 + - Offset: 0x5C4 + Symbol: t1 + Type: R_AARCH64_TLSDESC_LD64_LO12 Symbols: - Name: .text Type: STT_SECTION @@ -147,6 +184,18 @@ Binding: STB_GLOBAL Size: 0x4 DynamicSymbols: + - Name: pltSym1 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym2 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym3 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym4 + Type: STT_FUNC + Binding: STB_GLOBAL - Name: t1 Type: STT_TLS Section: .tbss Index: bolt/test/AArch64/Inputs/tls-trad.yaml =================================================================== --- bolt/test/AArch64/Inputs/tls-trad.yaml +++ bolt/test/AArch64/Inputs/tls-trad.yaml @@ -9,8 +9,15 @@ - Type: PT_LOAD Flags: [ PF_X, PF_R, PF_W ] FirstSec: .dynsym - LastSec: .got + LastSec: .eh_frame + Offset: 0x0 + VAddr: 0x0 Align: 0x10000 + - Type: PT_LOAD + Flags: [ PF_R, PF_W ] + VAddr: 0x10de0 + FirstSec: .tbss + LastSec: .got.plt - Type: PT_DYNAMIC Flags: [ PF_W, PF_R ] FirstSec: .dynamic @@ -34,17 +41,20 @@ Type: SHT_DYNSYM Flags: [ SHF_ALLOC ] Address: 0x250 + Offset: 0x250 Link: .dynstr AddressAlign: 0x8 - Name: .dynstr Type: SHT_STRTAB Flags: [ SHF_ALLOC ] Address: 0x340 + Offset: 0x340 AddressAlign: 0x1 - Name: .rela.dyn Type: SHT_RELA Flags: [ SHF_ALLOC ] Address: 0x400 + Offset: 0x400 Link: .dynsym AddressAlign: 0x8 Relocations: @@ -58,6 +68,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x540 + Offset: 0x540 AddressAlign: 0x10 EntSize: 0x10 Content: F07BBFA99000009011FE47F910E23F9120021FD61F2003D51F2003D51F2003D5900000B0110240F91002009120021FD6900000B0110640F91022009120021FD6900000B0110A40F91042009120021FD6 @@ -77,8 +88,31 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x6F0 + Offset: 0x6F0 AddressAlign: 0x8 Content: 1000000000000000017A520004781E011B0C1F0010000000180000009CFEFFFF3000000000000000100000002C000000B8FEFFFF40000000000000002000000040000000E4FEFFFF4800000000410E209D049E034293024EDEDDD30E00000000140000006400000008FFFFFF040000000000000000000000200000007C000000F8FEFFFF4000000000410E209D049E034293024CDEDDD30E0000000000000000 + - Name: .rela.plt + Type: SHT_RELA + Flags: [ SHF_ALLOC, SHF_INFO_LINK ] + Link: .dynsym + AddressAlign: 0x8 + Info: .got.plt + Relocations: + - Offset: 0x10FF8 + Symbol: pltSym1 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11000 + Symbol: pltSym2 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11008 + Symbol: pltSym3 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x11010 + Symbol: pltSym4 + Type: R_AARCH64_JUMP_SLOT + - Offset: 0x10fb0 + Symbol: t1 + Type: R_AARCH64_TLSDESC - Name: .tbss Type: SHT_NOBITS Flags: [ SHF_WRITE, SHF_ALLOC, SHF_TLS ] @@ -90,6 +124,7 @@ Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x10DF0 + Offset: 0xDF0 Link: .dynstr AddressAlign: 0x8 Entries: @@ -111,9 +146,17 @@ Type: SHT_PROGBITS Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x10FB0 + Offset: 0xFB0 AddressAlign: 0x8 EntSize: 0x8 Content: F00D010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + - Name: .got.plt + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC, SHF_WRITE ] + Address: 0x10FE8 + Offset: 0xFE8 + AddressAlign: 0x8 + Size: 0x30 - Name: .rela.text Type: SHT_RELA Flags: [ SHF_INFO_LINK ] @@ -137,9 +180,11 @@ - Name: .rela.text - Name: .eh_frame_hdr - Name: .eh_frame + - Name: .rela.plt - Name: .tbss - Name: .dynamic - Name: .got + - Name: .got.plt - Name: .symtab - Name: .strtab - Name: .shstrtab @@ -148,6 +193,23 @@ Type: STT_SECTION Section: .text Value: 0x590 +# Add functions because otherwise BOLT will complain + - Name: one + Type: STT_FUNC + Section: .text + Value: 0x540 + - Name: two + Type: STT_FUNC + Section: .text + Value: 0x560 + - Name: three + Type: STT_FUNC + Section: .text + Value: 0x570 + - Name: four + Type: STT_FUNC + Section: .text + Value: 0x580 - Name: t1 Type: STT_TLS Section: .tbss @@ -159,4 +221,16 @@ Section: .tbss Binding: STB_GLOBAL Size: 0x4 + - Name: pltSym1 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym2 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym3 + Type: STT_FUNC + Binding: STB_GLOBAL + - Name: pltSym4 + Type: STT_FUNC + Binding: STB_GLOBAL ... Index: bolt/test/AArch64/array_end.c =================================================================== --- bolt/test/AArch64/array_end.c +++ /dev/null @@ -1,23 +0,0 @@ -// Test checks that bolt properly finds end section label. -// Linker script contains gap after destructor array, so -// __init_array_end address would not be owned by any section. - -// REQUIRES: system-linux -// RUN: %clang %cflags -no-pie %s -o %t.exe -Wl,-q \ -// RUN: -Wl,-T %S/Inputs/array_end.lld_script -// RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm \ -// RUN: --print-only="callFini" | FileCheck %s - -// CHECK: adr [[REG:x[0-28]+]], "__fini_array_end/1" - -__attribute__((destructor)) void destr() {} - -__attribute__((noinline)) void callFini() { - extern void (*__fini_array_start[])(); - extern void (*__fini_array_end[])(); - unsigned long Count = __fini_array_end - __fini_array_start; - for (unsigned long I = 0; I < Count; ++I) - (*__fini_array_start[I])(); -} - -void _start() { callFini(); } Index: bolt/test/AArch64/array_end.test =================================================================== --- /dev/null +++ bolt/test/AArch64/array_end.test @@ -0,0 +1,43 @@ +// Test checks that bolt properly finds end section label. +// Linker script contains gap after destructor array, so +// __init_array_end address would not be owned by any section. + +// REQUIRES: system-linux +// RUN: %clang %cflags -no-pie %S/Inputs/array_end.c -o %t.exe -Wl,-q \ +// RUN: -Wl,-T %S/Inputs/array_end.lld_script +// RUN: llvm-bolt %t.exe -o %t.bolt --print-disasm \ +// RUN: --print-only="callFini" --debug-only=bolt 2>&1 | FileCheck %s + +// CHECK: BOLT-INFO: __fini_array_end is in the end of .fini_array +// CHECK: adr [[REG:x[0-28]+]], __fini_array_end + +// RUN: llvm-bolt %t.exe -o %t.bolt.rewritten --rewrite --print-disasm \ +// RUN: --print-only="callFini" --debug-only=bolt 2>&1 | FileCheck %s + +// Check assembly and symtab +// RUN: llvm-objdump --disassemble-symbols=callFini %t.bolt > %t.log +// RUN: llvm-readelf -WS %t.bolt | grep .fini_array >> %t.log +// RUN: llvm-readelf -Ws %t.bolt | grep __fini_array_end >> %t.log +// RUN: llvm-readelf -Ws %t.bolt | grep __fini_array_end >> %t.log + +// RUN: cat %t.log | FileCheck %s --check-prefix=CHECK-FIRST + +// CHECK-FIRST: adrp [[REG:x[0-28]+]], 0x[[#%x,PAGE:]] +// CHECK-FIRST: add [[REG]], [[REG]], #0x[[#%x,OFFSET:]] +// CHECK-FIRST: .fini_array PROGBITS {{[0]+}}[[#%x,SECADDR:]] {{[0]+}}[[#%x,SECOFFSET:]] {{[0]+}}[[#%x,SECSIZE:]] +// CHECK-FIRST: {{[0]+}}[[#%x,PAGE+OFFSET]] 0 NOTYPE LOCAL HIDDEN {{[0-9]+}} __fini_array_end +// CHECK-FIRST: {{[0]+}}[[#%x,SECADDR+SECSIZE]] 0 NOTYPE LOCAL HIDDEN {{[0-9]+}} __fini_array_end + +// Check assembly and symtab after rewrite +// RUN: llvm-objdump --disassemble-symbols=callFini %t.bolt.rewritten > %t.log +// RUN: llvm-readelf -WS %t.bolt.rewritten | grep .fini_array >> %t.log +// RUN: llvm-readelf -Ws %t.bolt.rewritten | grep __fini_array_end >> %t.log +// RUN: llvm-readelf -Ws %t.bolt.rewritten | grep __fini_array_end >> %t.log + +// RUN: cat %t.log | FileCheck %s --check-prefix=CHECK-SECOND + +// CHECK-SECOND: adrp [[REG:x[0-28]+]], 0x[[#%x,PAGE:]] +// CHECK-SECOND: add [[REG]], [[REG]], #0x[[#%x,OFFSET:]] +// CHECK-SECOND: .fini_array PROGBITS {{[0]+}}[[#%x,SECADDR:]] {{[0]+}}[[#%x,SECOFFSET:]] {{[0]+}}[[#%x,SECSIZE:]] +// CHECK-SECOND: {{[0]+}}[[#%x,PAGE+OFFSET]] 0 NOTYPE LOCAL HIDDEN {{[0-9]+}} __fini_array_end +// CHECK-SECOND: {{[0]+}}[[#%x,SECADDR+SECSIZE]] 0 NOTYPE LOCAL HIDDEN {{[0-9]+}} __fini_array_end Index: bolt/test/AArch64/got-ld64-relaxation.test =================================================================== --- bolt/test/AArch64/got-ld64-relaxation.test +++ bolt/test/AArch64/got-ld64-relaxation.test @@ -2,8 +2,7 @@ // to the ADR+ADD sequence is properly reconized and handled by bolt // RUN: yaml2obj %p/Inputs/got-ld64-relaxation.yaml &> %t.exe -// RUN: llvm-bolt %t.exe -o /dev/null --print-fix-relaxations \ -// RUN: --print-only=main | FileCheck %s +// RUN: llvm-bolt %t.exe -o %t.exe.bolt -print-only=_start -print-disasm | FileCheck %s -// CHECK: adrp x0, foo -// CHECK-NEXT: add x0, x0, :lo12:foo +// CHECK: adrp x1, foo +// CHECK-NEXT: add x1, x1, :lo12:foo Index: bolt/test/X86/Inputs/blarge.yaml =================================================================== --- bolt/test/X86/Inputs/blarge.yaml +++ bolt/test/X86/Inputs/blarge.yaml @@ -10,6 +10,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x00000000004005E0 + Offset: 0x5e0 AddressAlign: 0x0000000000000010 Content: FF35BA122000FF25BC1220000F1F4000FF25BA1220006800000000E9E0FFFFFFFF25B21220006801000000E9D0FFFFFFFF25AA1220006802000000E9C0FFFFFFFF25A21220006803000000E9B0FFFFFFFF259A1220006804000000E9A0FFFFFFFF25921220006805000000E990FFFFFFFF258A1220006806000000E980FFFFFFFF25821220006807000000E970FFFFFFFF257A1220006808000000E960FFFFFF - Name: .text @@ -28,6 +29,7 @@ Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x00000000006016B0 + Offset: 0x16B0 Link: .dynstr AddressAlign: 0x0000000000000008 Content: 01000000000000000100000000000000010000000000000072000000000000000C00000000000000C0054000000000000D000000000000003412400000000000190000000000000098166000000000001B0000000000000008000000000000001A00000000000000A0166000000000001C000000000000000800000000000000040000000000000048024000000000000500000000000000C803400000000000060000000000000090024000000000000A00000000000000AE000000000000000B000000000000001800000000000000150000000000000000000000000000000300000000000000981860000000000002000000000000000000000000000000140000000000000007000000000000001700000000000000E8044000000000000700000000000000D0044000000000000800000000000000000000000000000009000000000000001800000000000000FEFFFF6F000000009004400000000000FFFFFF6F000000000200000000000000F0FFFF6F000000007604400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 @@ -112,14 +114,15 @@ ProgramHeaders: - Type: PT_PHDR Flags: [ PF_X, PF_R ] - VAddr: 0x00000000004005E0 - PAddr: 0x00000000004005E0 - FirstSec: .noplt - LastSec: .noplt + VAddr: 0x0000000000400040 + PAddr: 0x0000000000400040 + FileSize: 0xe0 + MemSize: 0xe0 - Type: PT_LOAD Flags: [ PF_X, PF_R ] VAddr: 0x00000000004005E0 PAddr: 0x00000000004005E0 + Offset: 0x5e0 FirstSec: .noplt LastSec: .rodata - Type: PT_LOAD @@ -127,11 +130,11 @@ VAddr: 0x00000000006018F8 PAddr: 0x00000000006018F8 FirstSec: .data - LastSec: .bss - - Type: PT_DYNAMIC - Flags: [ PF_R, PF_W ] - VAddr: 0x00000000006016B0 - PAddr: 0x00000000006016B0 - FirstSec: .dynamic - LastSec: .dynamic + LastSec: .dynstr +# - Type: PT_DYNAMIC +# Flags: [ PF_R, PF_W ] +# VAddr: 0x00000000006016B0 +# PAddr: 0x00000000006016B0 +# FirstSec: .dynamic +# LastSec: .dynstr ... Index: bolt/test/X86/Inputs/broken_dynsym.yaml =================================================================== --- bolt/test/X86/Inputs/broken_dynsym.yaml +++ bolt/test/X86/Inputs/broken_dynsym.yaml @@ -7,7 +7,7 @@ ProgramHeaders: - Type: PT_LOAD FirstSec: .a - LastSec: .a + LastSec: .dynstr Align: 0x1000 Sections: - Name: .a Index: bolt/test/X86/Inputs/issue20.yaml =================================================================== --- bolt/test/X86/Inputs/issue20.yaml +++ bolt/test/X86/Inputs/issue20.yaml @@ -10,6 +10,7 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x00000000004003E0 + Offset: 0x3E0 AddressAlign: 0x0000000000000010 Content: 31ED4989D15E4889E24883E4F0505449C7C07005400048C7C10005400048C7C7CD044000E8B7FFFFFFF4660F1F440000B82F10600055482D281060004883F80E4889E577025DC3B8000000004885C074F45DBF28106000FFE00F1F8000000000B82810600055482D2810600048C1F8034889E54889C248C1EA3F4801D048D1F875025DC3BA000000004885D274F45D4889C6BF28106000FFE20F1F8000000000803D9D0B2000007511554889E5E87EFFFFFF5DC6058A0B200001F3C30F1F400048833D7809200000741EB8000000004885C0741455BF200E60004889E5FFD05DE97BFFFFFF0F1F00E973FFFFFF4831C0C34883E703FF24FD90054000B801000000EB13B802000000EB0CB803000000EB05B804000000C3660F1F84000000000041574189FF41564989F641554989D541544C8D25F808200055488D2DF8082000534C29E531DB48C1FD034883EC08E85DFEFFFF4885ED741E0F1F8400000000004C89EA4C89F64489FF41FF14DC4883C3014839EB75EA4883C4085B5D415C415D415E415FC390662E0F1F840000000000F3C3 - Name: .rodata @@ -17,11 +18,13 @@ Flags: [ SHF_ALLOC ] Address: 0x0000000000400580 AddressAlign: 0x0000000000000008 + Offset: 0x580 Content: 01000200000000000000000000000000DC04400000000000E304400000000000EA04400000000000F104400000000000 - Name: .dynamic Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x0000000000600E28 + Offset: 0xE28 Link: .dynstr AddressAlign: 0x0000000000000008 Content: 010000000000000001000000000000000C0000000000000090034000000000000D0000000000000074054000000000001900000000000000100E6000000000001B0000000000000008000000000000001A00000000000000180E6000000000001C000000000000000800000000000000F5FEFF6F000000009802400000000000050000000000000000034000000000000600000000000000B8024000000000000A0000000000000038000000000000000B0000000000000018000000000000001500000000000000000000000000000003000000000000000010600000000000020000000000000000000000000000001400000000000000070000000000000017000000000000007803400000000000070000000000000060034000000000000800000000000000000000000000000009000000000000001800000000000000FEFFFF6F000000004003400000000000FFFFFF6F000000000100000000000000F0FFFF6F000000003803400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 @@ -44,14 +47,22 @@ ProgramHeaders: - Type: PT_LOAD Flags: [ PF_X, PF_R ] - VAddr: 0x004003E0 - PAddr: 0x004003E0 + VAddr: 0x00400000 + PAddr: 0x00400000 + Offset: 0x0 FirstSec: .text - LastSec: .text - - Type: PT_DYNAMIC - Flags: [ PF_X, PF_R ] + LastSec: .rodata + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] VAddr: 0x00600E28 PAddr: 0x00600E28 + Offset: 0xE28 FirstSec: .dynamic - LastSec: .dynamic + LastSec: .dynstr +# - Type: PT_DYNAMIC +# Flags: [ PF_X, PF_R ] +# VAddr: 0x00600E28 +# PAddr: 0x00600E28 +# FirstSec: .dynamic +# LastSec: .dynamic ... Index: bolt/test/X86/Inputs/issue26.yaml =================================================================== --- bolt/test/X86/Inputs/issue26.yaml +++ bolt/test/X86/Inputs/issue26.yaml @@ -10,28 +10,31 @@ Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] Address: 0x00000000004003E0 + Offset: 0x3E0 AddressAlign: 0x0000000000000010 Content: 31ED4989D15E4889E24883E4F0505449C7C07005400048C7C10005400048C7C7FA044000E8B7FFFFFFF4660F1F440000B82F10600055482D281060004883F80E4889E577025DC3B8000000004885C074F45DBF28106000FFE00F1F8000000000B82810600055482D2810600048C1F8034889E54889C248C1EA3F4801D048D1F875025DC3BA000000004885D274F45D4889C6BF28106000FFE20F1F8000000000803D9D0B2000007511554889E5E87EFFFFFF5DC6058A0B200001F3C30F1F400048833D7809200000741EB8000000004885C0741455BF200E60004889E5FFD05DE97BFFFFFF0F1F00E973FFFFFF648B0425B0FCFFFF39C70F850C0000004839160F850400000048890EC3B8FFFFFFFFC34839FE0F84F0FFFFFFC34831C0C3669041574189FF41564989F641554989D541544C8D25F808200055488D2DF8082000534C29E531DB48C1FD034883EC08E85DFEFFFF4885ED741E0F1F8400000000004C89EA4C89F64489FF41FF14DC4883C3014839EB75EA4883C4085B5D415C415D415E415FC390662E0F1F840000000000F3C3 - - Name: .rela.text - Type: SHT_RELA - Flags: [ SHF_INFO_LINK ] - Link: .symtab - AddressAlign: 0x0000000000000008 - Info: .text - Relocations: - Name: .rodata Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] Address: 0x0000000000400580 + Offset: 0x580 AddressAlign: 0x0000000000000008 Content: '01000200000000000000000000000000' - Name: .dynamic Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x0000000000600E28 + Offset: 0xE28 Link: .dynstr AddressAlign: 0x0000000000000008 Content: 010000000000000001000000000000000C0000000000000090034000000000000D0000000000000074054000000000001900000000000000100E6000000000001B0000000000000008000000000000001A00000000000000180E6000000000001C000000000000000800000000000000F5FEFF6F000000009802400000000000050000000000000000034000000000000600000000000000B8024000000000000A0000000000000038000000000000000B0000000000000018000000000000001500000000000000000000000000000003000000000000000010600000000000020000000000000000000000000000001400000000000000070000000000000017000000000000007803400000000000070000000000000060034000000000000800000000000000000000000000000009000000000000001800000000000000FEFFFF6F000000004003400000000000FFFFFF6F000000000100000000000000F0FFFF6F000000003803400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + - Name: .rela.text + Type: SHT_RELA + Flags: [ SHF_INFO_LINK ] + Link: .symtab + AddressAlign: 0x0000000000000008 + Info: .text + Relocations: Symbols: - Name: FUNC Type: STT_FUNC @@ -57,22 +60,16 @@ Value: 0x0000000000400100 Binding: STB_GLOBAL ProgramHeaders: - - Type: PT_PHDR - Flags: [ PF_X, PF_R ] - VAddr: 0x004003E0 - PAddr: 0x004003E0 - FirstSec: .text - LastSec: .text - Type: PT_LOAD Flags: [ PF_X, PF_R ] - VAddr: 0x004003E0 - PAddr: 0x004003E0 + VAddr: 0x00400000 + PAddr: 0x00400000 + Offset: 0x0 FirstSec: .text - LastSec: .text - - Type: PT_DYNAMIC - Flags: [ PF_X, PF_R ] - VAddr: 0x00600E28 - PAddr: 0x00600E28 + LastSec: .rodata + - Type: PT_LOAD + Flags: [ PF_R, PF_W ] + VAddr: 0x600e28 FirstSec: .dynamic - LastSec: .dynamic + LastSec: .dynstr ... Index: bolt/test/X86/Inputs/plt-got-sec.yaml =================================================================== --- bolt/test/X86/Inputs/plt-got-sec.yaml +++ bolt/test/X86/Inputs/plt-got-sec.yaml @@ -10,6 +10,8 @@ Flags: [ PF_R ] VAddr: 0x400040 Align: 0x8 + FileSize: 0x2d8 + MemSize: 0x2d8 - Type: PT_INTERP Flags: [ PF_R ] FirstSec: .interp @@ -20,6 +22,7 @@ FirstSec: .interp LastSec: .rela.plt VAddr: 0x400000 + Offset: 0x0 Align: 0x1000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] Index: bolt/test/X86/Inputs/plt-mold.yaml =================================================================== --- bolt/test/X86/Inputs/plt-mold.yaml +++ bolt/test/X86/Inputs/plt-mold.yaml @@ -9,6 +9,8 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x40 + FileSize: 0x230 + MemSize: 0x230 Align: 0x8 - Type: PT_INTERP Flags: [ PF_R ] @@ -19,6 +21,8 @@ Flags: [ PF_R ] FirstSec: .interp LastSec: .rodata.str + VAddr: 0x0 + Offset: 0x0 Align: 0x1000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] Index: bolt/test/X86/Inputs/plt-sec-8-byte.yaml =================================================================== --- bolt/test/X86/Inputs/plt-sec-8-byte.yaml +++ bolt/test/X86/Inputs/plt-sec-8-byte.yaml @@ -9,7 +9,10 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x400040 + Offset: 0x40 Align: 0x8 + FileSize: 0x1F8 + MemSize: 0x1F8 - Type: PT_INTERP Flags: [ PF_R ] FirstSec: .interp @@ -19,6 +22,7 @@ Flags: [ PF_X, PF_R ] FirstSec: .interp LastSec: .eh_frame + Offset: 0x0 VAddr: 0x400000 Align: 0x200000 - Type: PT_LOAD Index: bolt/test/X86/Inputs/plt-sec.yaml =================================================================== --- bolt/test/X86/Inputs/plt-sec.yaml +++ bolt/test/X86/Inputs/plt-sec.yaml @@ -9,6 +9,9 @@ - Type: PT_PHDR Flags: [ PF_R ] VAddr: 0x400040 + Offset: 0x40 + FileSize: 0x2d8 + MemSize: 0x2d8 Align: 0x8 - Type: PT_INTERP Flags: [ PF_R ] @@ -20,18 +23,21 @@ FirstSec: .interp LastSec: .rela.plt VAddr: 0x400000 + Offset: 0x0 Align: 0x1000 - Type: PT_LOAD Flags: [ PF_X, PF_R ] FirstSec: .init LastSec: .fini VAddr: 0x401000 + Offset: 0x1000 Align: 0x1000 - Type: PT_LOAD Flags: [ PF_R ] FirstSec: .rodata LastSec: .eh_frame VAddr: 0x402000 + Offset: 0x2000 Align: 0x1000 - Type: PT_LOAD Flags: [ PF_W, PF_R ] Index: bolt/test/X86/Inputs/srol-bug-input.yaml =================================================================== --- bolt/test/X86/Inputs/srol-bug-input.yaml +++ bolt/test/X86/Inputs/srol-bug-input.yaml @@ -9,31 +9,34 @@ - Name: .text Type: SHT_PROGBITS Flags: [ SHF_ALLOC, SHF_EXECINSTR ] - Address: 0x0000000000400000 + Address: 0x0000000000400200 AddressAlign: 0x0000000000000010 + Offset: 0x200 Content: 0315fa0000002B15f40000002315ee0000000B15e80000003315e2000000500fB715da0000008A15d4000000668B15cd0000008B15c7000000488B15c00000003A15ba000000663B15b30000003B15ad000000483B15a60000008415a0000000668515990000008515930000004885158c000000C3 - Name: .rodata Type: SHT_PROGBITS Flags: [ SHF_ALLOC ] - Address: 0x0000000000400100 + Address: 0x0000000000400300 AddressAlign: 0x0000000000000100 + Offset: 0x300 Content: 010002000000000000000000 - Name: .dynamic Type: SHT_DYNAMIC Flags: [ SHF_WRITE, SHF_ALLOC ] Address: 0x000000000061ADA8 + Offset: 0xADA8 Link: .dynstr AddressAlign: 0x0000000000000008 Content: 01000000000000000100000000000000010000000000000096000000000000000100000000000000C400000000000000010000000000000001010000000000000C0000000000000028224000000000000D000000000000005C29410000000000190000000000000028A36100000000001B0000000000000008000000000000001A0000000000000030A36100000000001C000000000000000800000000000000F5FEFF6F0000000098024000000000000500000000000000300F4000000000000600000000000000D0024000000000000A00000000000000BC050000000000000B00000000000000180000000000000015000000000000000000000000000000030000000000000000B0610000000000020000000000000000000000000000001400000000000000070000000000000017000000000000006017400000000000070000000000000088164000000000000800000000000000000000000000000009000000000000001800000000000000FEFFFF6F00000000F815400000000000FFFFFF6F000000000200000000000000F0FFFF6F00000000EC14400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 Symbols: - Name: mydata Section: .rodata - Value: 0x0000000000400100 + Value: 0x0000000000400300 Binding: STB_GLOBAL - Name: myfunc Type: STT_FUNC Section: .text - Value: 0x0000000000400000 + Value: 0x0000000000400200 Binding: STB_GLOBAL DynamicSymbols: - Name: mydata @@ -41,22 +44,17 @@ Value: 0x0000000000400100 Binding: STB_GLOBAL ProgramHeaders: - - Type: PT_PHDR - Flags: [ PF_X, PF_R ] - VAddr: 0x00400000 - PAddr: 0x00400000 - FirstSec: .text - LastSec: .text - Type: PT_LOAD Flags: [ PF_X, PF_R ] VAddr: 0x00400000 PAddr: 0x00400000 + Offset: 0x0 FirstSec: .text - LastSec: .text - - Type: PT_DYNAMIC - Flags: [ PF_X, PF_R ] + LastSec: .rodata + - Type: PT_LOAD + Flags: [ PF_W, PF_R ] VAddr: 0x0061ADA8 - PAddr: 0x0064ADA8 + PAddr: 0x0061ADA8 FirstSec: .dynamic - LastSec: .dynamic + LastSec: .dynstr ... Index: bolt/test/X86/branch-data.test =================================================================== --- bolt/test/X86/branch-data.test +++ bolt/test/X86/branch-data.test @@ -10,7 +10,7 @@ CHECK: Address : 0x401170 CHECK: Size : 0x43 CHECK: MaxSize : 0x43 -CHECK: Offset : 0xcb0 +CHECK: Offset : 0x1170 CHECK: Section : .text CHECK: IsSimple : 1 CHECK: BB Count : 5 Index: bolt/test/X86/cfi-expr-rewrite.s =================================================================== --- bolt/test/X86/cfi-expr-rewrite.s +++ bolt/test/X86/cfi-expr-rewrite.s @@ -693,3 +693,4 @@ "blake2b_IV/1": "DATAat0x4013a0": "DATAat0x401380": +.long 0x0 Index: bolt/test/X86/double-rel.s =================================================================== --- bolt/test/X86/double-rel.s +++ bolt/test/X86/double-rel.s @@ -14,7 +14,7 @@ .data .globl VAR VAR: - .quad + .quad 0 .text .globl _start Index: bolt/test/X86/dummy-eh-frame-bug.s =================================================================== --- bolt/test/X86/dummy-eh-frame-bug.s +++ bolt/test/X86/dummy-eh-frame-bug.s @@ -9,7 +9,7 @@ ## after .text when no update is needed to .eh_frame. # CHECK: {{ .text}} PROGBITS [[#%x,ADDR:]] [[#%x,OFFSET:]] [[#%x,SIZE:]] -# CHECK-NEXT: 0000000000000000 [[#%x, OFFSET + SIZE]] +# CHECK-NEXT: 0000000000000000 [[#%.6x, OFFSET + SIZE]] .text .globl nocfi_function Index: bolt/test/X86/section-end-sym.s =================================================================== --- bolt/test/X86/section-end-sym.s +++ bolt/test/X86/section-end-sym.s @@ -9,10 +9,9 @@ # RUN: | FileCheck %s # CHECK: considering symbol etext for function -# CHECK-NEXT: rejecting as symbol points to end of its section +# CHECK-NEXT: BOLT-INFO: etext is in the end of .text # CHECK-NOT: Binary Function "etext{{.*}}" after building cfg - .text .globl _start .type _start,@function Index: bolt/test/reorder-data-writable-ptload.c =================================================================== --- bolt/test/reorder-data-writable-ptload.c +++ bolt/test/reorder-data-writable-ptload.c @@ -6,8 +6,8 @@ // RUN: -data %S/Inputs/reorder-data-writable-ptload.fdata // RUN: llvm-readelf -SlW %t.bolt | FileCheck %s -// CHECK: .bolt.org.data -// CHECK: {{.*}} .data PROGBITS [[#%x,ADDR:]] [[#%x,OFF:]] +// CHECK: .data +// CHECK: {{.*}} .data.hot PROGBITS [[#%x,ADDR:]] [[#%x,OFF:]] // CHECK: LOAD 0x{{.*}}[[#OFF]] 0x{{.*}}[[#ADDR]] {{.*}} RW volatile int cold1 = 42; Index: bolt/test/runtime/AArch64/adrrelaxationpass.s =================================================================== --- bolt/test/runtime/AArch64/adrrelaxationpass.s +++ bolt/test/runtime/AArch64/adrrelaxationpass.s @@ -48,10 +48,10 @@ .word 0xff # CHECK:
: -# CHECK-NEXT: adr x0, 0x{{[1-8a-f][0-9a-f]*}} -# CHECK-NEXT: adrp x1, 0x{{[1-8a-f][0-9a-f]*}} -# CHECK-NEXT: add x1, x1, #{{[1-8a-f][0-9a-f]*}} -# CHECK-NEXT: adrp x2, 0x{{[1-8a-f][0-9a-f]*}} -# CHECK-NEXT: add x2, x2, #{{[1-8a-f][0-9a-f]*}} -# CHECK-NEXT: adr x3, 0x{{[1-8a-f][0-9a-f]*}} +# CHECK-NEXT: adr x0, 0x{{[1-9a-f][0-9a-f]*}} +# CHECK-NEXT: adrp x1, 0x{{[1-9a-f][0-9a-f]*}} +# CHECK-NEXT: add x1, x1, #{{[1-9a-f][0-9a-f]*}} +# CHECK-NEXT: adrp x2, 0x{{[1-9a-f][0-9a-f]*}} +# CHECK-NEXT: add x2, x2, #{{[1-9a-f][0-9a-f]*}} +# CHECK-NEXT: adr x3, 0x{{[1-9a-f][0-9a-f]*}} # CHECK-ERROR: BOLT-ERROR: Cannot relax adr in non-simple function main Index: bolt/unittests/Core/BinaryContext.cpp =================================================================== --- bolt/unittests/Core/BinaryContext.cpp +++ bolt/unittests/Core/BinaryContext.cpp @@ -67,14 +67,17 @@ TEST_P(BinaryContextTester, BaseAddress) { // Check that base address calculation is correct for a binary with the // following segment layout: - BC->SegmentMapInfo[0] = SegmentInfo{0, 0x10e8c2b4, 0, 0x10e8c2b4, 0x1000}; + BC->SegmentMapInfo[0] = ProgramHeader(ELF::PT_LOAD, ELF::PF_R, 0, 0, 0, + 0x10e8c2b4, 0x10e8c2b4, 0x1000); BC->SegmentMapInfo[0x10e8d2b4] = - SegmentInfo{0x10e8d2b4, 0x3952faec, 0x10e8c2b4, 0x3952faec, 0x1000}; + ProgramHeader(ELF::PT_LOAD, ELF::PF_R, 0x10e8c2b4, 0x10e8d2b4, 0x10e8d2b4, + 0x3952faec, 0x3952faec, 0x1000); BC->SegmentMapInfo[0x4a3bddc0] = - SegmentInfo{0x4a3bddc0, 0x148e828, 0x4a3bbdc0, 0x148e828, 0x1000}; + ProgramHeader(ELF::PT_LOAD, ELF::PF_R, 0x4a3bbdc0, 0x4a3bddc0, 0x4a3bddc0, + 0x148e828, 0x148e828, 0x1000); BC->SegmentMapInfo[0x4b84d5e8] = - SegmentInfo{0x4b84d5e8, 0x294f830, 0x4b84a5e8, 0x3d3820, 0x1000}; - + ProgramHeader(ELF::PT_LOAD, ELF::PF_R, 0x4b84a5e8, 0x4b84d5e8, 0x4b84d5e8, + 0x3d3820, 0x294f830, 0x1000); std::optional BaseAddress = BC->getBaseAddressForMapping(0x7f13f5556000, 0x10e8c000); ASSERT_TRUE(BaseAddress.has_value());