diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -703,9 +703,16 @@ return Instructions.erase(II); } + /// Store meta-data of instruction to NOP instruction + /// and insert it in place of erased one + bool storeInstructionMetadata(iterator II); + /// Erase non-pseudo instruction at a given iterator \p II. /// Return iterator following the removed instruction. iterator eraseInstruction(iterator II) { + if (storeInstructionMetadata(II)) + return std::next(II); + adjustNumPseudos(*II, -1); return Instructions.erase(II); } @@ -744,12 +751,9 @@ /// Return iterator pointing to the first inserted instruction. template iterator replaceInstruction(iterator II, Itr Begin, Itr End) { - adjustNumPseudos(*II, -1); + II = eraseInstruction(II); adjustNumPseudos(Begin, End, 1); - - auto I = II - Instructions.begin(); - Instructions.insert(Instructions.erase(II), Begin, End); - return I + Instructions.begin(); + return Instructions.insert(II, Begin, End); } iterator replaceInstruction(iterator II, @@ -769,6 +773,14 @@ return Instructions.insert(At, NewInst); } + iterator insertInstructions(iterator At, InstructionListType &Instrs) { + for (MCInst &NewInst : Instrs) { + At = insertInstruction(At, NewInst); + ++At; + } + return At; + } + /// Helper to retrieve any terminators in \p BB before \p Pos. This is used /// to skip CFI instructions and to retrieve the first terminator instruction /// in basic blocks with two terminators (conditional jump and unconditional diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -19,6 +19,7 @@ #include "bolt/Core/JumpTable.h" #include "bolt/Core/MCPlusBuilder.h" #include "bolt/RuntimeLibs/RuntimeLibrary.h" +#include "bolt/Utils/NameResolver.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/iterator.h" @@ -233,6 +234,7 @@ void deregisterSectionName(const BinarySection &Section); public: + static Expected> createBinaryContext(const ObjectFile *File, bool IsPIC, std::unique_ptr DwCtx); @@ -479,6 +481,20 @@ return BinaryFunctions; } + /// Return BF by name. If the global function was not found return the first + /// found local + BinaryFunction *getBinaryFunctionByName(StringRef Name) { + BinaryData *Data = getFirstBinaryDataByName(Name); + if (!Data) + return nullptr; + + return getBinaryFunctionAtAddress(Data->getAddress()); + } + + const BinaryFunction *getBinaryFunctionByName(StringRef Name) const { + return const_cast(this)->getBinaryFunctionByName(Name); + } + /// Create BOLT-injected function BinaryFunction *createInjectedBinaryFunction(const std::string &Name, bool IsSimple = true); @@ -870,6 +886,19 @@ return nullptr; } + /// Return BinaryData for the given \p Name + /// If the data was not found return first local BinaryData or nullptr + BinaryData *getFirstBinaryDataByName(StringRef Name) { + BinaryData *Data = getBinaryDataByName(Name); + if (!Data) + return getBinaryDataByName(NameResolver::uniquifyID(Name, 1)); + return Data; + } + + const BinaryData *getFirstBinaryDataByName(StringRef Name) const { + return const_cast(this)->getFirstBinaryDataByName(Name); + } + /// Return true if \p SymbolName was generated internally and was not present /// in the input binary. bool isInternalSymbolName(const StringRef Name) { @@ -1220,6 +1249,9 @@ uint64_t computeInstructionSize(const MCInst &Inst, const MCCodeEmitter *Emitter = nullptr) const { + if (isAArch64()) + return 4; + if (auto Size = MIB->getAnnotationWithDefault(Inst, "Size")) return Size; @@ -1347,6 +1379,10 @@ /* DWARFMustBeAtTheEnd */ false)); return Streamer; } + + /// Returns std pair of binary functons desired and maximum alignment bytes + std::pair getBFAlignment(BinaryFunction &Function, + bool EmitColdPart) const; }; template > diff --git a/bolt/include/bolt/Core/BinaryData.h b/bolt/include/bolt/Core/BinaryData.h --- a/bolt/include/bolt/Core/BinaryData.h +++ b/bolt/include/bolt/Core/BinaryData.h @@ -53,6 +53,8 @@ uint64_t Address{0}; /// Size of this data (can be 0). uint64_t Size{0}; + /// Output size of this data (if changed). + uint64_t OutputSize{0}; /// Alignment of this data. uint16_t Alignment{1}; @@ -133,7 +135,8 @@ StringRef getOutputSectionName() const; uint64_t getOutputAddress() const; uint64_t getOutputOffset() const { return OutputOffset; } - uint64_t getOutputSize() const { return Size; } + uint64_t getOutputSize() const { return OutputSize ? OutputSize : Size; } + void setOutputSize(uint64_t NewSize) { OutputSize = NewSize; } bool isMoved() const; bool containsAddress(uint64_t Address) const { diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -75,6 +75,14 @@ ICP_ALL /// Perform ICP on calls and jump tables. }; +enum IndexOrder { + GO_FIRST_BF_INDEX = 0, + FIRST_BF_INDEX = 1, + GO_UNUSED_BF_INDEX = -3U, + GO_LAST_BF_INDEX = -2U, + INVALID_BF_INDEX = -1U, +}; + /// Information on a single indirect call to a particular callee. struct IndirectCallProfile { MCSymbol *Symbol; @@ -247,7 +255,7 @@ uint64_t MaxSize{std::numeric_limits::max()}; /// Alignment requirements for the function. - uint16_t Alignment{2}; + uint16_t Alignment{MinAlign}; /// Maximum number of bytes used for alignment of hot part of the function. uint16_t MaxAlignmentBytes{0}; @@ -255,6 +263,9 @@ /// Maximum number of bytes used for alignment of cold part of the function. uint16_t MaxColdAlignmentBytes{0}; + // If function is golang stores the offset to functab entry + uint32_t GolangFunctabOffset{0}; + const MCSymbol *PersonalityFunction{nullptr}; uint8_t PersonalityEncoding{dwarf::DW_EH_PE_sdata4 | dwarf::DW_EH_PE_pcrel}; @@ -356,6 +367,9 @@ /// True if another function body was merged into this one. bool HasFunctionsFoldedInto{false}; + /// True if we have a suspicion that the function was written in ASM + bool IsAsm{false}; + /// Name for the section this function code should reside in. std::string CodeSectionName; @@ -411,7 +425,7 @@ const MCSymbol *PLTSymbol{nullptr}; /// Function order for streaming into the destination binary. - uint32_t Index{-1U}; + uint32_t Index{INVALID_BF_INDEX}; /// Get basic block index assuming it belongs to this function. unsigned getIndex(const BinaryBasicBlock *BB) const { @@ -1053,15 +1067,14 @@ uint64_t getOutputSize() const { return OutputSize; } - /// Does this function have a valid streaming order index? - bool hasValidIndex() const { return Index != -1U; } + /// Returns true if this function has a valid streaming order index + bool hasValidIndex() const { return Index != INVALID_BF_INDEX; } /// Get the streaming order index for this function. uint32_t getIndex() const { return Index; } /// Set the streaming order index for this function. void setIndex(uint32_t Idx) { - assert(!hasValidIndex()); Index = Idx; } @@ -1299,6 +1312,9 @@ /// Return true if all CFG edges have local successors. bool hasCanonicalCFG() const { return HasCanonicalCFG; } + /// Return true if we have a suspicion that the function was written in ASM + bool isAsm() const { return IsAsm; } + /// Return true if the original function code has all necessary relocations /// to track addresses of functions emitted to new locations. bool hasExternalRefRelocations() const { return HasExternalRefRelocations; } @@ -1332,6 +1348,15 @@ /// Return true if other functions were folded into this one. bool hasFunctionsFoldedInto() const { return HasFunctionsFoldedInto; } + /// Sets golang funtab offset for this function + void setGolangFunctabOffset(uint32_t Offset) { GolangFunctabOffset = Offset; } + + /// Return true if the function is presented in original go functab + bool isGolang() const { return GolangFunctabOffset != 0; } + + /// Get golang funtab offset for this function + uint32_t getGolangFunctabOffset() const { return GolangFunctabOffset; } + /// If this function was folded, return the function it was folded into. BinaryFunction *getFoldedIntoFunction() const { return FoldedIntoFunction; } @@ -1691,6 +1716,8 @@ void setHasCanonicalCFG(bool V) { HasCanonicalCFG = V; } + void setIsAsm(bool V) { IsAsm = V; } + void setFolded(BinaryFunction *BF) { FoldedIntoFunction = BF; } /// Indicate that another function body was merged with this function. @@ -2241,14 +2268,14 @@ size_t Estimate = 0; for (const BinaryBasicBlock &BB : blocks()) if (BB.isCold()) - Estimate += BC.computeCodeSize(BB.begin(), BB.end()); + Estimate += BB.estimateSize(); return Estimate; } size_t estimateSize() const { size_t Estimate = 0; for (const BinaryBasicBlock &BB : blocks()) - Estimate += BC.computeCodeSize(BB.begin(), BB.end()); + Estimate += BB.estimateSize(); return Estimate; } diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -94,6 +94,8 @@ // Set by ExecutableFileMemoryManager. uint32_t Index{0}; // Section index in the output file. mutable bool IsReordered{false}; // Have the contents been reordered? + mutable bool IsChanged{false}; // The section is changed and OutputContents + // should be used for emission. bool IsAnonymous{false}; // True if the name should not be included // in the output file. @@ -286,6 +288,7 @@ } } bool isReordered() const { return IsReordered; } + bool isChanged() const { return IsChanged; } bool isAnonymous() const { return IsAnonymous; } unsigned getELFType() const { return ELFType; } unsigned getELFFlags() const { return ELFFlags; } @@ -341,6 +344,9 @@ /// Does this section have any pending relocations? bool hasPendingRelocations() const { return !PendingRelocations.empty(); } + /// Does this section have any dynamic relocations? + bool hasDynamicRelocations() const { return !DynamicRelocations.empty(); } + /// Remove non-pending relocation with the given /p Offset. bool removeRelocationAt(uint64_t Offset) { auto Itr = Relocations.find(Offset); @@ -348,6 +354,19 @@ Relocations.erase(Itr); return true; } + + return false; + } + + /// Remove the dynamic relocation (if any) at the given /p Offset. + bool removeDynamicRelocationAt(uint64_t Offset) { + Relocation Key{Offset, 0, 0, 0, 0}; + auto Itr = DynamicRelocations.find(Key); + if (Itr != DynamicRelocations.end()) { + DynamicRelocations.erase(Itr); + return true; + } + return false; } @@ -355,7 +374,7 @@ /// Add a new relocation at the given /p Offset. void addRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0, + uint64_t Addend = 0, uint64_t Value = 0, bool Pending = false) { assert(Offset < getSize() && "offset not within section bounds"); if (!Pending) { @@ -368,7 +387,7 @@ /// Add a dynamic relocation at the given /p Offset. void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0) { + uint64_t Addend = 0, uint64_t Value = 0) { assert(Offset < getSize() && "offset not within section bounds"); DynamicRelocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); } @@ -397,7 +416,7 @@ return Itr != Relocations.end() ? &*Itr : nullptr; } - /// Lookup the relocation (if any) at the given /p Offset. + /// Lookup the dynamic relocation (if any) at the given /p Offset. const Relocation *getDynamicRelocationAt(uint64_t Offset) const { Relocation Key{Offset, 0, 0, 0, 0}; auto Itr = DynamicRelocations.find(Key); @@ -424,6 +443,11 @@ const uint8_t *getOutputData() const { return reinterpret_cast(getOutputContents().data()); } + void setOutputContents(uint8_t *Data, uint64_t Size) { + IsChanged = true; + OutputSize = Size; + OutputContents = StringRef(reinterpret_cast(Data), Size); + } StringRef getOutputContents() const { return OutputContents; } uint64_t getAllocAddress() const { return reinterpret_cast(getOutputData()); diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -1722,6 +1722,21 @@ return Index; } + /// Copy annotations from one instruction to other + void copyAnnotationInst(const MCInst &From, MCInst &To, + AllocatorIdTy AllocatorId = 0) { + const MCInst *AnnotationInst = getAnnotationInst(From); + if (!AnnotationInst) + return; + + for (unsigned I = 0; I < AnnotationInst->getNumOperands(); ++I) { + const int64_t Imm = AnnotationInst->getOperand(I).getImm(); + const unsigned Index = extractAnnotationIndex(Imm); + const int64_t Value = extractAnnotationValue(Imm); + setAnnotationOpValue(To, Index, Value, AllocatorId); + } + } + /// Store an annotation value on an MCInst. This assumes the annotation /// is not already present. template diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -106,12 +106,18 @@ /// Return code for a PC-relative 8-byte relocation static uint64_t getPC64(); + /// Return code for a ABS 4-byte relocation + static uint64_t getAbs32(); + /// Return code for a ABS 8-byte relocation static uint64_t getAbs64(); /// Return code for a RELATIVE relocation static uint64_t getRelative(); + /// Return code for ABS relocation based on size + static uint64_t getAbs(uint8_t Size); + /// Return true if this relocation is PC-relative. Return false otherwise. bool isPCRelative() const { return isPCRelative(Type); } @@ -119,6 +125,9 @@ /// otherwise. bool isRelative() const { return isRelative(Type); } + /// Return true if this relocation is IRELATIVE type. Return false otherwise. + bool isIRelative() const { return isIRelative(Type); } + /// Emit relocation at a current \p Streamer' position. The caller is /// responsible for setting the position correctly. size_t emit(MCStreamer *Streamer) const; diff --git a/bolt/include/bolt/Utils/NameResolver.h b/bolt/include/bolt/Utils/NameResolver.h --- a/bolt/include/bolt/Utils/NameResolver.h +++ b/bolt/include/bolt/Utils/NameResolver.h @@ -29,9 +29,13 @@ public: /// Return unique version of the \p Name in the form "Name". + static std::string uniquifyID(StringRef Name, uint64_t ID) { + return (Name + Twine(Sep) + Twine(ID)).str(); + } + std::string uniquify(StringRef Name) { const uint64_t ID = ++Counters[Name]; - return (Name + Twine(Sep) + Twine(ID)).str(); + return uniquifyID(Name, ID); } /// For uniquified \p Name, return the original form (that may no longer be diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp --- a/bolt/lib/Core/BinaryBasicBlock.cpp +++ b/bolt/lib/Core/BinaryBasicBlock.cpp @@ -635,5 +635,21 @@ LocSyms.reset(nullptr); } +bool BinaryBasicBlock::storeInstructionMetadata(BinaryBasicBlock::iterator II) { + // In case the instruction has locked annotation we will save all + // its annotations to noop instruction and store it in place of + // erased instruction + BinaryContext &BC = Function->getBinaryContext(); + if (!BC.MIB->hasAnnotation(*II, "Locked")) + return false; + + MCInst Noop; + BC.MIB->createNoop(Noop); + std::unique_lock Lock(BC.CtxMutex); + BC.MIB->copyAnnotationInst(*II, Noop); + Instructions[std::distance(Instructions.begin(), II)] = Noop; + return true; +} + } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -2372,5 +2372,18 @@ return OutputRanges; } +std::pair +BinaryContext::getBFAlignment(BinaryFunction &Function, + bool EmitColdPart) const { + unsigned Alignment = Function.getAlignment(); + if (HasRelocations) { + unsigned MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes() + : Function.getMaxAlignmentBytes(); + return std::make_pair(Alignment, MaxAlignBytes); + } + + return std::make_pair(Alignment, Alignment); +} + } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -435,8 +435,8 @@ // Check if special alignment for macro-fusion is needed. bool MayNeedMacroFusionAlignment = - (opts::AlignMacroOpFusion == MFT_ALL) || - (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount()); + ((opts::AlignMacroOpFusion == MFT_ALL) || + (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount())); BinaryBasicBlock::const_iterator MacroFusionPair; if (MayNeedMacroFusionAlignment) { MacroFusionPair = BB->getMacroOpFusionPair(); diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp --- a/bolt/lib/Core/BinarySection.cpp +++ b/bolt/lib/Core/BinarySection.cpp @@ -16,6 +16,8 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/Support/CommandLine.h" +#include + #define DEBUG_TYPE "bolt" using namespace llvm; @@ -69,7 +71,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer, const Twine &SectionName) const { - StringRef SectionContents = getContents(); + StringRef SectionContents = isChanged() ? getOutputContents() : getContents(); MCSectionELF *ELFSection = BC.Ctx->getELFSection(SectionName, getELFType(), getELFFlags()); @@ -164,6 +166,11 @@ } BinarySection::~BinarySection() { + if (isChanged()) { + munmap(getOutputData(), getOutputSize()); + return; + } + if (isReordered()) { delete[] getData(); return; diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -636,6 +636,12 @@ return isPCRelativeX86(Type); } +uint64_t Relocation::getAbs32() { + if (Arch == Triple::aarch64) + return ELF::R_AARCH64_ABS32; + return ELF::R_X86_64_32; +} + uint64_t Relocation::getAbs64() { if (Arch == Triple::aarch64) return ELF::R_AARCH64_ABS64; @@ -648,6 +654,17 @@ return ELF::R_X86_64_RELATIVE; } +uint64_t Relocation::getAbs(uint8_t Size) { + if (Size == sizeof(uint64_t)) + return Relocation::getAbs64(); + else if (Size == sizeof(uint32_t)) + return Relocation::getAbs32(); + + llvm_unreachable("Wrong relocation size"); + return 0; + +} + size_t Relocation::emit(MCStreamer *Streamer) const { const size_t Size = getSizeForType(Type); MCContext &Ctx = Streamer->getContext(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -16,8 +16,10 @@ #include "bolt/Core/MCPlusBuilder.h" #include "bolt/Core/ParallelUtilities.h" #include "bolt/Core/Relocation.h" +#include "bolt/Passes/BinaryPasses.h" #include "bolt/Passes/CacheMetrics.h" #include "bolt/Passes/ReorderFunctions.h" +#include "bolt/Passes/SplitFunctions.h" #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Profile/DataAggregator.h" #include "bolt/Profile/DataReader.h" @@ -82,6 +84,15 @@ extern cl::list ReorderData; extern cl::opt ReorderFunctions; extern cl::opt TimeBuild; +extern cl::opt PreserveBlocksAlignment; +extern cl::opt AlignBlocks; +extern cl::opt NoInline; +extern cl::opt StringOps; +extern cl::opt FrameOptimization; +extern cl::opt SplitFunctions; +extern llvm::cl::opt InsertRetpolines; +extern cl::opt InstructionsLowering; +extern cl::list Peepholes; cl::opt AllowStripped("allow-stripped", cl::desc("allow processing of stripped binaries"), @@ -1848,6 +1859,13 @@ opts::HotTextMoveSections.addValue(".never_hugify"); } + if ((BC->isAArch64()) && + (opts::AlignBlocks || opts::PreserveBlocksAlignment)) { + errs() << "BOLT-WARNING: Disabling block alignment\n"; + opts::AlignBlocks = false; + opts::PreserveBlocksAlignment = false; + } + if (opts::UseOldText && !BC->OldTextSectionAddress) { errs() << "BOLT-WARNING: cannot use old .text as the section was not found" "\n"; @@ -3133,6 +3151,10 @@ void RewriteInstance::disassembleFunctions() { NamedRegionTimer T("disassembleFunctions", "disassemble functions", TimerGroupName, TimerGroupDesc, opts::TimeRewrite); + // Create annotation indices to allow lock-free execution + BC->MIB->getOrCreateAnnotationIndex("Size"); + BC->MIB->getOrCreateAnnotationIndex("Locked"); + for (auto &BFI : BC->getBinaryFunctions()) { BinaryFunction &Function = BFI.second; @@ -4228,6 +4250,7 @@ // Write/re-write program headers. Phnum = Obj.getHeader().e_phnum; + if (PHDRTableOffset) { // Writing new pheader table and adding one new entry for R+X segment. Phnum += 1; @@ -4334,6 +4357,7 @@ } AddedSegment = true; } + OS.write(reinterpret_cast(&NewPhdr), sizeof(NewPhdr)); } @@ -5026,9 +5050,10 @@ Function->getCodeSection(FF->getFragmentNum())->getIndex(); } else { // Check if the symbol belongs to moved data object and update it. - BinaryData *BD = opts::ReorderData.empty() - ? nullptr - : BC->getBinaryDataAtAddress(Symbol.st_value); + BinaryData *BD = + opts::ForceToDataRelocations || !opts::ReorderData.empty() + ? BC->getBinaryDataAtAddress(Symbol.st_value) + : nullptr; if (BD && BD->isMoved() && !BD->isJumpTable()) { assert((!BD->getSize() || !Symbol.st_size || Symbol.st_size == BD->getSize()) && @@ -5043,6 +5068,7 @@ << " (" << OutputSection.getIndex() << ")\n"); NewSymbol.st_shndx = OutputSection.getIndex(); NewSymbol.st_value = BD->getOutputAddress(); + NewSymbol.st_size = BD->getOutputSize(); } else { // Otherwise just update the section for the symbol. if (Symbol.st_shndx < ELF::SHN_LORESERVE) @@ -5169,6 +5195,24 @@ addSymbol("__hot_data_end"); } + auto addSectionSymbol = [&](uint64_t Address, unsigned Index) { + ELFSymTy Symbol; + Symbol.st_value = Address; + Symbol.st_shndx = Index; + Symbol.st_name = AddToStrTab(""); + Symbol.st_size = 0; + Symbol.st_other = 0; + Symbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_SECTION); + Symbols.emplace_back(Symbol); + }; + + for (BinarySection &Section : BC->allocatableSections()) { + if (Section.hasSectionRef()) + continue; + + addSectionSymbol(Section.getOutputAddress(), Section.getIndex()); + } + // Put local symbols at the beginning. llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) { if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL) @@ -5434,8 +5478,15 @@ getNewFunctionOrDataAddress(SectionInputAddress + Rel.Offset); RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset; - if (Rel.Symbol) { - SymbolIdx = getOutputDynamicSymbolIndex(Symbol); + if (Symbol) { + if (!IsRelative && !Rel.isIRelative()) { + SymbolIdx = getOutputDynamicSymbolIndex(Symbol); + } else { + // The R_*_(I)RELATIVE relocation inserted by BOLT + ErrorOr Address = BC->getSymbolValue(*Symbol); + if (Address) + Addend += getNewFunctionOrDataAddress(*Address); + } } else { // Usually this case is used for R_*_(I)RELATIVE relocations const uint64_t Address = getNewFunctionOrDataAddress(Addend);