diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -1028,7 +1028,7 @@ /// Register dynamic relocation at \p Address. void addDynamicRelocation(uint64_t Address, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0); + uint64_t Addend, uint64_t Value, bool IsJmpRel); /// Return a dynamic relocation registered at a given \p Address, or nullptr /// if there is no dynamic relocation at such address. diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -296,6 +296,16 @@ return make_range(Relocations.begin(), Relocations.end()); } + /// Iterate over all dynamic relocations for this section. + iterator_range dynamicRelocations() { + return make_range(DynamicRelocations.begin(), DynamicRelocations.end()); + } + + /// Iterate over all dynamic relocations for this section. + iterator_range dynamicRelocations() const { + return make_range(DynamicRelocations.begin(), DynamicRelocations.end()); + } + /// Does this section have any non-pending relocations? bool hasRelocations() const { return !Relocations.empty(); } @@ -329,9 +339,10 @@ /// Add a dynamic relocation at the given /p Offset. void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0) { + uint64_t Addend, uint64_t Value, bool IsJmpRel) { assert(Offset < getSize() && "offset not within section bounds"); - DynamicRelocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); + DynamicRelocations.emplace( + Relocation{Offset, Symbol, Type, Addend, Value, IsJmpRel}); } /// Add relocation against the original contents of this section. diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -45,10 +45,16 @@ /// value of this relocation. uint64_t Addend; - /// The computed relocation value extracted from the binary file. - /// Used to validate relocation correctness. + /// The computed relocation value extracted from the binary file + /// for static relocations, that is used to validate relocation correctness. + /// The dynamic relocations stores the symbol value used to patch dynamic + /// relocations. uint64_t Value; + /// The flag is used for dynamic relocations and is set to true if + /// the relocation is read from DT_JMPREL, otherwise it came from DT_RELA. + bool IsJmpRel = 0; + /// Return size of the given relocation \p Type. static size_t getSizeForType(uint64_t Type); @@ -89,6 +95,9 @@ /// Return true if relocation type is for thread local storage. static bool isTLS(uint64_t Type); + /// Return code for a NONE relocation + static uint64_t getNone(); + /// Return code for a PC-relative 4-byte relocation static uint64_t getPC32(); @@ -98,6 +107,10 @@ /// Return true if this relocation is PC-relative. Return false otherwise. bool isPCRelative() const { return isPCRelative(Type); } + /// Return true if this relocation is R_*_RELATIVE type. Return false + /// otherwise. + bool isRelative() const { return isRelative(Type); } + /// Emit relocation at a current \p Streamer' position. The caller is /// responsible for setting the position correctly. size_t emit(MCStreamer *Streamer) const; diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -124,7 +124,7 @@ void processLKSMPLocks(); /// Read relocations from a given section. - void readDynamicRelocations(const object::SectionRef &Section); + void readDynamicRelocations(const object::SectionRef &Section, bool IsJmpRel); /// Read relocations from a given section. void readRelocations(const object::SectionRef &Section); @@ -201,6 +201,10 @@ /// \p OldAddress address in the original binary. uint64_t getNewFunctionAddress(uint64_t OldAddress); + /// Return address of a function or moved data in the new binary + /// corresponding to \p OldAddress address in the original binary. + uint64_t getNewFunctionOrDataAddress(uint64_t OldAddress); + /// Return value for the symbol \p Name in the output. uint64_t getNewValueForSymbol(const StringRef Name); @@ -426,6 +430,7 @@ /// Location and size of dynamic relocations. Optional DynamicRelocationsAddress; uint64_t DynamicRelocationsSize{0}; + uint64_t DynamicRelativeRelocationsCount{0}; /// PLT relocations are special kind of dynamic relocations stored separately. Optional PLTRelocationsAddress; diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -1831,11 +1831,11 @@ void BinaryContext::addDynamicRelocation(uint64_t Address, MCSymbol *Symbol, uint64_t Type, uint64_t Addend, - uint64_t Value) { + uint64_t Value, bool IsJmpRel) { ErrorOr Section = getSectionForAddress(Address); assert(Section && "cannot find section for address"); Section->addDynamicRelocation(Address - Section->getAddress(), Symbol, Type, - Addend, Value); + Addend, Value, IsJmpRel); } bool BinaryContext::removeRelocationAt(uint64_t Address) { diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -531,11 +531,7 @@ return isGOTX86(Type); } -bool Relocation::isNone(uint64_t Type) { - if (Arch == Triple::aarch64) - return Type == ELF::R_AARCH64_NONE; - return Type == ELF::R_X86_64_NONE; -} +bool Relocation::isNone(uint64_t Type) { return Type == getNone(); } bool Relocation::isRelative(uint64_t Type) { if (Arch == Triple::aarch64) @@ -555,10 +551,10 @@ return isTLSX86(Type); } -bool Relocation::isPCRelative(uint64_t Type) { +uint64_t Relocation::getNone() { if (Arch == Triple::aarch64) - return isPCRelativeAArch64(Type); - return isPCRelativeX86(Type); + return ELF::R_AARCH64_NONE; + return ELF::R_X86_64_NONE; } uint64_t Relocation::getPC32() { @@ -573,6 +569,12 @@ return ELF::R_X86_64_PC64; } +bool Relocation::isPCRelative(uint64_t Type) { + if (Arch == Triple::aarch64) + return isPCRelativeAArch64(Type); + return isPCRelativeX86(Type); +} + size_t Relocation::emit(MCStreamer *Streamer) const { const size_t Size = getSizeForType(Type); MCContext &Ctx = Streamer->getContext(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -1691,6 +1691,40 @@ auto *ELF64BE = cast(Obj); return getRelocationAddend(ELF64BE, Rel); } + +template +uint32_t getRelocationSymbol(const ELFObjectFile *Obj, + const RelocationRef &RelRef) { + using ELFShdrTy = typename ELFT::Shdr; + uint32_t Symbol = 0; + const ELFFile &EF = Obj->getELFFile(); + DataRefImpl Rel = RelRef.getRawDataRefImpl(); + const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a)); + switch (RelocationSection->sh_type) { + default: + llvm_unreachable("unexpected relocation section type"); + case ELF::SHT_REL: + Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL()); + break; + case ELF::SHT_RELA: + Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL()); + break; + } + + return Symbol; +} + +uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj, + const RelocationRef &Rel) { + if (auto *ELF32LE = dyn_cast(Obj)) + return getRelocationSymbol(ELF32LE, Rel); + if (auto *ELF64LE = dyn_cast(Obj)) + return getRelocationSymbol(ELF64LE, Rel); + if (auto *ELF32BE = dyn_cast(Obj)) + return getRelocationSymbol(ELF32BE, Rel); + auto *ELF64BE = cast(Obj); + return getRelocationSymbol(ELF64BE, Rel); +} } // anonymous namespace bool RewriteInstance::analyzeRelocation( @@ -1820,7 +1854,8 @@ if (PLTRelSectionOrErr->getSize() != PLTRelocationsSize) report_error("section size mismatch for DT_PLTRELSZ", errc::executable_format_error); - readDynamicRelocations(PLTRelSectionOrErr->getSectionRef()); + readDynamicRelocations(PLTRelSectionOrErr->getSectionRef(), + /*IsJmpRel*/ true); } // The rest of dynamic relocations - DT_RELA. @@ -1833,7 +1868,8 @@ if (DynamicRelSectionOrErr->getSize() != DynamicRelocationsSize) report_error("section size mismatch for DT_RELASZ", errc::executable_format_error); - readDynamicRelocations(DynamicRelSectionOrErr->getSectionRef()); + readDynamicRelocations(DynamicRelSectionOrErr->getSectionRef(), + /*IsJmpRel*/ false); } } @@ -2057,7 +2093,8 @@ } } -void RewriteInstance::readDynamicRelocations(const SectionRef &Section) { +void RewriteInstance::readDynamicRelocations(const SectionRef &Section, + bool IsJmpRel) { assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected"); LLVM_DEBUG({ @@ -2075,6 +2112,7 @@ MCSymbol *Symbol = nullptr; uint64_t SymbolAddress = 0; const uint64_t Addend = getRelocationAddend(InputFile, Rel); + const uint32_t SymbolIdx = getRelocationSymbol(InputFile, Rel); symbol_iterator SymbolIter = Rel.getSymbol(); if (SymbolIter != InputFile->symbol_end()) { @@ -2095,7 +2133,8 @@ << " : + 0x" << Twine::utohexstr(Addend) << '\n' ); - BC->addDynamicRelocation(Rel.getOffset(), Symbol, Rel.getType(), Addend); + BC->addDynamicRelocation(Rel.getOffset(), Symbol, Rel.getType(), Addend, + SymbolIdx, IsJmpRel); } } @@ -4708,28 +4747,111 @@ RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile *File) { using Elf_Rela = typename ELFT::Rela; raw_fd_ostream &OS = Out->os(); + const ELFFile &EF = File->getELFFile(); - for (BinarySection &RelaSection : BC->allocatableRelaSections()) { - for (const RelocationRef &Rel : RelaSection.getSectionRef().relocations()) { - uint64_t RType = Rel.getType(); - if (!Relocation::isRelative(RType) && !Relocation::isIRelative(RType)) - continue; - DataRefImpl DRI = Rel.getRawDataRefImpl(); - const Elf_Rela *RelA = File->getRela(DRI); - auto Address = RelA->r_addend; - uint64_t NewAddress = getNewFunctionAddress(Address); - if (!NewAddress) - continue; - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching (I)RELATIVE " - << RelaSection.getName() << " entry 0x" - << Twine::utohexstr(Address) << " with 0x" - << Twine::utohexstr(NewAddress) << '\n'); - Elf_Rela NewRelA = *RelA; - NewRelA.r_addend = NewAddress; - OS.pwrite(reinterpret_cast(&NewRelA), sizeof(NewRelA), - reinterpret_cast(RelA) - File->getData().data()); + uint64_t RelDynOffset = 0, RelDynEndOffset = 0; + uint64_t RelPltOffset = 0, RelPltEndOffset = 0; + + auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start, + uint64_t &End) { + ErrorOr Section = BC->getSectionForAddress(Address); + StringRef SectionContents = + cantFail(Section->getSectionRef().getContents()); + Start = SectionContents.data() - File->getData().data(); + End = Start + Section->getSize(); + }; + + if (!DynamicRelocationsAddress && !PLTRelocationsAddress) + return; + + if (DynamicRelocationsAddress) + setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset, + RelDynEndOffset); + + if (PLTRelocationsAddress) + setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset, + RelPltEndOffset); + + DynamicRelativeRelocationsCount = 0; + + auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) { + OS.pwrite(reinterpret_cast(RelA), sizeof(*RelA), Offset); + Offset += sizeof(*RelA); + }; + + auto writeRelocations = [&](bool PatchRelative) { + for (BinarySection &Section : BC->allocatableSections()) { + for (const Relocation &Rel : Section.dynamicRelocations()) { + bool IsRelative = Rel.isRelative(); + if (PatchRelative != IsRelative) + continue; + + if (IsRelative) + ++DynamicRelativeRelocationsCount; + + Elf_Rela NewRelA; + uint64_t SectionAddress = Section.getOutputAddress(); + SectionAddress = + SectionAddress == 0 ? Section.getAddress() : SectionAddress; + uint64_t Addend = Rel.Addend; + uint32_t SymbolIdx = (uint32_t)Rel.Value; + if (!SymbolIdx) { + if (Rel.Symbol) { + // This is the case when we are creating relocation by ourselves + // during bolt work + ErrorOr Address = BC->getSymbolValue(*Rel.Symbol); + if (Address) + Addend += getNewFunctionOrDataAddress(*Address); + } else { + // Usually this case is used for R_*_(I)RELATIVE relocations + uint64_t Address = getNewFunctionOrDataAddress(Addend); + if (Address) + Addend = Address; + } + } + + NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL()); + NewRelA.r_offset = SectionAddress + Rel.Offset; + NewRelA.r_addend = Addend; + + uint64_t &Offset = Rel.IsJmpRel ? RelPltOffset : RelDynOffset; + uint64_t &EndOffset = Rel.IsJmpRel ? RelPltEndOffset : RelDynEndOffset; + if (!Offset || !EndOffset) { + errs() << "BOLT_ERROR: Invalid offsets for dynamic relocation\n"; + exit(1); + } + + if (Offset + sizeof(NewRelA) > EndOffset) { + errs() << "BOLT_ERROR: Offset overflow for dynamic relocation\n"; + exit(1); + } + + writeRela(&NewRelA, Offset); + } } - } + }; + + // The dynamic linker expects R_*_RELATIVE relocations to be emitted first + writeRelocations(/* PatchRelative */ true); + writeRelocations(/* PatchRelative */ false); + + auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) { + if (!Offset) + return; + + typename ELFObjectFile::Elf_Rela RelA; + RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL()); + RelA.r_offset = 0; + RelA.r_addend = 0; + while (Offset < EndOffset) + writeRela(&RelA, Offset); + + assert(Offset == EndOffset && "Unexpected section overflow"); + }; + + // Fill the rest of the sections with R_*_NONE relocations + fillNone(RelDynOffset, RelDynEndOffset); + fillNone(RelPltOffset, RelPltEndOffset); } template @@ -4745,7 +4867,8 @@ } } if (!GOTSection.getObject()) { - errs() << "BOLT-INFO: no .got section found\n"; + if (!BC->IsStaticExecutable) + errs() << "BOLT-INFO: no .got section found\n"; return; } @@ -4804,6 +4927,9 @@ default: ShouldPatch = false; break; + case ELF::DT_RELACOUNT: + NewDE.d_un.d_val = DynamicRelativeRelocationsCount; + break; case ELF::DT_INIT: case ELF::DT_FINI: { if (BC->HasRelocations) { @@ -4906,14 +5032,21 @@ case ELF::DT_PLTRELSZ: PLTRelocationsSize = Dyn.getVal(); break; + case ELF::DT_RELACOUNT: + DynamicRelativeRelocationsCount = Dyn.getVal(); + break; } } - if (!DynamicRelocationsAddress) + if (!DynamicRelocationsAddress || !DynamicRelocationsSize) { + DynamicRelocationsAddress.reset(); DynamicRelocationsSize = 0; + } - if (!PLTRelocationsAddress) + if (!PLTRelocationsAddress || !PLTRelocationsSize) { + PLTRelocationsAddress.reset(); PLTRelocationsSize = 0; + } } uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) { @@ -4926,6 +5059,17 @@ return Function->getOutputAddress(); } +uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) { + if (uint64_t Function = getNewFunctionAddress(OldAddress)) + return Function; + + const BinaryData *BD = BC->getBinaryDataAtAddress(OldAddress); + if (BD && BD->isMoved()) + return BD->getOutputAddress(); + + return 0; +} + void RewriteInstance::rewriteFile() { std::error_code EC; Out = std::make_unique(opts::OutputFilename, EC, @@ -5098,14 +5242,14 @@ // Copy non-allocatable sections once allocatable part is finished. rewriteNoteSections(); - // Patch dynamic section/segment. - patchELFDynamic(); - if (BC->HasRelocations) { patchELFAllocatableRelaSections(); patchELFGOT(); } + // Patch dynamic section/segment. + patchELFDynamic(); + // Update ELF book-keeping info. patchELFSectionHeaderTable();