diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -296,6 +296,16 @@ return make_range(Relocations.begin(), Relocations.end()); } + /// Iterate over all dynamic relocations for this section. + iterator_range dynamicRelocations() { + return make_range(DynamicRelocations.begin(), DynamicRelocations.end()); + } + + /// Iterate over all dynamic relocations for this section. + iterator_range dynamicRelocations() const { + return make_range(DynamicRelocations.begin(), DynamicRelocations.end()); + } + /// Does this section have any non-pending relocations? bool hasRelocations() const { return !Relocations.empty(); } diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -45,8 +45,10 @@ /// value of this relocation. uint64_t Addend; - /// The computed relocation value extracted from the binary file. - /// Used to validate relocation correctness. + /// The computed relocation value extracted from the binary file + /// for static relocations, that is used to validate relocation correctness. + /// The dynamic relocations stores the symbol value used to patch dynamic + /// relocations. uint64_t Value; /// Return size of the given relocation \p Type. @@ -68,6 +70,9 @@ /// if necessary. static uint64_t extractValue(uint64_t Type, uint64_t Contents, uint64_t PC); + /// Retyrn true of relocation should be locationed in jmprel section + static bool isJmpRel(uint64_t Type); + /// Return true if relocation type is PC-relative. Return false otherwise. static bool isPCRelative(uint64_t Type); @@ -89,6 +94,9 @@ /// Return true if relocation type is for thread local storage. static bool isTLS(uint64_t Type); + /// Return code for a NONE relocation + static uint64_t getNone(); + /// Return code for a PC-relative 4-byte relocation static uint64_t getPC32(); @@ -98,6 +106,10 @@ /// Return true if this relocation is PC-relative. Return false otherwise. bool isPCRelative() const { return isPCRelative(Type); } + /// Return true if this relocation is R_*_RELATIVE type. Return false + /// otherwise. + bool isRelative() const { return isRelative(Type); } + /// Emit relocation at a current \p Streamer' position. The caller is /// responsible for setting the position correctly. size_t emit(MCStreamer *Streamer) const; diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -201,6 +201,10 @@ /// \p OldAddress address in the original binary. uint64_t getNewFunctionAddress(uint64_t OldAddress); + /// Return address of a function or moved data in the new binary + /// corresponding to \p OldAddress address in the original binary. + uint64_t getNewFunctionOrDataAddress(uint64_t OldAddress); + /// Return value for the symbol \p Name in the output. uint64_t getNewValueForSymbol(const StringRef Name); @@ -426,6 +430,7 @@ /// Location and size of dynamic relocations. Optional DynamicRelocationsAddress; uint64_t DynamicRelocationsSize{0}; + uint64_t DynamicRelativeRelocationsCount{0}; /// PLT relocations are special kind of dynamic relocations stored separately. Optional PLTRelocationsAddress; diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -424,6 +424,26 @@ } } +bool isJmpRelX86(uint64_t Type) { + switch (Type) { + default: + return false; + case ELF::R_X86_64_JUMP_SLOT: + case ELF::R_X86_64_IRELATIVE: + return true; + } +} + +bool isJmpRelAArch64(uint64_t Type) { + switch (Type) { + default: + return false; + case ELF::R_AARCH64_JUMP_SLOT: + case ELF::R_AARCH64_IRELATIVE: + return true; + } +} + bool isPCRelativeX86(uint64_t Type) { switch (Type) { default: @@ -532,9 +552,7 @@ } bool Relocation::isNone(uint64_t Type) { - if (Arch == Triple::aarch64) - return Type == ELF::R_AARCH64_NONE; - return Type == ELF::R_X86_64_NONE; + return Type == getNone(); } bool Relocation::isRelative(uint64_t Type) { @@ -555,10 +573,16 @@ return isTLSX86(Type); } -bool Relocation::isPCRelative(uint64_t Type) { +bool Relocation::isJmpRel(uint64_t Type) { if (Arch == Triple::aarch64) - return isPCRelativeAArch64(Type); - return isPCRelativeX86(Type); + return isJmpRelAArch64(Type); + return isJmpRelX86(Type); +} + +uint64_t Relocation::getNone() { + if (Arch == Triple::aarch64) + return ELF::R_AARCH64_NONE; + return ELF::R_X86_64_NONE; } uint64_t Relocation::getPC32() { @@ -573,6 +597,12 @@ return ELF::R_X86_64_PC64; } +bool Relocation::isPCRelative(uint64_t Type) { + if (Arch == Triple::aarch64) + return isPCRelativeAArch64(Type); + return isPCRelativeX86(Type); +} + size_t Relocation::emit(MCStreamer *Streamer) const { const size_t Size = getSizeForType(Type); MCContext &Ctx = Streamer->getContext(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -1691,6 +1691,40 @@ auto *ELF64BE = cast(Obj); return getRelocationAddend(ELF64BE, Rel); } + +template +uint32_t getRelocationSymbol(const ELFObjectFile *Obj, + const RelocationRef &RelRef) { + using ELFShdrTy = typename ELFT::Shdr; + uint32_t Symbol = 0; + const ELFFile &EF = Obj->getELFFile(); + DataRefImpl Rel = RelRef.getRawDataRefImpl(); + const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a)); + switch (RelocationSection->sh_type) { + default: + llvm_unreachable("unexpected relocation section type"); + case ELF::SHT_REL: + Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL()); + break; + case ELF::SHT_RELA: + Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL()); + break; + } + + return Symbol; +} + +uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj, + const RelocationRef &Rel) { + if (auto *ELF32LE = dyn_cast(Obj)) + return getRelocationSymbol(ELF32LE, Rel); + if (auto *ELF64LE = dyn_cast(Obj)) + return getRelocationSymbol(ELF64LE, Rel); + if (auto *ELF32BE = dyn_cast(Obj)) + return getRelocationSymbol(ELF32BE, Rel); + auto *ELF64BE = cast(Obj); + return getRelocationSymbol(ELF64BE, Rel); +} } // anonymous namespace bool RewriteInstance::analyzeRelocation( @@ -2075,6 +2109,7 @@ MCSymbol *Symbol = nullptr; uint64_t SymbolAddress = 0; const uint64_t Addend = getRelocationAddend(InputFile, Rel); + const uint32_t SymbolIdx = getRelocationSymbol(InputFile, Rel); symbol_iterator SymbolIter = Rel.getSymbol(); if (SymbolIter != InputFile->symbol_end()) { @@ -2095,7 +2130,8 @@ << " : + 0x" << Twine::utohexstr(Addend) << '\n' ); - BC->addDynamicRelocation(Rel.getOffset(), Symbol, Rel.getType(), Addend); + BC->addDynamicRelocation(Rel.getOffset(), Symbol, Rel.getType(), Addend, + SymbolIdx); } } @@ -4709,28 +4745,123 @@ RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile *File) { using Elf_Rela = typename ELFT::Rela; raw_fd_ostream &OS = Out->os(); + const ELFFile &EF = File->getELFFile(); - for (BinarySection &RelaSection : BC->allocatableRelaSections()) { - for (const RelocationRef &Rel : RelaSection.getSectionRef().relocations()) { - uint64_t RType = Rel.getType(); - if (!Relocation::isRelative(RType) && !Relocation::isIRelative(RType)) - continue; - DataRefImpl DRI = Rel.getRawDataRefImpl(); - const Elf_Rela *RelA = File->getRela(DRI); - auto Address = RelA->r_addend; - uint64_t NewAddress = getNewFunctionAddress(Address); - if (!NewAddress) - continue; - LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching (I)RELATIVE " - << RelaSection.getName() << " entry 0x" - << Twine::utohexstr(Address) << " with 0x" - << Twine::utohexstr(NewAddress) << '\n'); - Elf_Rela NewRelA = *RelA; - NewRelA.r_addend = NewAddress; - OS.pwrite(reinterpret_cast(&NewRelA), sizeof(NewRelA), - reinterpret_cast(RelA) - File->getData().data()); + uint64_t RelDynOffset = 0, RelDynEndOffset = 0; + uint64_t RelPltOffset = 0, RelPltEndOffset = 0; + + auto getRelOffset = [&](uint64_t Type) -> uint64_t & { + if (Relocation::isJmpRel(Type)) + return RelPltOffset; + return RelDynOffset; + }; + + auto getRelEndOffset = [&](uint64_t Type) -> uint64_t & { + if (Relocation::isJmpRel(Type)) + return RelPltEndOffset; + return RelDynEndOffset; + }; + + auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start, + uint64_t &End) { + ErrorOr Section = BC->getSectionForAddress(Address); + StringRef SectionContents = + cantFail(Section->getSectionRef().getContents()); + Start = SectionContents.data() - File->getData().data(); + End = Start + Section->getSize(); + }; + + if (!DynamicRelocationsAddress && !PLTRelocationsAddress) + return; + + if (DynamicRelocationsAddress) + setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset, + RelDynEndOffset); + + if (PLTRelocationsAddress) + setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset, + RelPltEndOffset); + + DynamicRelativeRelocationsCount = 0; + + auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) { + OS.pwrite(reinterpret_cast(RelA), sizeof(*RelA), Offset); + Offset += sizeof(*RelA); + }; + + auto writeRelocations = [&](bool PatchRelative) { + for (BinarySection &Section : BC->allocatableSections()) { + for (const Relocation &Rel : Section.dynamicRelocations()) { + bool IsRelative = Rel.isRelative(); + if (PatchRelative != IsRelative) + continue; + + if (IsRelative) + ++DynamicRelativeRelocationsCount; + + Elf_Rela NewRelA; + uint64_t SectionAddress = Section.getOutputAddress(); + SectionAddress = + SectionAddress == 0 ? Section.getAddress() : SectionAddress; + uint64_t Addend = Rel.Addend; + uint32_t SymbolIdx = (uint32_t)Rel.Value; + if (!SymbolIdx) { + if (Rel.Symbol) { + // This is the case when we are creating relocation by ourselves + // during bolt work + ErrorOr Address = BC->getSymbolValue(*Rel.Symbol); + if (Address) + Addend += getNewFunctionOrDataAddress(*Address); + } else { + // Usually this case is used for R_*_(I)RELATIVE relocations + uint64_t Address = getNewFunctionOrDataAddress(Addend); + if (Address) + Addend = Address; + } + } + + NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL()); + NewRelA.r_offset = SectionAddress + Rel.Offset; + NewRelA.r_addend = Addend; + + uint64_t &Offset = getRelOffset(Rel.Type); + uint64_t &EndOffset = getRelEndOffset(Rel.Type); + if (!Offset || !EndOffset) { + errs() << "BOLT_ERROR: Invalid offsets for dynamic relocation\n"; + exit(1); + } + + if (Offset + sizeof(NewRelA) > EndOffset) { + errs() << "BOLT_ERROR: Offset overflow for dynamic relocation\n"; + exit(1); + } + + writeRela(&NewRelA, Offset); + } } - } + }; + + // The dynamic linker expects R_*_RELATIVE relocations to be emitted first + writeRelocations(/* PatchRelative */ true); + writeRelocations(/* PatchRelative */ false); + + auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) { + if (!Offset) + return; + + typename ELFObjectFile::Elf_Rela RelA; + RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL()); + RelA.r_offset = 0; + RelA.r_addend = 0; + while (Offset < EndOffset) + writeRela(&RelA, Offset); + + assert(Offset == EndOffset && "Unexpected section overflow"); + }; + + // Fill the rest of the sections with R_*_NONE relocations + fillNone(RelDynOffset, RelDynEndOffset); + fillNone(RelPltOffset, RelPltEndOffset); } template @@ -4746,7 +4877,8 @@ } } if (!GOTSection.getObject()) { - errs() << "BOLT-INFO: no .got section found\n"; + if (!BC->IsStaticExecutable) + errs() << "BOLT-INFO: no .got section found\n"; return; } @@ -4805,6 +4937,9 @@ default: ShouldPatch = false; break; + case ELF::DT_RELACOUNT: + NewDE.d_un.d_val = DynamicRelativeRelocationsCount; + break; case ELF::DT_INIT: case ELF::DT_FINI: { if (BC->HasRelocations) { @@ -4907,14 +5042,21 @@ case ELF::DT_PLTRELSZ: PLTRelocationsSize = Dyn.getVal(); break; + case ELF::DT_RELACOUNT: + DynamicRelativeRelocationsCount = Dyn.getVal(); + break; } } - if (!DynamicRelocationsAddress) + if (!DynamicRelocationsAddress || !DynamicRelocationsSize) { + DynamicRelocationsAddress.reset(); DynamicRelocationsSize = 0; + } - if (!PLTRelocationsAddress) + if (!PLTRelocationsAddress || !PLTRelocationsSize) { + PLTRelocationsAddress.reset(); PLTRelocationsSize = 0; + } } uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) { @@ -4927,6 +5069,17 @@ return Function->getOutputAddress(); } +uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) { + if (uint64_t Function = getNewFunctionAddress(OldAddress)) + return Function; + + const BinaryData *BD = BC->getBinaryDataAtAddress(OldAddress); + if (BD && BD->isMoved()) + return BD->getOutputAddress(); + + return 0; +} + void RewriteInstance::rewriteFile() { std::error_code EC; Out = std::make_unique(opts::OutputFilename, EC, @@ -5099,14 +5252,14 @@ // Copy non-allocatable sections once allocatable part is finished. rewriteNoteSections(); - // Patch dynamic section/segment. - patchELFDynamic(); - if (BC->HasRelocations) { patchELFAllocatableRelaSections(); patchELFGOT(); } + // Patch dynamic section/segment. + patchELFDynamic(); + // Update ELF book-keeping info. patchELFSectionHeaderTable();