diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -1036,7 +1036,9 @@ /// Return a relocation registered at a given \p Address, or nullptr if there /// is no relocation at such address. - const Relocation *getRelocationAt(uint64_t Address); + const Relocation *getRelocationAt(uint64_t Address) const; + + Relocation *getRelocationAt(uint64_t Address); /// Register a presence of PC-relative relocation at the given \p Address. void addPCRelativeDataRelocation(uint64_t Address) { @@ -1049,7 +1051,9 @@ /// Return a dynamic relocation registered at a given \p Address, or nullptr /// if there is no dynamic relocation at such address. - const Relocation *getDynamicRelocationAt(uint64_t Address); + const Relocation *getDynamicRelocationAt(uint64_t Address) const; + + Relocation *getDynamicRelocationAt(uint64_t Address); /// Remove registered relocation at a given \p Address. bool removeRelocationAt(uint64_t Address); diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -56,7 +56,7 @@ // Relocations associated with this section. Relocation offsets are // wrt. to the original section address and size. - using RelocationSetType = std::set>; + using RelocationSetType = std::map; RelocationSetType Relocations; // Dynamic relocations associated with this section. Relocation offsets are @@ -286,7 +286,6 @@ return containsAddress(Address) && Address + Size <= getEndAddress(); } - /// Iterate over all non-pending relocations for this section. iterator_range relocations() { return make_range(Relocations.begin(), Relocations.end()); } @@ -330,7 +329,7 @@ bool Pending = false) { assert(Offset < getSize() && "offset not within section bounds"); if (!Pending) { - Relocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); + Relocations[Offset] = Relocation{Offset, Symbol, Type, Addend, Value}; } else { PendingRelocations.emplace_back( Relocation{Offset, Symbol, Type, Addend, Value}); @@ -341,7 +340,8 @@ void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, uint64_t Addend, uint64_t Value = 0) { assert(Offset < getSize() && "offset not within section bounds"); - DynamicRelocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); + DynamicRelocations[Offset] = + Relocation{Offset, Symbol, Type, Addend, Value}; } /// Add relocation against the original contents of this section. @@ -363,16 +363,23 @@ BinaryPatcher *getPatcher() { return Patcher.get(); } /// Lookup the relocation (if any) at the given /p Offset. - const Relocation *getRelocationAt(uint64_t Offset) const { + Relocation *getRelocationAt(uint64_t Offset) { auto Itr = Relocations.find(Offset); - return Itr != Relocations.end() ? &*Itr : nullptr; + return Itr != Relocations.end() ? &Itr->second : nullptr; + } + + const Relocation *getRelocationAt(uint64_t Offset) const { + return const_cast(this)->getRelocationAt(Offset); } /// Lookup the relocation (if any) at the given /p Offset. + Relocation *getDynamicRelocationAt(uint64_t Offset) { + auto Itr = DynamicRelocations.find(Offset); + return Itr != DynamicRelocations.end() ? &Itr->second : nullptr; + } + const Relocation *getDynamicRelocationAt(uint64_t Offset) const { - Relocation Key{Offset, 0, 0, 0, 0}; - auto Itr = DynamicRelocations.find(Key); - return Itr != DynamicRelocations.end() ? &*Itr : nullptr; + return const_cast(this)->getDynamicRelocationAt(Offset); } uint64_t hash(const BinaryData &BD) const { @@ -421,7 +428,7 @@ /// Emit the section as data, possibly with relocations. Use name \p NewName // for the section during emission if non-empty. - void emitAsData(MCStreamer &Streamer, StringRef NewName = StringRef()) const; + void emitAsData(MCStreamer &Streamer, StringRef NewName = StringRef()); using SymbolResolverFuncTy = llvm::function_ref; diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -49,6 +49,17 @@ /// Used to validate relocation correctness. uint64_t Value; + /// The label symbol associated with this relocation. + MCSymbol *Label{nullptr}; + + /// The output offset of this relocation + uint64_t OutputOffset{0}; + + /// Get output offset if set or original relocation offset + uint64_t getOutputOffset() const { + return OutputOffset ? OutputOffset : Offset; + } + /// Return size in bytes of the given relocation \p Type. static size_t getSizeForType(uint64_t Type); @@ -106,8 +117,9 @@ bool isRelative() const { return isRelative(Type); } /// Emit relocation at a current \p Streamer' position. The caller is - /// responsible for setting the position correctly. - size_t emit(MCStreamer *Streamer) const; + /// responsible for setting the position correctly. Setting \p EmitLabel + /// to true emits and sets Label MCSymbol value. + size_t emit(MCStreamer *Streamer, bool EmitLabel = false); /// Print a relocation to \p OS. void print(raw_ostream &OS) const; diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -300,7 +300,8 @@ bool BinaryContext::validateHoles() const { bool Valid = true; for (BinarySection &Section : sections()) { - for (const Relocation &Rel : Section.relocations()) { + for (const auto &It : Section.relocations()) { + const Relocation &Rel = It.second; uint64_t RelAddr = Rel.Offset + Section.getAddress(); const BinaryData *BD = getBinaryDataContainingAddress(RelAddr); if (!BD) { @@ -1856,7 +1857,7 @@ return Section->removeRelocationAt(Address - Section->getAddress()); } -const Relocation *BinaryContext::getRelocationAt(uint64_t Address) { +Relocation *BinaryContext::getRelocationAt(uint64_t Address) { ErrorOr Section = getSectionForAddress(Address); if (!Section) return nullptr; @@ -1864,7 +1865,11 @@ return Section->getRelocationAt(Address - Section->getAddress()); } -const Relocation *BinaryContext::getDynamicRelocationAt(uint64_t Address) { +const Relocation *BinaryContext::getRelocationAt(uint64_t Address) const { + return const_cast(this)->getRelocationAt(Address); +} + +Relocation *BinaryContext::getDynamicRelocationAt(uint64_t Address) { ErrorOr Section = getSectionForAddress(Address); if (!Section) return nullptr; @@ -1872,6 +1877,11 @@ return Section->getDynamicRelocationAt(Address - Section->getAddress()); } +const Relocation * +BinaryContext::getDynamicRelocationAt(uint64_t Address) const { + return const_cast(this)->getDynamicRelocationAt(Address); +} + void BinaryContext::markAmbiguousRelocations(BinaryData &BD, const uint64_t Address) { auto setImmovable = [&](BinaryData &BD) { diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -562,7 +562,7 @@ if (It->first >= EndOffset) break; - const Relocation &Relocation = It->second; + Relocation &Relocation = It->second; if (FunctionOffset < Relocation.Offset) { Streamer.emitBytes( FunctionContents.slice(FunctionOffset, Relocation.Offset)); @@ -575,7 +575,7 @@ << Twine::utohexstr(Relocation.Offset) << " with size " << Relocation::getSizeForType(Relocation.Type) << '\n'); - FunctionOffset += Relocation.emit(&Streamer); + FunctionOffset += Relocation.emit(&Streamer, /* EmitLabel */ true); } assert(FunctionOffset <= EndOffset && "overflow error"); diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -4126,6 +4126,19 @@ setOutputSize(Layout.getSymbolOffset(*getFunctionEndLabel())); } + if (hasConstantIsland()) { + for (auto &It : Islands->Relocations) { + const uint64_t Address = getAddress() + It.first; + Relocation *DynRel = BC.getDynamicRelocationAt(Address); + if (!DynRel) + continue; + + const Relocation &Rel = It.second; + assert(Rel.Label && "Expected relocation to have a label"); + DynRel->OutputOffset = Layout.getSymbolOffset(*Rel.Label); + } + } + // Update basic block output ranges for the debug info, if we have // secondary entry points in the symbol table to update or if writing BAT. if (!opts::UpdateDebugSections && !isMultiEntry() && diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp --- a/bolt/lib/Core/BinarySection.cpp +++ b/bolt/lib/Core/BinarySection.cpp @@ -41,17 +41,18 @@ uint64_t Offset = BD.getAddress() - getAddress(); const uint64_t EndOffset = BD.getEndAddress() - getAddress(); - auto Begin = Relocations.lower_bound(Relocation{Offset, 0, 0, 0, 0}); - auto End = Relocations.upper_bound(Relocation{EndOffset, 0, 0, 0, 0}); + auto Begin = Relocations.lower_bound(Offset); + auto End = Relocations.upper_bound(EndOffset); const StringRef Contents = getContents(); hash_code Hash = hash_combine(hash_value(BD.getSize()), hash_value(BD.getSectionName())); while (Begin != End) { - const Relocation &Rel = *Begin++; - Hash = hash_combine( - Hash, hash_value(Contents.substr(Offset, Begin->Offset - Offset))); + const Relocation &Rel = Begin->second; + Begin = std::next(Begin, 1); + Hash = hash_combine(Hash, hash_value(Contents.substr( + Offset, Begin->second.Offset - Offset))); if (BinaryData *RelBD = BC.getBinaryDataByName(Rel.Symbol->getName())) Hash = hash_combine(Hash, hash(*RelBD, Cache)); Offset = Rel.Offset + Rel.getSize(); @@ -65,7 +66,7 @@ return Hash; } -void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) const { +void BinarySection::emitAsData(MCStreamer &Streamer, StringRef NewName) { StringRef SectionName = !NewName.empty() ? NewName : getName(); StringRef SectionContents = getContents(); MCSectionELF *ELFSection = @@ -85,7 +86,8 @@ Streamer.emitBytes(SectionContents); } else { uint64_t SectionOffset = 0; - for (const Relocation &Relocation : relocations()) { + for (auto It : relocations()) { + Relocation &Relocation = It.second; assert(Relocation.Offset < SectionContents.size() && "overflow detected"); // Skip undefined symbols. if (BC.UndefinedSymbols.count(Relocation.Symbol)) @@ -191,9 +193,12 @@ if (isTLS()) OS << " (tls)"; - if (opts::PrintRelocations) - for (const Relocation &R : relocations()) - OS << "\n " << R; + if (opts::PrintRelocations) { + for (const auto &It : relocations()) { + const Relocation &Rel = It.second; + OS << "\n " << Rel; + } + } } BinarySection::RelocationSetType @@ -201,7 +206,8 @@ assert(PendingRelocations.empty() && "reodering pending relocations not supported"); RelocationSetType NewRelocations; - for (const Relocation &Rel : relocations()) { + for (const auto &It : relocations()) { + const Relocation &Rel = It.second; uint64_t RelAddr = Rel.Offset + getAddress(); BinaryData *BD = BC.getBinaryDataContainingAddress(RelAddr); BD = BD->getAtomicRoot(); @@ -211,14 +217,13 @@ continue; Relocation NewRel(Rel); - uint64_t RelOffset = RelAddr - BD->getAddress(); - NewRel.Offset = BD->getOutputOffset() + RelOffset; + const uint64_t RelOffset = + BD->getOutputOffset() + (RelAddr - BD->getAddress()); + NewRel.Offset = RelOffset; assert(NewRel.Offset < getSize()); LLVM_DEBUG(dbgs() << "BOLT-DEBUG: moving " << Rel << " -> " << NewRel << "\n"); - auto Res = NewRelocations.emplace(std::move(NewRel)); - (void)Res; - assert(Res.second && "Can't overwrite existing relocation"); + NewRelocations[RelOffset] = std::move(NewRel); } return NewRelocations; } diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -581,12 +581,15 @@ return isPCRelativeX86(Type); } -size_t Relocation::emit(MCStreamer *Streamer) const { +size_t Relocation::emit(MCStreamer *Streamer, bool EmitLabel) { const size_t Size = getSizeForType(Type); MCContext &Ctx = Streamer->getContext(); + if (EmitLabel || isPCRelative(Type)) { + Label = Ctx.createTempSymbol(); + Streamer->emitLabel(Label); + } + if (isPCRelative(Type)) { - MCSymbol *TempLabel = Ctx.createNamedTempSymbol(); - Streamer->emitLabel(TempLabel); const MCExpr *Value = nullptr; if (Symbol) { Value = MCSymbolRefExpr::create(Symbol, Ctx); @@ -597,8 +600,8 @@ } else { Value = MCConstantExpr::create(Addend, Ctx); } - Value = MCBinaryExpr::createSub( - Value, MCSymbolRefExpr::create(TempLabel, Ctx), Ctx); + Value = MCBinaryExpr::createSub(Value, MCSymbolRefExpr::create(Label, Ctx), + Ctx); Streamer->emitValue(Value, Size); return Size; diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -4919,7 +4919,8 @@ auto writeRelocations = [&](bool PatchRelative) { for (BinarySection &Section : BC->allocatableSections()) { - for (const Relocation &Rel : Section.dynamicRelocations()) { + for (const auto &It : Section.dynamicRelocations()) { + const Relocation &Rel = It.second; const bool IsRelative = Rel.isRelative(); if (PatchRelative != IsRelative) continue; @@ -4931,12 +4932,11 @@ uint64_t SectionAddress = Section.getOutputAddress(); SectionAddress = SectionAddress == 0 ? Section.getAddress() : SectionAddress; - MCSymbol *Symbol = Rel.Symbol; uint32_t SymbolIdx = 0; uint64_t Addend = Rel.Addend; if (Rel.Symbol) { - SymbolIdx = getOutputDynamicSymbolIndex(Symbol); + SymbolIdx = getOutputDynamicSymbolIndex(Rel.Symbol); } else { // Usually this case is used for R_*_(I)RELATIVE relocations const uint64_t Address = getNewFunctionOrDataAddress(Addend); @@ -4945,8 +4945,8 @@ } NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL()); - NewRelA.r_offset = SectionAddress + Rel.Offset; NewRelA.r_addend = Addend; + NewRelA.r_offset = SectionAddress + Rel.getOutputOffset(); const bool IsJmpRel = !!(IsJmpRelocation.find(Rel.Type) != IsJmpRelocation.end()); diff --git a/bolt/test/AArch64/constant-island-dyn-rel.s b/bolt/test/AArch64/constant-island-dyn-rel.s new file mode 100644 --- /dev/null +++ b/bolt/test/AArch64/constant-island-dyn-rel.s @@ -0,0 +1,53 @@ +// This test checks that the dynamic relocation offset is fixed properly after +// the bolt tool. The addr relaxation pass will replace the adr instruction to +// adrp + add, which will extend the function size and the constant island +// offset will change, so the dynamic R_*_RELATIVE relocation offset in it +// should be properly updated. + +# RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \ +# RUN: %s -o %t.o +# RUN: %clang %cflags -fPIC -pie %t.o -o %t.exe -Wl,-q \ +# RUN: -nostdlib -Wl,-z,notext +# RUN: llvm-bolt %t.exe -o %t.bolt -use-old-text=0 -lite=0 -adr-relaxation=true +# RUN: llvm-objdump -d --dynamic-reloc --disassemble-symbols='$d' %t.bolt | \ +# RUN: FileCheck %s + +# CHECK: [[#%x,ADDR:]] R_AARCH64_RELATIVE +# CHECK: [[#ADDR]] <$d>: + +.data +.align 8 +flag: + .xword 1 + +.text +.align 4 +.global +.type dummy, %function +dummy: + add x0, x0, #1 + ret + +.global +.type exitLocal, %function +exitLocal: + mov x0, x0 + ret + +.global _start +.type _start, %function +_start: + adr x0, flag + ldr x0, [x0] + adr x1, .Lci + ldr x1, [x1] + cmp x0, #0 + b.ne .J1 + blr x1 +.J1: + mov x0, #0 + mov x1, x1 + blr x1 +.Lci: + .xword exitLocal + .xword 0