Index: ELF/Relocations.h =================================================================== --- ELF/Relocations.h +++ ELF/Relocations.h @@ -125,19 +125,34 @@ bool createThunks(ArrayRef OutputSections); private: - void mergeThunks(OutputSection *OS, std::vector &Thunks); + void mergeThunks(); ThunkSection *getOSThunkSec(ThunkSection *&TS, OutputSection *OS); ThunkSection *getISThunkSec(InputSection *IS, OutputSection *OS); std::pair getThunk(SymbolBody &Body, uint32_t Type); + ThunkSection *addThunkSection(OutputSection *OS, uint64_t Off); + + uint32_t Pass = 0; // Track Symbols that already have a Thunk llvm::DenseMap ThunkedSymbols; + // Find a Thunk from the Thunks symbol definition, we can use this to find + // the Thunk from a relocation to the Thunks symbol definition. + llvm::DenseMap Thunks; + // Track InputSections that have a ThunkSection placed in front llvm::DenseMap ThunkedSections; - // Track the ThunksSections that need to be inserted into an OutputSection + // All the ThunkSections that we have created, organised by OutputSection + // will contain a mix of ThunkSections that have been created this pass, and + // ThunkSections that have been merged into the OutputSection on previous + // passes std::map> ThunkSections; + + // All the ThunkSections we have created this pass, organised per + // OutputSection. All the ThunkSections in NewThunkSections will be merged + // into the OutputSection if they contain any Thunks. + std::map> NewThunkSections; }; // Return a int64_t to make sure we get the sign extension out of the way as Index: ELF/Relocations.cpp =================================================================== --- ELF/Relocations.cpp +++ ELF/Relocations.cpp @@ -933,39 +933,41 @@ // in the Sections vector, and recalculate the InputSection output section // offsets. // This may invalidate any output section offsets stored outside of InputSection -template -void ThunkCreator::mergeThunks(OutputSection *OS, - std::vector &Thunks) { - // Order Thunks in ascending OutSecOff - auto ThunkCmp = [](const ThunkSection *A, const ThunkSection *B) { - return A->OutSecOff < B->OutSecOff; - }; - std::stable_sort(Thunks.begin(), Thunks.end(), ThunkCmp); - - // Merge sorted vectors of Thunks and InputSections by OutSecOff - std::vector Tmp; - Tmp.reserve(OS->Sections.size() + Thunks.size()); - auto MergeCmp = [](const InputSection *A, const InputSection *B) { - // std::merge requires a strict weak ordering. - if (A->OutSecOff < B->OutSecOff) - return true; - if (A->OutSecOff == B->OutSecOff) - // Check if Thunk is immediately before any specific Target InputSection - // for example Mips LA25 Thunks. - if (auto *TA = dyn_cast(A)) - if (TA && TA->getTargetInputSection() == B) - return true; - return false; - }; - std::merge(OS->Sections.begin(), OS->Sections.end(), Thunks.begin(), - Thunks.end(), std::back_inserter(Tmp), MergeCmp); - OS->Sections = std::move(Tmp); - - // We need to insert the sections into the linker script input section - // commands for the correct output order - if (Script->Opt.HasSections) - Script->syncWithOS(OS); - OS->assignOffsets(); +template void ThunkCreator::mergeThunks() { + for (auto &KV : NewThunkSections) { + OutputSection *OS = KV.first; + std::vector &Thunks = KV.second; + + // Order Thunks in ascending OutSecOff + auto ThunkCmp = [](const ThunkSection *A, const ThunkSection *B) { + return A->OutSecOff < B->OutSecOff; + }; + std::stable_sort(Thunks.begin(), Thunks.end(), ThunkCmp); + + // Merge sorted vectors of Thunks and InputSections by OutSecOff + std::vector Tmp; + Tmp.reserve(OS->Sections.size() + Thunks.size()); + auto MergeCmp = [](const InputSection *A, const InputSection *B) { + // std::merge requires a strict weak ordering. + if (A->OutSecOff < B->OutSecOff) + return true; + if (A->OutSecOff == B->OutSecOff) + // Check if Thunk is immediately before any specific Target InputSection + // for example Mips LA25 Thunks. + if (auto *TA = dyn_cast(A)) + if (TA && TA->getTargetInputSection() == B) + return true; + return false; + }; + std::merge(OS->Sections.begin(), OS->Sections.end(), Thunks.begin(), + Thunks.end(), std::back_inserter(Tmp), MergeCmp); + OS->Sections = std::move(Tmp); + + // We need to insert the sections into the linker script input section + // commands for the correct output order + if (Script->Opt.HasSections) + Script->syncWithOS(OS); + } } template @@ -978,8 +980,7 @@ if ((IS->Flags & SHF_EXECINSTR) == 0) break; } - TS = make(OS, Off); - ThunkSections[OS].push_back(TS); + TS = addThunkSection(OS, Off); } return TS; } @@ -991,8 +992,7 @@ if (TS) return TS; auto *TOS = cast(IS->OutSec); - TS = make(TOS, IS->OutSecOff); - ThunkSections[TOS].push_back(TS); + TS = addThunkSection(TOS, IS->OutSecOff); ThunkedSections[IS] = TS; return TS; } @@ -1006,6 +1006,17 @@ return std::make_pair(res.first->second, res.second); } +// Make a new ThunkSection at offset Off and it to the persistent and per pass +// records per OutputSection. +template +ThunkSection *ThunkCreator::addThunkSection(OutputSection *OS, + uint64_t Off) { + auto *TS = make(OS, Off); + ThunkSections[OS].push_back(TS); + NewThunkSections[OS].push_back(TS); + return TS; +} + // Process all relocations from the InputSections that have been assigned // to OutputSections and redirect through Thunks if needed. // @@ -1019,6 +1030,15 @@ template bool ThunkCreator::createThunks( ArrayRef OutputSections) { + bool AddressesChanged = false; + if (Pass > 0) { + NewThunkSections.clear(); + if (Pass == 2) + // With existing Thunks pass 0 will create Thunks, pass 1 will + // create no more Thunks so if we get to 2 something has gone wrong. + fatal("Thunk creation not converged in sufficient number of passes"); + } + // Create all the Thunks and insert them into synthetic ThunkSections. The // ThunkSections are later inserted back into the OutputSection. @@ -1028,14 +1048,19 @@ for (OutputSection *OS : OutputSections) { ThunkSection *OSTS = nullptr; for (InputSection *IS : OS->Sections) { + if (dyn_cast(IS) != nullptr) + // Do not create Thunks for relocations from Thunks + continue; for (Relocation &Rel : IS->Relocations) { SymbolBody &Body = *Rel.Sym; - if (!Target->needsThunk(Rel.Expr, Rel.Type, IS->File, Body)) + if (Thunks.find(&Body) != Thunks.end() || + !Target->needsThunk(Rel.Expr, Rel.Type, IS->File, Body)) continue; Thunk *T; bool IsNew; std::tie(T, IsNew) = getThunk(Body, Rel.Type); if (IsNew) { + AddressesChanged = true; // Find or create a ThunkSection for the new Thunk ThunkSection *TS; if (auto *TIS = T->getTargetInputSection()) @@ -1043,6 +1068,7 @@ else TS = getOSThunkSec(OSTS, OS); TS->addThunk(T); + Thunks[T->ThunkSym] = T; } // Redirect relocation to Thunk, we never go via the PLT to a Thunk Rel.Sym = T->ThunkSym; @@ -1050,11 +1076,16 @@ } } } - - // Merge all created synthetic ThunkSections back into OutputSection - for (auto &KV : ThunkSections) - mergeThunks(KV.first, KV.second); - return !ThunkSections.empty(); + if (AddressesChanged) { + // Merge all newly created ThunkSections back into OutputSection + mergeThunks(); + // Update the offsets of each OutputSection we have added ThunkSections to + for (auto &KV : ThunkSections) + KV.first->assignOffsets(); + ++Pass; + return true; + } + return false; } template void elf::scanRelocations(InputSectionBase &); Index: ELF/Writer.cpp =================================================================== --- ELF/Writer.cpp +++ ELF/Writer.cpp @@ -1186,9 +1186,10 @@ }; AddressesForThunks(); ThunkCreator TC; - if (TC.createThunks(OutputSections)) + while (TC.createThunks(OutputSections)) { applySynthetic({In::MipsGot}, [](SyntheticSection *SS) { SS->updateAllocSize(); }); + } } // Fill other section headers. The dynamic table is finalized // at the end because some tags like RELSZ depend on result