Index: ELF/OutputSections.h =================================================================== --- ELF/OutputSections.h +++ ELF/OutputSections.h @@ -94,7 +94,7 @@ } virtual void finalize() {} - virtual void writeTo(uint8_t *Buf) = 0; + virtual void writeTo(uint8_t *Buf) {} virtual ~OutputSectionBase() = default; protected: @@ -540,6 +540,8 @@ static SymbolTableSection *DynSymTab; static SymbolTableSection *SymTab; static Elf_Phdr *TlsPhdr; + static OutputSectionBase *ElfHeader; + static OutputSectionBase *ProgramHeaders; }; template DynamicSection *Out::Dynamic; @@ -562,6 +564,8 @@ template SymbolTableSection *Out::DynSymTab; template SymbolTableSection *Out::SymTab; template typename Out::Elf_Phdr *Out::TlsPhdr; +template OutputSectionBase *Out::ElfHeader; +template OutputSectionBase *Out::ProgramHeaders; } // namespace elf2 } // namespace lld Index: ELF/Writer.cpp =================================================================== --- ELF/Writer.cpp +++ ELF/Writer.cpp @@ -13,6 +13,7 @@ #include "SymbolTable.h" #include "Target.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Support/FileOutputBuffer.h" @@ -41,6 +42,19 @@ void run(); private: + // This describes a program header entry. + // Each contains type, access flags and range of output sections that will be + // placed in it. + struct Phdr { + Phdr(unsigned Type, unsigned Flags) { + H.p_type = Type; + H.p_flags = Flags; + } + Elf_Phdr H = {}; + OutputSectionBase *First = nullptr; + OutputSectionBase *Last = nullptr; + }; + void copyLocalSymbols(); void addReservedSymbols(); bool createSections(); @@ -52,7 +66,7 @@ void scanRelocs(InputSection &C); void scanRelocs(InputSectionBase &S, const Elf_Shdr &RelSec); - void updateRelro(Elf_Phdr *Cur, Elf_Phdr *GnuRelroPhdr, uintX_t VA); + void createPhdrs(); void assignAddresses(); void buildSectionMap(); void fixAbsoluteSymbols(); @@ -67,7 +81,6 @@ bool isOutputDynamic() const { return !Symtab.getSharedFiles().empty() || Config->Shared; } - int getPhdrsNum() const; OutputSection *getBss(); void addCommonSymbols(std::vector &Syms); @@ -78,18 +91,22 @@ BumpPtrAllocator Alloc; std::vector *> OutputSections; std::vector>> OwningSections; - unsigned getNumSections() const { return OutputSections.size() + 1; } + + // We create a section for the ELF header and one for the program headers. + const unsigned NumDummySections = 2; + ArrayRef *> getSections() const { + return makeArrayRef(OutputSections).slice(NumDummySections); + } + unsigned getNumSections() const { + return OutputSections.size() + 1 - NumDummySections; + } void addRelIpltSymbols(); void addStartEndSymbols(); void addStartStopSymbols(OutputSectionBase *Sec); - void setPhdr(Elf_Phdr *PH, uint32_t Type, uint32_t Flags, uintX_t FileOff, - uintX_t VA, uintX_t Size, uintX_t Align); - void copyPhdr(Elf_Phdr *PH, OutputSectionBase *From); - bool HasRelro = false; SymbolTable &Symtab; - std::vector Phdrs; + std::vector Phdrs; uintX_t FileSize; uintX_t SectionHeaderOff; @@ -105,6 +122,8 @@ template static bool shouldUseRela() { return ELFT::Is64Bits; } template void elf2::writeResult(SymbolTable *Symtab) { + typedef typename ELFFile::uintX_t uintX_t; + // Create singleton output sections. bool IsRela = shouldUseRela(); DynamicSection Dynamic(*Symtab); @@ -117,6 +136,10 @@ StringTableSection ShStrTab(".shstrtab", false); SymbolTableSection DynSymTab(*Symtab, DynStrTab); + OutputSectionBase ElfHeader("", 0, SHF_ALLOC); + OutputSectionBase ProgramHeaders("", 0, SHF_ALLOC); + ProgramHeaders.updateAlign(sizeof(uintX_t)); + // Instantiate optional output sections if they are needed. std::unique_ptr> GnuHashTab; std::unique_ptr> GotPlt; @@ -159,6 +182,8 @@ Out::Opd = nullptr; Out::OpdBuf = nullptr; Out::TlsPhdr = nullptr; + Out::ElfHeader = &ElfHeader; + Out::ProgramHeaders = &ProgramHeaders; Writer(*Symtab).run(); } @@ -171,6 +196,7 @@ addReservedSymbols(); if (!createSections()) return; + createPhdrs(); assignAddresses(); fixAbsoluteSymbols(); if (!openFile()) @@ -547,6 +573,8 @@ } template static bool isRelroSection(OutputSectionBase *Sec) { + if (!Config->ZRelro) + return false; typename OutputSectionBase::uintX_t Flags = Sec->getFlags(); if (!(Flags & SHF_ALLOC) || !(Flags & SHF_WRITE)) return false; @@ -889,6 +917,9 @@ // Create output section objects and add them to OutputSections. template bool Writer::createSections() { + OutputSections.push_back(Out::ElfHeader); + OutputSections.push_back(Out::ProgramHeaders); + // Add .interp first because some loaders want to see that section // on the first page of the executable file when loaded into memory. if (needsInterpSection()) @@ -994,12 +1025,10 @@ std::stable_sort(OutputSections.begin(), OutputSections.end(), compareSections); - for (unsigned I = 0, N = OutputSections.size(); I < N; ++I) { - OutputSections[I]->SectionIndex = I + 1; - HasRelro |= (Config->ZRelro && isRelroSection(OutputSections[I])); - } + for (unsigned I = NumDummySections, N = OutputSections.size(); I < N; ++I) + OutputSections[I]->SectionIndex = I + 1 - NumDummySections; - for (OutputSectionBase *Sec : OutputSections) + for (OutputSectionBase *Sec : getSections()) Sec->setSHName(Out::ShStrTab->addString(Sec->getName())); // Finalizers fix each section's size. @@ -1165,141 +1194,150 @@ return PT_LOAD; } -template -void Writer::updateRelro(Elf_Phdr *Cur, Elf_Phdr *GnuRelroPhdr, - uintX_t VA) { - if (!GnuRelroPhdr->p_type) - setPhdr(GnuRelroPhdr, PT_GNU_RELRO, PF_R, Cur->p_offset, Cur->p_vaddr, - VA - Cur->p_vaddr, 1 /*p_align*/); - GnuRelroPhdr->p_filesz = VA - Cur->p_vaddr; - GnuRelroPhdr->p_memsz = VA - Cur->p_vaddr; -} - -// Visits all sections to create PHDRs and to assign incremental, -// non-overlapping addresses to output sections. -template void Writer::assignAddresses() { - uintX_t VA = Target->getVAStart() + sizeof(Elf_Ehdr); - uintX_t FileOff = sizeof(Elf_Ehdr); +// Decide which program headers to create and which sections to include in each +// one. +template void Writer::createPhdrs() { + auto AddHdr = [this](unsigned Type, unsigned Flags) { + return &*Phdrs.emplace(Phdrs.end(), Type, Flags); + }; - // Calculate and reserve the space for the program header first so that - // the first section can start right after the program header. - Phdrs.resize(getPhdrsNum()); - size_t PhdrSize = sizeof(Elf_Phdr) * Phdrs.size(); + auto AddSec = [](Phdr &Hdr, OutputSectionBase *Sec) { + Hdr.Last = Sec; + if (!Hdr.First) + Hdr.First = Sec; + Hdr.H.p_align = std::max(Hdr.H.p_align, Sec->getAlign()); + }; // The first phdr entry is PT_PHDR which describes the program header itself. - setPhdr(&Phdrs[0], PT_PHDR, PF_R, FileOff, VA, PhdrSize, - /*Align=*/sizeof(uintX_t)); - FileOff += PhdrSize; - VA += PhdrSize; + Phdr &Hdr = *AddHdr(PT_PHDR, PF_R); + AddSec(Hdr, Out::ProgramHeaders); // PT_INTERP must be the second entry if exists. - int PhdrIdx = 0; - Elf_Phdr *Interp = nullptr; - if (needsInterpSection()) - Interp = &Phdrs[++PhdrIdx]; + if (needsInterpSection()) { + Phdr &Hdr = *AddHdr(PT_INTERP, toPhdrFlags(Out::Interp->getFlags())); + AddSec(Hdr, Out::Interp); + } // Add the first PT_LOAD segment for regular output sections. - setPhdr(&Phdrs[++PhdrIdx], PT_LOAD, PF_R, 0, Target->getVAStart(), FileOff, - Target->PageSize); + uintX_t Flags = PF_R; + Phdr *Load = AddHdr(PT_LOAD, Flags); + AddSec(*Load, Out::ElfHeader); - Elf_Phdr GnuRelroPhdr = {}; - Elf_Phdr TlsPhdr{}; - bool RelroAligned = false; - uintX_t ThreadBssOffset = 0; - // Create phdrs as we assign VAs and file offsets to all output sections. + Phdr TlsHdr(PT_TLS, PF_R); + Phdr RelRo(PT_GNU_RELRO, PF_R); for (OutputSectionBase *Sec : OutputSections) { - Elf_Phdr *PH = &Phdrs[PhdrIdx]; - if (needsPhdr(Sec)) { - uintX_t Flags = toPhdrFlags(Sec->getFlags()); - bool InRelRo = Config->ZRelro && (Flags & PF_W) && isRelroSection(Sec); - bool FirstNonRelRo = GnuRelroPhdr.p_type && !InRelRo && !RelroAligned; - if (FirstNonRelRo || PH->p_flags != Flags) { - VA = alignTo(VA, Target->PageSize); - FileOff = alignTo(FileOff, Target->PageSize); - if (FirstNonRelRo) - RelroAligned = true; - } - - if (PH->p_flags != Flags) { - // Flags changed. Create a new PT_LOAD. - PH = &Phdrs[++PhdrIdx]; - uint32_t PTType = (Config->EMachine != EM_AMDGPU) ? (uint32_t)PT_LOAD - : getAmdgpuPhdr(Sec); - setPhdr(PH, PTType, Flags, FileOff, VA, 0, Target->PageSize); - } - - if (Sec->getFlags() & SHF_TLS) { - if (!TlsPhdr.p_vaddr) - setPhdr(&TlsPhdr, PT_TLS, PF_R, FileOff, VA, 0, Sec->getAlign()); - if (Sec->getType() != SHT_NOBITS) - VA = alignTo(VA, Sec->getAlign()); - uintX_t TVA = alignTo(VA + ThreadBssOffset, Sec->getAlign()); - Sec->setVA(TVA); - TlsPhdr.p_memsz += Sec->getSize(); - if (Sec->getType() == SHT_NOBITS) { - ThreadBssOffset = TVA - VA + Sec->getSize(); - } else { - TlsPhdr.p_filesz += Sec->getSize(); - VA += Sec->getSize(); - } - TlsPhdr.p_align = std::max(TlsPhdr.p_align, Sec->getAlign()); - } else { - VA = alignTo(VA, Sec->getAlign()); - Sec->setVA(VA); - VA += Sec->getSize(); - } - if (InRelRo) - updateRelro(PH, &GnuRelroPhdr, VA); + if (!needsPhdr(Sec)) + break; + + // If flags changed then we want new load segment. + uintX_t NewFlags = toPhdrFlags(Sec->getFlags()); + if (Flags != NewFlags) { + uint32_t LoadType = (Config->EMachine == EM_AMDGPU) ? getAmdgpuPhdr(Sec) + : (uint32_t)PT_LOAD; + Load = AddHdr(LoadType, NewFlags); + Flags = NewFlags; } - - FileOff = alignTo(FileOff, Sec->getAlign()); - Sec->setFileOffset(FileOff); - if (Sec->getType() != SHT_NOBITS) - FileOff += Sec->getSize(); - if (needsPhdr(Sec)) { - PH->p_filesz = FileOff - PH->p_offset; - PH->p_memsz = VA - PH->p_vaddr; + // If we meet TLS section then we create TLS header + // and put all TLS sections inside for futher use when + // assign addresses. + if (Sec->getFlags() & SHF_TLS) { + AddSec(TlsHdr, Sec); + if (Sec->getType() == SHT_NOBITS) + continue; } - } - if (TlsPhdr.p_vaddr) { - // The TLS pointer goes after PT_TLS. At least glibc will align it, - // so round up the size to make sure the offsets are correct. - TlsPhdr.p_memsz = alignTo(TlsPhdr.p_memsz, TlsPhdr.p_align); - Phdrs[++PhdrIdx] = TlsPhdr; - Out::TlsPhdr = &Phdrs[PhdrIdx]; + AddSec(*Load, Sec); + + if (isRelroSection(Sec)) + AddSec(RelRo, Sec); } + // Add the TLS segment unless it's empty. + if (TlsHdr.First) + Phdrs.push_back(std::move(TlsHdr)); + // Add an entry for .dynamic. if (isOutputDynamic()) { - Elf_Phdr *PH = &Phdrs[++PhdrIdx]; - PH->p_type = PT_DYNAMIC; - copyPhdr(PH, Out::Dynamic); + Phdr &Hdr = + *AddHdr(PT_DYNAMIC, toPhdrFlags(Out::Dynamic->getFlags())); + AddSec(Hdr, Out::Dynamic); } - if (HasRelro) { - Elf_Phdr *PH = &Phdrs[++PhdrIdx]; - *PH = GnuRelroPhdr; - } + // PT_GNU_RELRO includes all sections that should be marked as + // read-only by dynamic linker after proccessing relocations. + if (RelRo.First) + Phdrs.push_back(std::move(RelRo)); + // PT_GNU_EH_FRAME is a special section pointing on .eh_frame_hdr. if (Out::EhFrameHdr->Live) { - Elf_Phdr *PH = &Phdrs[++PhdrIdx]; - PH->p_type = PT_GNU_EH_FRAME; - copyPhdr(PH, Out::EhFrameHdr); + Phdr &Hdr = *AddHdr(PT_GNU_EH_FRAME, + toPhdrFlags(Out::EhFrameHdr->getFlags())); + AddSec(Hdr, Out::EhFrameHdr); } // PT_GNU_STACK is a special section to tell the loader to make the // pages for the stack non-executable. - if (!Config->ZExecStack) { - Elf_Phdr *PH = &Phdrs[++PhdrIdx]; - PH->p_type = PT_GNU_STACK; - PH->p_flags = PF_R | PF_W; + if (!Config->ZExecStack) + AddHdr(PT_GNU_STACK, PF_R | PF_W); +} + +// Visits all headers in PhdrTable and assigns the adresses to +// the output sections. Also creates common and special headers. +template void Writer::assignAddresses() { + Out::ElfHeader->setSize(sizeof(Elf_Ehdr)); + size_t PhdrSize = sizeof(Elf_Phdr) * Phdrs.size(); + Out::ProgramHeaders->setSize(PhdrSize); + + // The first section of each PT_LOAD and the first section after PT_GNU_RELRO + // have to be page aligned so that the dynamic linker can set the permissions. + SmallPtrSet *, 4> PageAlign; + for (const Phdr &P : Phdrs) { + if (P.H.p_type == PT_GNU_RELRO) { + // Find the first section after PT_GNU_RELRO. If it is in a PT_LOAD + // and is not tls, we have to align it to a page. We don't have to + // align tls since TLS NOBITS takes no space. + auto I = std::find(OutputSections.begin(), OutputSections.end(), P.Last); + ++I; + if (I != OutputSections.end() && needsPhdr(*I) && + !((*I)->getFlags() & SHF_TLS)) + PageAlign.insert(*I); + } + + // FIXME: why create empty PT_LOAD? + if (P.H.p_type == PT_LOAD && P.First) + PageAlign.insert(P.First); } - // Fix up PT_INTERP as we now know the address of .interp section. - if (Interp) { - Interp->p_type = PT_INTERP; - copyPhdr(Interp, Out::Interp); + uintX_t ThreadBssOffset = 0; + uintX_t VA = Target->getVAStart(); + uintX_t FileOff = 0; + + for (OutputSectionBase *Sec : OutputSections) { + uintX_t Align = Sec->getAlign(); + if (PageAlign.count(Sec)) + Align = std::max(Align, Target->PageSize); + + FileOff = alignTo(FileOff, Align); + Sec->setFileOffset(FileOff); + if (Sec->getType() != SHT_NOBITS) + FileOff += Sec->getSize(); + + if (needsPhdr(Sec)) { + // Don't allocate VA space for TLS NOBITS sections. The PT_TLS PHDR is + // responsible for allocating space for them, not the PT_LOAD that + // contains the TLS initialization image. + bool IsTls = Sec->getFlags() & SHF_TLS; + if (IsTls && Sec->getType() == SHT_NOBITS) { + uintX_t TVA = VA + ThreadBssOffset; + TVA = alignTo(TVA, Align); + Sec->setVA(TVA); + ThreadBssOffset = TVA - VA + Sec->getSize(); + } else { + VA = alignTo(VA, Align); + Sec->setVA(VA); + VA += Sec->getSize(); + } + } } // Add space for section headers. @@ -1309,37 +1347,31 @@ // Update "_end" and "end" symbols so that they // point to the end of the data segment. ElfSym::End.st_value = VA; -} -// Returns the number of PHDR entries. -template int Writer::getPhdrsNum() const { - bool Tls = false; - int I = 2; // 2 for PT_PHDR and first PT_LOAD - if (needsInterpSection()) - ++I; - if (isOutputDynamic()) - ++I; - if (!Config->ZExecStack) - ++I; - uintX_t Last = PF_R; - for (OutputSectionBase *Sec : OutputSections) { - if (!needsPhdr(Sec)) - continue; - if (Sec->getFlags() & SHF_TLS) - Tls = true; - uintX_t Flags = toPhdrFlags(Sec->getFlags()); - if (Last != Flags) { - Last = Flags; - ++I; + for (Phdr &PHdr : Phdrs) { + Elf_Phdr &H = PHdr.H; + if (PHdr.First) { + OutputSectionBase *Last = PHdr.Last; + H.p_filesz = Last->getFileOff() - PHdr.First->getFileOff(); + if (Last->getType() != SHT_NOBITS) + H.p_filesz += Last->getSize(); + H.p_memsz = Last->getVA() + Last->getSize() - PHdr.First->getVA(); + H.p_offset = PHdr.First->getFileOff(); + H.p_vaddr = PHdr.First->getVA(); + } + if (PHdr.H.p_type == PT_LOAD) + H.p_align = Target->PageSize; + else if (PHdr.H.p_type == PT_GNU_RELRO) + H.p_align = 1; + H.p_paddr = H.p_vaddr; + + // The TLS pointer goes after PT_TLS. At least glibc will align it, + // so round up the size to make sure the offsets are correct. + if (PHdr.H.p_type == PT_TLS) { + Out::TlsPhdr = &H; + H.p_memsz = alignTo(H.p_memsz, H.p_align); } } - if (Tls) - ++I; - if (HasRelro) - ++I; - if (Out::EhFrameHdr->Live) - ++I; - return I; } static uint32_t getELFFlags() { @@ -1412,11 +1444,15 @@ EHdr->e_shstrndx = Out::ShStrTab->SectionIndex; // Write the program header table. - memcpy(Buf + EHdr->e_phoff, &Phdrs[0], Phdrs.size() * sizeof(Phdrs[0])); + auto *HBuf = reinterpret_cast(Buf + EHdr->e_phoff); + for (Phdr &P : Phdrs) { + memcpy(HBuf, &P.H, sizeof(Phdrs[0].H)); + ++HBuf; + } // Write the section header table. Note that the first table entry is null. auto SHdrs = reinterpret_cast(Buf + EHdr->e_shoff); - for (OutputSectionBase *Sec : OutputSections) + for (OutputSectionBase *Sec : getSections()) Sec->writeHeaderTo(++SHdrs); } @@ -1446,31 +1482,6 @@ Sec->writeTo(Buf + Sec->getFileOff()); } -template -void Writer::setPhdr(Elf_Phdr *PH, uint32_t Type, uint32_t Flags, - uintX_t FileOff, uintX_t VA, uintX_t Size, - uintX_t Align) { - PH->p_type = Type; - PH->p_flags = Flags; - PH->p_offset = FileOff; - PH->p_vaddr = VA; - PH->p_paddr = VA; - PH->p_filesz = Size; - PH->p_memsz = Size; - PH->p_align = Align; -} - -template -void Writer::copyPhdr(Elf_Phdr *PH, OutputSectionBase *From) { - PH->p_flags = toPhdrFlags(From->getFlags()); - PH->p_offset = From->getFileOff(); - PH->p_vaddr = From->getVA(); - PH->p_paddr = From->getVA(); - PH->p_filesz = From->getSize(); - PH->p_memsz = From->getSize(); - PH->p_align = From->getAlign(); -} - template void Writer::buildSectionMap() { for (const std::pair> &OutSec : Config->OutputSections)