[ELF] - Implemented @tlsgd optimization (GD->IE case, x64).

[grimar]

Summary

"Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5 x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows how GD can be optimized to IE.
This patch implements this optimization.

[ruiu]

ELF/OutputSections.cpp

226–231

There is really a concerning trend that this function gets longer and longer over time. Probably it's time to split up into multiple small functions with comments.

[grimar]

Review comments addressed.

ELF/OutputSections.cpp

226–231

Done.

[grimar]

Refactored, rebased.

[ruiu]

ELF/OutputSections.cpp

196–201

Can you reduce the nest level by moving this if outside of the outer `if? I mean

if (Body && Target->isTlsOptimized(Type, Body)) {
  P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
                      Target->getTlsGotReloc(), Config->Mips64EL);
  P->r_offset = Out<ELFT>::Got->getEntryAddr(*Body);
  return true;
}

if (Body && Target->isTlsGlobalDynamicReloc(Type)) {
  ...
}

ELF/Target.cpp

457–459

I don't think these magic numbers are good. In fact it seems that you don't need them. Please see the comment below.

580

This can be

if (canBePreempted(&S, true))

[grimar]

Addressed review comments.

ELF/OutputSections.cpp

196–201

Refactored, what about this one ?

ELF/Target.cpp

457–459

Cool ! Fixed.

580

Done.

[ruiu]

LGTM with a few nits.

ELF/Target.cpp

465

Remove the outermost ().

512

Remove newline at end of this line.

ELF/InputSection.cpp

if (Target->isTlsGlobalDynamicReloc(Type) &&
         !Target->isTlsOptimized(Type, &Body)) {
       Target->relocateOne(BufLoc, BufEnd, Type, AddrLoc,
                           Out<ELFT>::Got->getEntryAddr(Body) +
                               getAddend<ELFT>(RI));
       continue;
     }
 
     if (Target->isTlsOptimized(Type, &Body)) {
+      uintX_t SymVA = Target->relocNeedsGot(Type, Body)
+                          ? Out<ELFT>::Got->getEntryAddr(Body)
+                          : getSymVA<ELFT>(Body);
       // By optimizing TLS relocations, it is sometimes needed to skip
       // relocations that immediately follow TLS relocations. This function
       // knows how many slots we need to skip.
-      I += Target->relocateTlsOptimize(BufLoc, BufEnd, Type, AddrLoc,
-                                       getSymVA<ELFT>(Body));
+      I += Target->relocateTlsOptimize(BufLoc, BufEnd, Type, AddrLoc, SymVA,
+                                       Body);
       continue;
     }
 
     uintX_t SymVA = getSymVA<ELFT>(Body);
     if (Target->relocNeedsPlt(Type, Body)) {
       SymVA = Out<ELFT>::Plt->getEntryAddr(Body);
       Type = Target->getPltRefReloc(Type);
     } else if (Target->relocNeedsGot(Type, Body)) {

ELF/OutputSections.cpp

RelocationSection<ELFT>::RelocationSection(StringRef Name, bool IsRela)
     : OutputSectionBase<ELFT>(Name,
                               IsRela ? llvm::ELF::SHT_RELA : llvm::ELF::SHT_REL,
                               llvm::ELF::SHF_ALLOC),
       IsRela(IsRela) {
   this->Header.sh_entsize = IsRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
   this->Header.sh_addralign = ELFT::Is64Bits ? 8 : 4;
 }
 
 template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *Buf) {
   const unsigned EntrySize = IsRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
   for (const DynamicReloc<ELFT> &Rel : Relocs) {
     auto *P = reinterpret_cast<Elf_Rel *>(Buf);
     Buf += EntrySize;
 
     // Skip placeholder for global dynamic TLS relocation pair. It was already
     // handled by the previous relocation.
     if (!Rel.C || !Rel.RI)
       continue;
 
     InputSectionBase<ELFT> &C = *Rel.C;
     const Elf_Rel &RI = *Rel.RI;
     uint32_t SymIndex = RI.getSymbol(Config->Mips64EL);
     const ObjectFile<ELFT> &File = *C.getFile();
     SymbolBody *Body = File.getSymbolBody(SymIndex);
     if (Body)
       Body = Body->repl();
 
     uint32_t Type = RI.getType(Config->Mips64EL);
 
     if (Target->isTlsLocalDynamicReloc(Type)) {
       P->setSymbolAndType(0, Target->getTlsModuleIndexReloc(),
                           Config->Mips64EL);
       P->r_offset =
           Out<ELFT>::Got->getVA() + Out<ELFT>::LocalModuleTlsIndexOffset;
       continue;
     }
 
     if (Body && Target->isTlsGlobalDynamicReloc(Type)) {
+      if (Target->isTlsOptimized(Type, Body)) {
+        P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
+                            Target->getTlsGotReloc(), Config->Mips64EL);
+        P->r_offset = Out<ELFT>::Got->getEntryAddr(*Body);
+        continue;
+      }
       P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
                           Target->getTlsModuleIndexReloc(), Config->Mips64EL);
       P->r_offset = Out<ELFT>::Got->getEntryAddr(*Body);
       auto *PNext = reinterpret_cast<Elf_Rel *>(Buf);
       PNext->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
                               Target->getTlsOffsetReloc(), Config->Mips64EL);
       PNext->r_offset = Out<ELFT>::Got->getEntryAddr(*Body) + sizeof(uintX_t);
       continue;

ELF/Target.h

virtual void writePltEntry(uint8_t *Buf, uint64_t GotAddr,
                              uint64_t GotEntryAddr, uint64_t PltEntryAddr,
                              int32_t Index, unsigned RelOff) const = 0;
   virtual bool isRelRelative(uint32_t Type) const;
   virtual bool relocNeedsCopy(uint32_t Type, const SymbolBody &S) const;
   virtual bool relocNeedsGot(uint32_t Type, const SymbolBody &S) const = 0;
   virtual bool relocNeedsPlt(uint32_t Type, const SymbolBody &S) const = 0;
   virtual void relocateOne(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type,
                            uint64_t P, uint64_t SA) const = 0;
-  virtual bool isTlsOptimized(unsigned Type, const SymbolBody *S) const;
+  virtual bool isTlsOptimized(unsigned Type, const SymbolBody *S,
+                              bool *Partial = nullptr) const;
   virtual unsigned relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
-                                       uint32_t Type, uint64_t P,
-                                       uint64_t SA) const;
+                                       uint32_t Type, uint64_t P, uint64_t SA,
+                                       const SymbolBody &S) const;
   virtual ~TargetInfo();
 
 protected:
   unsigned PageSize = 4096;
 
   // On freebsd x86_64 the first page cannot be mmaped.
   // On linux that is controled by vm.mmap_min_addr. At least on some x86_64
   // installs that is 65536, so the first 15 pages cannot be used.

ELF/Target.cpp

void writePltEntry(uint8_t *Buf, uint64_t GotAddr, uint64_t GotEntryAddr,
                      uint64_t PltEntryAddr, int32_t Index,
                      unsigned RelOff) const override;
   bool relocNeedsCopy(uint32_t Type, const SymbolBody &S) const override;
   bool relocNeedsGot(uint32_t Type, const SymbolBody &S) const override;
   bool relocNeedsPlt(uint32_t Type, const SymbolBody &S) const override;
   void relocateOne(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type, uint64_t P,
                    uint64_t SA) const override;
   bool isRelRelative(uint32_t Type) const override;
-  bool isTlsOptimized(unsigned Type, const SymbolBody *S) const override;
+  bool isTlsOptimized(unsigned Type, const SymbolBody *S,
+                      bool *Partial = nullptr) const override;
   unsigned relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type,
-                               uint64_t P, uint64_t SA) const override;
+                               uint64_t P, uint64_t SA,
+                               const SymbolBody &S) const override;
 
 private:
   void relocateTlsLdToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
   void relocateTlsGdToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
+  void relocateTlsGdToIe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
+                         uint64_t SA) const;
   void relocateTlsIeToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
 };
 
 class PPC64TargetInfo final : public TargetInfo {
 public:
   PPC64TargetInfo();
   void writeGotPltEntry(uint8_t *Buf, uint64_t Plt) const override;
TargetInfo *createTarget() {
   case EM_X86_64:
     return new X86_64TargetInfo();
   }
   error("Unknown target machine");
 }
 
 TargetInfo::~TargetInfo() {}
 
-bool TargetInfo::isTlsOptimized(unsigned Type, const SymbolBody *S) const {
+bool TargetInfo::isTlsOptimized(unsigned Type, const SymbolBody *S,
+                                bool *Partial) const {
   return false;
 }
 
 uint64_t TargetInfo::getVAStart() const { return Config->Shared ? 0 : VAStart; }
 
 bool TargetInfo::relocNeedsCopy(uint32_t Type, const SymbolBody &S) const {
   return false;
 }
 
 unsigned TargetInfo::getGotRefReloc(unsigned Type) const { return GotRefReloc; }
 
 unsigned TargetInfo::getPltRefReloc(unsigned Type) const { return PCRelReloc; }
 
 bool TargetInfo::isRelRelative(uint32_t Type) const { return true; }
 
 unsigned TargetInfo::relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
-                                         uint32_t Type, uint64_t P,
-                                         uint64_t SA) const {
+                                         uint32_t Type, uint64_t P, uint64_t SA,
+                                         const SymbolBody &S) const {
   return 0;
 }
 
 void TargetInfo::writeGotHeaderEntries(uint8_t *Buf) const {}
 
 void TargetInfo::writeGotPltHeaderEntries(uint8_t *Buf) const {}
 
 X86TargetInfo::X86TargetInfo() {
bool X86_64TargetInfo::relocNeedsCopy(uint32_t Type,
   if (Type == R_X86_64_32S || Type == R_X86_64_32 || Type == R_X86_64_PC32 ||
       Type == R_X86_64_64)
     if (auto *SS = dyn_cast<SharedSymbol<ELF64LE>>(&S))
       return SS->Sym.getType() == STT_OBJECT;
   return false;
 }
 
 bool X86_64TargetInfo::relocNeedsGot(uint32_t Type, const SymbolBody &S) const {
+  bool PartialOpt = false;
+  if (S.isTLS() && Target->isTlsGlobalDynamicReloc(Type))
+    return Target->isTlsOptimized(Type, &S, &PartialOpt) && PartialOpt;
+
   if (Type == R_X86_64_GOTTPOFF)
     return !isTlsOptimized(Type, &S);
   return Type == R_X86_64_GOTTPOFF || Type == R_X86_64_GOTPCREL ||
          relocNeedsPlt(Type, S);
 }
 
 bool X86_64TargetInfo::isTlsDynReloc(unsigned Type) const {
-  return Type == R_X86_64_GOTTPOFF;
+  return Type == R_X86_64_GOTTPOFF || Type == R_X86_64_TLSGD;
 }
 
 unsigned X86_64TargetInfo::getPltRefReloc(unsigned Type) const {
   if (Type == R_X86_64_PLT32)
     return R_X86_64_PC32;
   return Type;
 }
 
bool X86_64TargetInfo::isRelRelative(uint32_t Type) const {
   case R_X86_64_PC8:
   case R_X86_64_PLT32:
   case R_X86_64_DTPOFF32:
   case R_X86_64_DTPOFF64:
     return true;
   }
 }
 
-bool X86_64TargetInfo::isTlsOptimized(unsigned Type,
-                                      const SymbolBody *S) const {
+bool X86_64TargetInfo::isTlsOptimized(unsigned Type, const SymbolBody *S,
+                                      bool *Partial) const {
   if (Config->Shared || (S && !S->isTLS()))
     return false;
+  if (Partial && Type == R_X86_64_TLSGD && canBePreempted(S, true))
+    *Partial = true;
   return Type == R_X86_64_TLSLD || Type == R_X86_64_DTPOFF32 ||
-         (Type == R_X86_64_TLSGD && !canBePreempted(S, true)) ||
+         Type == R_X86_64_TLSGD ||
          (Type == R_X86_64_GOTTPOFF && !canBePreempted(S, true));
 }
 
 // "Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5
 // x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows
 // how LD can be optimized to LE:
 //   leaq bar@tlsld(%rip), %rdi
 //   callq __tls_get_addr@PLT
 //   leaq bar@dtpoff(%rax), %rcx
void X86_64TargetInfo::relocateTlsGdToLe(uint8_t *Loc, uint8_t *BufEnd,
   const uint8_t Inst[] = {
       0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
       0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00              // lea x@tpoff,%rax
   };
   memcpy(Loc - 4, Inst, sizeof(Inst));
   relocateOne(Loc + 8, BufEnd, R_X86_64_TPOFF32, P, SA);
 }
 
+// "Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5
+// x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows
+// how
+// GD can be optimized to IE:
+//  .byte 0x66
+//  leaq x@tlsgd(%rip), %rdi
+//  .word 0x6666
+//  rex64
+//  call __tls_get_addr@plt
+// Is converted to:
+//  mov %fs:0x0,%rax
+//  addq x@tpoff,%rax
+void X86_64TargetInfo::relocateTlsGdToIe(uint8_t *Loc, uint8_t *BufEnd,
+                                         uint64_t P, uint64_t SA) const {
+  const uint8_t Inst[] = {
+      0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
+      0x48, 0x03, 0x05, 0x00, 0x00, 0x00, 0x00              // addq x@tpoff,%rax
+  };
+  memcpy(Loc - 4, Inst, sizeof(Inst));
+  relocateOne(Loc + 8, BufEnd, R_X86_64_TPOFF64, P + 12, SA);
+}
+
 // In some conditions, R_X86_64_GOTTPOFF relocation can be optimized to
 // R_X86_64_TPOFF32 so that R_X86_64_TPOFF32 so that it does not use GOT.
 // This function does that. Read "ELF Handling For Thread-Local Storage,
 // 5.5 x86-x64 linker optimizations" (http://www.akkadia.org/drepper/tls.pdf)
 // by Ulrich Drepper for details.
 void X86_64TargetInfo::relocateTlsIeToLe(uint8_t *Loc, uint8_t *BufEnd,
                                          uint64_t P, uint64_t SA) const {
   // Ulrich's document section 6.5 says that @gottpoff(%rip) must be

 
 // This function applies a TLS relocation with an optimization as described
 // in the Ulrich's document. As a result of rewriting instructions at the
 // relocation target, relocations immediately follow the TLS relocation (which
 // would be applied to rewritten instructions) may have to be skipped.
 // This function returns a number of relocations that need to be skipped.
 unsigned X86_64TargetInfo::relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
                                                uint32_t Type, uint64_t P,
-                                               uint64_t SA) const {
+                                               uint64_t SA,
+                                               const SymbolBody &S) const {
   switch (Type) {
   case R_X86_64_GOTTPOFF:
     relocateTlsIeToLe(Loc, BufEnd, P, SA);
     return 0;
   case R_X86_64_TLSLD:
     relocateTlsLdToLe(Loc, BufEnd, P, SA);
     // The next relocation should be against __tls_get_addr, so skip it
     return 1;
-  case R_X86_64_TLSGD:
+  case R_X86_64_TLSGD: {
+    bool PartialOpt = false;
+    if (isTlsOptimized(Type, &S, &PartialOpt) && PartialOpt)
+      relocateTlsGdToIe(Loc, BufEnd, P, SA);
+    else
       relocateTlsGdToLe(Loc, BufEnd, P, SA);
     // The next relocation should be against __tls_get_addr, so skip it
     return 1;
+  }
   case R_X86_64_DTPOFF32:
     relocateOne(Loc, BufEnd, R_X86_64_TPOFF32, P, SA);
     return 0;
   }
   llvm_unreachable("Unknown TLS optimization");
 }
 
 void X86_64TargetInfo::relocateOne(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type,

ELF/Writer.cpp

for (const RelType &RI : Rels) {
     if (Body && Body->isUndefined() && !Body->isWeak())
       if (auto *S = dyn_cast<SharedSymbol<ELFT>>(Body->repl()))
         S->File->IsUsed = true;
 
     if (Body)
       Body = Body->repl();
 
     if (Body && Body->isTLS() && Target->isTlsGlobalDynamicReloc(Type)) {
-      if (Target->isTlsOptimized(Type, Body))
+      bool PartialOpt = false;
+      bool Optimized = Target->isTlsOptimized(Type, Body, &PartialOpt);
+      if (Optimized && !PartialOpt)
         continue;
+
+      if (!Optimized) {
         if (Body->isInGot())
           continue;
         Out<ELFT>::Got->addDynTlsEntry(Body);
         Out<ELFT>::RelaDyn->addReloc({&C, &RI});
         Out<ELFT>::RelaDyn->addReloc({nullptr, nullptr});
         Body->setUsedInDynamicReloc();
         continue;
       }
+    }
 
     if ((Body && Body->isTLS()) && !Target->isTlsDynReloc(Type))
       continue;
 
     bool NeedsGot = false;
     bool NeedsPlt = false;
     if (Body) {
       if (auto *E = dyn_cast<SharedSymbol<ELFT>>(Body)) {

test/ELF/Inputs/tls-opt-gdie.s

+.type tlsshared0,@object
+.section .tbss,"awT",@nobits
+.globl tlsshared0
+.align 4
+tlsshared0:
+ .long 0
+ .size tlsshared0, 4
+
+.type  tlsshared1,@object
+.globl tlsshared1
+.align 4
+tlsshared1:
+ .long 0
+ .size tlsshared1, 4
+
+.text
+.globl __tls_get_addr
+.align 16, 0x90
+.type __tls_get_addr,@function
+__tls_get_addr:

test/ELF/tls-opt-gdie.s

+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/tls-opt-gdie.s -o %tso.o
+// RUN: ld.lld -shared %tso.o -o %t.so
+// RUN: ld.lld %t.o %t.so -o %t1
+// RUN: llvm-readobj -s -r %t1 | FileCheck --check-prefix=RELOC %s
+// RUN: llvm-objdump -d %t1 | FileCheck --check-prefix=DISASM %s
+
+//RELOC:      Section {
+//RELOC:      Index: 9
+//RELOC-NEXT: Name: .got
+//RELOC-NEXT: Type: SHT_PROGBITS
+//RELOC-NEXT: Flags [
+//RELOC-NEXT:   SHF_ALLOC
+//RELOC-NEXT:   SHF_WRITE
+//RELOC-NEXT: ]
+//RELOC-NEXT: Address: 0x120E0
+//RELOC-NEXT: Offset: 0x20E0
+//RELOC-NEXT: Size: 16
+//RELOC-NEXT: Link: 0
+//RELOC-NEXT: Info: 0
+//RELOC-NEXT: AddressAlignment: 8
+//RELOC-NEXT: EntrySize: 0
+//RELOC-NEXT: }
+//RELOC:      Relocations [
+//RELOC-NEXT:   Section (4) .rela.dyn {
+//RELOC-NEXT:     0x120E0 R_X86_64_TPOFF64 tlsshared0 0x0
+//RELOC-NEXT:     0x120E8 R_X86_64_TPOFF64 tlsshared1 0x0
+//RELOC-NEXT:   }
+//RELOC-NEXT:   Section (5) .rela.plt {
+//RELOC-NEXT:     0x12108 R_X86_64_JUMP_SLOT __tls_get_addr 0x0
+//RELOC-NEXT:   }
+//RELOC-NEXT: ]
+
+//0x11009 + (4304 + 7) = 0x120E0
+//0x11019 + (4296 + 7) = 0x120E8
+// DISASM:      Disassembly of section .text:
+// DISASM-NEXT: _start:
+// DISASM-NEXT: 11000: 64 48 8b 04 25 00 00 00 00 movq %fs:0, %rax
+// DISASM-NEXT: 11009: 48 03 05 d0 10 00 00       addq 4304(%rip), %rax
+// DISASM-NEXT: 11010: 64 48 8b 04 25 00 00 00 00 movq %fs:0, %rax
+// DISASM-NEXT: 11019: 48 03 05 c8 10 00 00       addq 4296(%rip), %rax
+
+.section .text
+.globl _start
+_start:
+ .byte 0x66
+ leaq tlsshared0@tlsgd(%rip),%rdi
+ .word 0x6666
+ rex64
+ call __tls_get_addr@plt
+ .byte 0x66
+ leaq tlsshared1@tlsgd(%rip),%rdi
+ .word 0x6666
+ rex64
+ call __tls_get_addr@plt