[ELF] - Implemented @tlsgd optimization (GD->IE case, x64).

[grimar]

Summary

"Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5 x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows how GD can be optimized to IE.
This patch implements this optimization.

[ruiu]

ELF/OutputSections.cpp

226–231 ↗

(On Diff #41192)

There is really a concerning trend that this function gets longer and longer over time. Probably it's time to split up into multiple small functions with comments.

[grimar]

Review comments addressed.

ELF/OutputSections.cpp

226–231 ↗

(On Diff #41192)

Done.

[grimar]

Refactored, rebased.

[ruiu]

ELF/OutputSections.cpp

221–226 ↗

(On Diff #41622)

Can you reduce the nest level by moving this if outside of the outer `if? I mean

if (Body && Target->isTlsOptimized(Type, Body)) {
  P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
                      Target->getTlsGotReloc(), Config->Mips64EL);
  P->r_offset = Out<ELFT>::Got->getEntryAddr(*Body);
  return true;
}

if (Body && Target->isTlsGlobalDynamicReloc(Type)) {
  ...
}

ELF/Target.cpp

494–495 ↗

(On Diff #41622)

I don't think these magic numbers are good. In fact it seems that you don't need them. Please see the comment below.

624 ↗

(On Diff #41622)

This can be

if (canBePreempted(&S, true))

[grimar]

Addressed review comments.

ELF/OutputSections.cpp

221–226 ↗

(On Diff #41622)

Refactored, what about this one ?

ELF/Target.cpp

494–495 ↗

(On Diff #41622)

Cool ! Fixed.

624 ↗

(On Diff #41622)

Done.

[ruiu]

LGTM with a few nits.

ELF/Target.cpp

499 ↗

(On Diff #41769)

Remove the outermost ().

548 ↗

(On Diff #41769)

Remove newline at end of this line.

lld/trunk/ELF/InputSection.cpp

if (Target->isTlsGlobalDynamicReloc(Type) &&
         !Target->isTlsOptimized(Type, &Body)) {
       Target->relocateOne(BufLoc, BufEnd, Type, AddrLoc,
                           Out<ELFT>::Got->getGlobalDynAddr(Body) +
                               getAddend<ELFT>(RI));
       continue;
     }
 
     if (Target->isTlsOptimized(Type, &Body)) {
+      uintX_t SymVA = Target->relocNeedsGot(Type, Body)
+                          ? Out<ELFT>::Got->getEntryAddr(Body)
+                          : getSymVA<ELFT>(Body);
       // By optimizing TLS relocations, it is sometimes needed to skip
       // relocations that immediately follow TLS relocations. This function
       // knows how many slots we need to skip.
-      I += Target->relocateTlsOptimize(BufLoc, BufEnd, Type, AddrLoc,
-                                       getSymVA<ELFT>(Body));
+      I += Target->relocateTlsOptimize(BufLoc, BufEnd, Type, AddrLoc, SymVA,
+                                       Body);
       continue;
     }
 
     uintX_t SymVA = getSymVA<ELFT>(Body);
     if (Target->relocNeedsPlt(Type, Body)) {
       SymVA = Out<ELFT>::Plt->getEntryAddr(Body);
       Type = Target->getPltRefReloc(Type);
     } else if (Target->relocNeedsGot(Type, Body)) {

lld/trunk/ELF/OutputSections.cpp

bool RelocationSection<ELFT>::applyTlsDynamicReloc(SymbolBody *Body,
                                                    uint32_t Type, Elf_Rel *P,
                                                    Elf_Rel *N) {
   if (Target->isTlsLocalDynamicReloc(Type)) {
     P->setSymbolAndType(0, Target->getTlsModuleIndexReloc(), Config->Mips64EL);
     P->r_offset = Out<ELFT>::Got->getLocalTlsIndexVA();
     return true;
   }
 
-  if (Body && Target->isTlsGlobalDynamicReloc(Type)) {
-    P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
-                        Target->getTlsModuleIndexReloc(), Config->Mips64EL);
-    P->r_offset = Out<ELFT>::Got->getGlobalDynAddr(*Body);
-    N->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
-                        Target->getTlsOffsetReloc(), Config->Mips64EL);
-    N->r_offset = Out<ELFT>::Got->getGlobalDynAddr(*Body) + sizeof(uintX_t);
-    return true;
-  }
-  return false;
-}
+  if (!Body || !Target->isTlsGlobalDynamicReloc(Type))
+    return false;
+
+  if (Target->isTlsOptimized(Type, Body)) {
+    P->setSymbolAndType(Body->getDynamicSymbolTableIndex(),
+                        Target->getTlsGotReloc(), Config->Mips64EL);
+    P->r_offset = Out<ELFT>::Got->getEntryAddr(*Body);
+    return true;
+  }
+
 
 template <class ELFT> void RelocationSection<ELFT>::writeTo(uint8_t *Buf) {
   const unsigned EntrySize = IsRela ? sizeof(Elf_Rela) : sizeof(Elf_Rel);
   for (const DynamicReloc<ELFT> &Rel : Relocs) {
     auto *P = reinterpret_cast<Elf_Rel *>(Buf);
     Buf += EntrySize;
 
     // Skip placeholder for global dynamic TLS relocation pair. It was already

lld/trunk/ELF/Target.h

public:
   virtual bool relocNeedsCopy(uint32_t Type, const SymbolBody &S) const;
   virtual bool relocNeedsGot(uint32_t Type, const SymbolBody &S) const = 0;
   virtual bool relocNeedsPlt(uint32_t Type, const SymbolBody &S) const = 0;
   virtual void relocateOne(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type,
                            uint64_t P, uint64_t SA,
                            uint8_t *PairedLoc = nullptr) const = 0;
   virtual bool isTlsOptimized(unsigned Type, const SymbolBody *S) const;
   virtual unsigned relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
-                                       uint32_t Type, uint64_t P,
-                                       uint64_t SA) const;
+                                       uint32_t Type, uint64_t P, uint64_t SA,
+                                       const SymbolBody &S) const;
   virtual ~TargetInfo();
 
 protected:
   unsigned PageSize = 4096;
 
   // On freebsd x86_64 the first page cannot be mmaped.
   // On linux that is controled by vm.mmap_min_addr. At least on some x86_64
   // installs that is 65536, so the first 15 pages cannot be used.

lld/trunk/ELF/Target.cpp

public:
   bool relocNeedsCopy(uint32_t Type, const SymbolBody &S) const override;
   bool relocNeedsGot(uint32_t Type, const SymbolBody &S) const override;
   bool relocNeedsPlt(uint32_t Type, const SymbolBody &S) const override;
   void relocateOne(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type, uint64_t P,
                    uint64_t SA, uint8_t *PairedLoc = nullptr) const override;
   bool isRelRelative(uint32_t Type) const override;
   bool isTlsOptimized(unsigned Type, const SymbolBody *S) const override;
   unsigned relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd, uint32_t Type,
-                               uint64_t P, uint64_t SA) const override;
+                               uint64_t P, uint64_t SA,
+                               const SymbolBody &S) const override;
 
 private:
   void relocateTlsLdToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
   void relocateTlsGdToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
+  void relocateTlsGdToIe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
+                         uint64_t SA) const;
   void relocateTlsIeToLe(uint8_t *Loc, uint8_t *BufEnd, uint64_t P,
                          uint64_t SA) const;
 };
 
 class PPC64TargetInfo final : public TargetInfo {
 public:
   PPC64TargetInfo();
   void writeGotPltEntry(uint8_t *Buf, uint64_t Plt) const override;
bool TargetInfo::relocNeedsCopy(uint32_t Type, const SymbolBody &S) const {
   return false;
 }
 
 unsigned TargetInfo::getPltRefReloc(unsigned Type) const { return PCRelReloc; }
 
 bool TargetInfo::isRelRelative(uint32_t Type) const { return true; }
 
 unsigned TargetInfo::relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
-                                         uint32_t Type, uint64_t P,
-                                         uint64_t SA) const {
+                                         uint32_t Type, uint64_t P, uint64_t SA,
+                                         const SymbolBody &S) const {
   return 0;
 }
 
 void TargetInfo::writeGotHeaderEntries(uint8_t *Buf) const {}
 
 void TargetInfo::writeGotPltHeaderEntries(uint8_t *Buf) const {}
 
 X86TargetInfo::X86TargetInfo() {
bool X86_64TargetInfo::relocNeedsCopy(uint32_t Type,
   if (Type == R_X86_64_32S || Type == R_X86_64_32 || Type == R_X86_64_PC32 ||
       Type == R_X86_64_64)
     if (auto *SS = dyn_cast<SharedSymbol<ELF64LE>>(&S))
       return SS->Sym.getType() == STT_OBJECT;
   return false;
 }
 
 bool X86_64TargetInfo::relocNeedsGot(uint32_t Type, const SymbolBody &S) const {
+  if (Type == R_X86_64_TLSGD)
+    return Target->isTlsOptimized(Type, &S) && canBePreempted(&S, true);
   if (Type == R_X86_64_GOTTPOFF)
     return !isTlsOptimized(Type, &S);
-  return Type == R_X86_64_GOTTPOFF || Type == R_X86_64_GOTPCREL ||
-         relocNeedsPlt(Type, S);
+  return Type == R_X86_64_GOTPCREL || relocNeedsPlt(Type, S);
 }
 
 bool X86_64TargetInfo::isTlsDynReloc(unsigned Type) const {
-  return Type == R_X86_64_GOTTPOFF;
+  return Type == R_X86_64_GOTTPOFF || Type == R_X86_64_TLSGD;
 }
 
 unsigned X86_64TargetInfo::getPltRefReloc(unsigned Type) const {
   if (Type == R_X86_64_PLT32)
     return R_X86_64_PC32;
   return Type;
 }
 
case R_X86_64_PLT32:
     return true;
   }
 }
 
 bool X86_64TargetInfo::isTlsOptimized(unsigned Type,
                                       const SymbolBody *S) const {
   if (Config->Shared || (S && !S->isTLS()))
     return false;
-  return Type == R_X86_64_TLSLD || Type == R_X86_64_DTPOFF32 ||
-         (Type == R_X86_64_TLSGD && !canBePreempted(S, true)) ||
+  return Type == R_X86_64_TLSGD || Type == R_X86_64_TLSLD ||
+         Type == R_X86_64_DTPOFF32 ||
          (Type == R_X86_64_GOTTPOFF && !canBePreempted(S, true));
 }
 
 // "Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5
 // x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows
 // how LD can be optimized to LE:
 //   leaq bar@tlsld(%rip), %rdi
 //   callq __tls_get_addr@PLT
void X86_64TargetInfo::relocateTlsGdToLe(uint8_t *Loc, uint8_t *BufEnd,
   const uint8_t Inst[] = {
       0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
       0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00              // lea x@tpoff,%rax
   };
   memcpy(Loc - 4, Inst, sizeof(Inst));
   relocateOne(Loc + 8, BufEnd, R_X86_64_TPOFF32, P, SA);
 }
 
+// "Ulrich Drepper, ELF Handling For Thread-Local Storage" (5.5
+// x86-x64 linker optimizations, http://www.akkadia.org/drepper/tls.pdf) shows
+// how GD can be optimized to IE:
+//  .byte 0x66
+//  leaq x@tlsgd(%rip), %rdi
+//  .word 0x6666
+//  rex64
+//  call __tls_get_addr@plt
+// Is converted to:
+//  mov %fs:0x0,%rax
+//  addq x@tpoff,%rax
+void X86_64TargetInfo::relocateTlsGdToIe(uint8_t *Loc, uint8_t *BufEnd,
+                                         uint64_t P, uint64_t SA) const {
+  const uint8_t Inst[] = {
+      0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0x0,%rax
+      0x48, 0x03, 0x05, 0x00, 0x00, 0x00, 0x00              // addq x@tpoff,%rax
+  };
+  memcpy(Loc - 4, Inst, sizeof(Inst));
+  relocateOne(Loc + 8, BufEnd, R_X86_64_TPOFF64, P + 12, SA);
+}
+
 // In some conditions, R_X86_64_GOTTPOFF relocation can be optimized to
 // R_X86_64_TPOFF32 so that R_X86_64_TPOFF32 so that it does not use GOT.
 // This function does that. Read "ELF Handling For Thread-Local Storage,
 // 5.5 x86-x64 linker optimizations" (http://www.akkadia.org/drepper/tls.pdf)
 // by Ulrich Drepper for details.
 void X86_64TargetInfo::relocateTlsIeToLe(uint8_t *Loc, uint8_t *BufEnd,
                                          uint64_t P, uint64_t SA) const {
   // Ulrich's document section 6.5 says that @gottpoff(%rip) must be

 
 // This function applies a TLS relocation with an optimization as described
 // in the Ulrich's document. As a result of rewriting instructions at the
 // relocation target, relocations immediately follow the TLS relocation (which
 // would be applied to rewritten instructions) may have to be skipped.
 // This function returns a number of relocations that need to be skipped.
 unsigned X86_64TargetInfo::relocateTlsOptimize(uint8_t *Loc, uint8_t *BufEnd,
                                                uint32_t Type, uint64_t P,
-                                               uint64_t SA) const {
+                                               uint64_t SA,
+                                               const SymbolBody &S) const {
   switch (Type) {
   case R_X86_64_DTPOFF32:
     relocateOne(Loc, BufEnd, R_X86_64_TPOFF32, P, SA);
     return 0;
   case R_X86_64_GOTTPOFF:
     relocateTlsIeToLe(Loc, BufEnd, P, SA);
     return 0;
-  case R_X86_64_TLSGD:
+  case R_X86_64_TLSGD: {
+    if (canBePreempted(&S, true))
+      relocateTlsGdToIe(Loc, BufEnd, P, SA);
+    else
       relocateTlsGdToLe(Loc, BufEnd, P, SA);
     // The next relocation should be against __tls_get_addr, so skip it
     return 1;
+  }
   case R_X86_64_TLSLD:
     relocateTlsLdToLe(Loc, BufEnd, P, SA);
     // The next relocation should be against __tls_get_addr, so skip it
     return 1;
   }
   llvm_unreachable("Unknown TLS optimization");
 }
 

lld/trunk/ELF/Writer.cpp

for (const RelType &RI : Rels) {
     if (Body && Body->isUndefined() && !Body->isWeak())
       if (auto *S = dyn_cast<SharedSymbol<ELFT>>(Body->repl()))
         S->File->IsUsed = true;
 
     if (Body)
       Body = Body->repl();
 
     if (Body && Body->isTLS() && Target->isTlsGlobalDynamicReloc(Type)) {
-      if (Target->isTlsOptimized(Type, Body))
-        continue;
-      if (Out<ELFT>::Got->addDynTlsEntry(Body)) {
+      bool Opt = Target->isTlsOptimized(Type, Body);
+      if (!Opt && Out<ELFT>::Got->addDynTlsEntry(Body)) {
         Out<ELFT>::RelaDyn->addReloc({&C, &RI});
         Out<ELFT>::RelaDyn->addReloc({nullptr, nullptr});
         Body->setUsedInDynamicReloc();
+        continue;
       }
+      if (!canBePreempted(Body, true))
         continue;
     }
 
     if (Body && Body->isTLS() && !Target->isTlsDynReloc(Type))
       continue;
 
     bool NeedsGot = false;
     bool NeedsPlt = false;
     if (Body) {

lld/trunk/test/ELF/Inputs/tls-opt-gdie.s

+.type tlsshared0,@object
+.section .tbss,"awT",@nobits
+.globl tlsshared0
+.align 4
+tlsshared0:
+ .long 0
+ .size tlsshared0, 4
+
+.type  tlsshared1,@object
+.globl tlsshared1
+.align 4
+tlsshared1:
+ .long 0
+ .size tlsshared1, 4
+
+.text
+.globl __tls_get_addr
+.align 16, 0x90
+.type __tls_get_addr,@function
+__tls_get_addr:

lld/trunk/test/ELF/tls-opt-gdie.s

+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/tls-opt-gdie.s -o %tso.o
+// RUN: ld.lld -shared %tso.o -o %t.so
+// RUN: ld.lld %t.o %t.so -o %t1
+// RUN: llvm-readobj -s -r %t1 | FileCheck --check-prefix=RELOC %s
+// RUN: llvm-objdump -d %t1 | FileCheck --check-prefix=DISASM %s
+
+//RELOC:      Section {
+//RELOC:      Index: 9
+//RELOC-NEXT: Name: .got
+//RELOC-NEXT: Type: SHT_PROGBITS
+//RELOC-NEXT: Flags [
+//RELOC-NEXT:   SHF_ALLOC
+//RELOC-NEXT:   SHF_WRITE
+//RELOC-NEXT: ]
+//RELOC-NEXT: Address: 0x120E0
+//RELOC-NEXT: Offset: 0x20E0
+//RELOC-NEXT: Size: 16
+//RELOC-NEXT: Link: 0
+//RELOC-NEXT: Info: 0
+//RELOC-NEXT: AddressAlignment: 8
+//RELOC-NEXT: EntrySize: 0
+//RELOC-NEXT: }
+//RELOC:      Relocations [
+//RELOC-NEXT:   Section (4) .rela.dyn {
+//RELOC-NEXT:     0x120E0 R_X86_64_TPOFF64 tlsshared0 0x0
+//RELOC-NEXT:     0x120E8 R_X86_64_TPOFF64 tlsshared1 0x0
+//RELOC-NEXT:   }
+//RELOC-NEXT:   Section (5) .rela.plt {
+//RELOC-NEXT:     0x12108 R_X86_64_JUMP_SLOT __tls_get_addr 0x0
+//RELOC-NEXT:   }
+//RELOC-NEXT: ]
+
+//0x11009 + (4304 + 7) = 0x120E0
+//0x11019 + (4296 + 7) = 0x120E8
+// DISASM:      Disassembly of section .text:
+// DISASM-NEXT: _start:
+// DISASM-NEXT: 11000: 64 48 8b 04 25 00 00 00 00 movq %fs:0, %rax
+// DISASM-NEXT: 11009: 48 03 05 d0 10 00 00       addq 4304(%rip), %rax
+// DISASM-NEXT: 11010: 64 48 8b 04 25 00 00 00 00 movq %fs:0, %rax
+// DISASM-NEXT: 11019: 48 03 05 c8 10 00 00       addq 4296(%rip), %rax
+
+.section .text
+.globl _start
+_start:
+ .byte 0x66
+ leaq tlsshared0@tlsgd(%rip),%rdi
+ .word 0x6666
+ rex64
+ call __tls_get_addr@plt
+ .byte 0x66
+ leaq tlsshared1@tlsgd(%rip),%rdi
+ .word 0x6666
+ rex64
+ call __tls_get_addr@plt