[ELF] Support non-RAX/non-adjacent R_X86_64_GOTPC32_TLSDESC/R_X86_64_TLSDESC_CALL

[MaskRay]

Summary

The current TLSDESC optimization code assumes:

leaq x@tlsdesc(%rip), %rax
call *x@tlscall(%rax)       # adjacent

From https://gitlab.freedesktop.org/mesa/mesa/-/issues/5665 , it seems that the
two instructions may not be adjacent in GCC 10's output:

leaq x@tlsdesc(%rip), %rax
something else
call *x@tlscall(%rax)

This patch supports the case. While here, support non-RAX registers for
R_X86_64_GOTPC32_TLSDESC, in case the compiler generates inefficient:

leaq x@tlsdesc(%rip), %rcx  # or %rdx, %rbx, %rdi, ...
movq %rcx, %rax
call *x@tlscall(%rax)       # GNU ld/gold error for non-RAX

[skan]

lld/ELF/Arch/X86_64.cpp

452

REX.X and REX.B does not affect RIP-relative addressing. I believe we should use mask
0xf8 for loc[-3].

[skan]

lld/ELF/Arch/X86_64.cpp

459

Here we'd like to keep the fixed prefix 0100 and set REX.B to the value of REX.R. We already checked REX.W =1 before. I think code can be simplified to

loc[-3] = 0x48 | ((loc[-3] >>2) & 1);

461

Here we set mod=11, reg=000, and set the rm to the value of reg. LGTM.

[MaskRay]

address comments

[MaskRay]

lld/ELF/Arch/X86_64.cpp

452

I think we should still use 0xfb. I cannot get an assembler to use the REX.X and REX.B bits.

[MaskRay]

Change description

[MaskRay]

update description

lld/ELF/Arch/X86_64.cpp

X86_64::X86_64() {
   trapInstr = {0xcc, 0xcc, 0xcc, 0xcc}; // 0xcc = INT3
   nopInstrs = nopInstructions;
 
   // Align to the large page size (known as a superpage or huge page).
   // FreeBSD automatically promotes large, superpage-aligned allocations.
   defaultImageBase = 0x200000;
 }
 
-int X86_64::getTlsGdRelaxSkip(RelType type) const { return 2; }
+int X86_64::getTlsGdRelaxSkip(RelType type) const {
+  // TLSDESC relocations are processed separately. See relaxTlsGdToLe below.
+  return type == R_X86_64_GOTPC32_TLSDESC || type == R_X86_64_TLSDESC_CALL ? 1
+                                                                           : 2;
+}
 
 // Opcodes for the different X86_64 jmp instructions.
 enum JmpInsnOpcode : uint32_t {
   J_JMP_32,
   J_JNE_32,
   J_JE_32,
   J_JG_32,
   J_JGE_32,
const uint8_t inst[] = {
         0x00, 0x00,                            // mov %fs:0x0,%rax
         0x48, 0x8d, 0x80, 0,    0,    0,    0, // lea x@tpoff,%rax
     };
     memcpy(loc - 4, inst, sizeof(inst));
 
     // The original code used a pc relative relocation and so we have to
     // compensate for the -4 in had in the addend.
     write32le(loc + 8, val + 4);
-  } else {
-    // Convert
-    //   lea x@tlsgd(%rip), %rax
-    //   call *(%rax)
-    // to the following two instructions.
-    assert(rel.type == R_X86_64_GOTPC32_TLSDESC);
-    if (memcmp(loc - 3, "\x48\x8d\x05", 3)) {
-      error(getErrorLocation(loc - 3) + "R_X86_64_GOTPC32_TLSDESC must be used "
-                                        "in callq *x@tlsdesc(%rip), %rax");
+  } else if (rel.type == R_X86_64_GOTPC32_TLSDESC) {
+    // Convert leaq x@tlsdesc(%rip), %REG to movq $x@tpoff, %REG.
+    if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
+        (loc[-1] & 0xc7) != 0x05) {
+      errorOrWarn(getErrorLocation(loc - 3) +
+                  "R_X86_64_GOTPC32_TLSDESC must be used "
+                  "in leaq x@tlsdesc(%rip), %REG");
       return;
     }
-    // movq $x@tpoff(%rip),%rax
+    loc[-3] = 0x48 | ((loc[-3] >> 2) & 1);
     loc[-2] = 0xc7;
-    loc[-1] = 0xc0;
+    loc[-1] = 0xc0 | ((loc[-1] >> 3) & 7);
     write32le(loc, val + 4);
-    // xchg ax,ax
-    loc[4] = 0x66;
-    loc[5] = 0x90;
+  } else {
+    // Convert call *x@tlsdesc(%REG) to xchg ax, ax.
+    assert(rel.type == R_X86_64_TLSDESC_CALL);
+    loc[0] = 0x66;
+    loc[1] = 0x90;
   }
 }
 
 void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
                             uint64_t val) const {
   if (rel.type == R_X86_64_TLSGD) {
     // Convert
     //   .byte 0x66
     //   leaq x@tlsgd(%rip), %rdi
     //   .word 0x6666
     //   rex64
     //   call __tls_get_addr@plt
     // to the following two instructions.
     const uint8_t inst[] = {
         0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00,
         0x00, 0x00,                            // mov %fs:0x0,%rax
         0x48, 0x03, 0x05, 0,    0,    0,    0, // addq x@gottpoff(%rip),%rax
     };
     memcpy(loc - 4, inst, sizeof(inst));
 
     // Both code sequences are PC relatives, but since we are moving the
     // constant forward by 8 bytes we have to subtract the value by 8.
     write32le(loc + 8, val - 8);
-  } else {
-    // Convert
-    //   lea x@tlsgd(%rip), %rax
-    //   call *(%rax)
-    // to the following two instructions.
+  } else if (rel.type == R_X86_64_GOTPC32_TLSDESC) {
+    // Convert leaq x@tlsdesc(%rip), %REG to movq x@gottpoff(%rip), %REG.
     assert(rel.type == R_X86_64_GOTPC32_TLSDESC);
-    if (memcmp(loc - 3, "\x48\x8d\x05", 3)) {
-      error(getErrorLocation(loc - 3) + "R_X86_64_GOTPC32_TLSDESC must be used "
-                                        "in callq *x@tlsdesc(%rip), %rax");
+    if ((loc[-3] & 0xfb) != 0x48 || loc[-2] != 0x8d ||
+        (loc[-1] & 0xc7) != 0x05) {
+      errorOrWarn(getErrorLocation(loc - 3) +
+                  "R_X86_64_GOTPC32_TLSDESC must be used "
+                  "in leaq x@tlsdesc(%rip), %REG");
       return;
     }
-    // movq x@gottpoff(%rip),%rax
     loc[-2] = 0x8b;
     write32le(loc, val);
-    // xchg ax,ax
-    loc[4] = 0x66;
-    loc[5] = 0x90;
+  } else {
+    // Convert call *x@tlsdesc(%rax) to xchg ax, ax.
+    assert(rel.type == R_X86_64_TLSDESC_CALL);
+    loc[0] = 0x66;
+    loc[1] = 0x90;
   }
 }
 
 // In some conditions, R_X86_64_GOTTPOFF relocation can be optimized to
 // R_X86_64_TPOFF32 so that it does not use GOT.
 void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &,
                             uint64_t val) const {
   uint8_t *inst = loc - 3;

lld/docs/ReleaseNotes.rst

 * ``e_entry`` no longer falls back to the address of ``.text`` if the entry symbol does not exist.
   Instead, a value of 0 will be written.
   (`D110014 <https://reviews.llvm.org/D110014>`_)
 
 Architecture specific changes:
 
 * The x86-32 port now supports TLSDESC (``-mtls-dialect=gnu2``).
   (`D112582 <https://reviews.llvm.org/D112582>`_)
+* The x86-64 port now handles non-RAX/non-adjacent ``R_X86_64_GOTPC32_TLSDESC``
+  and ``R_X86_64_TLSDESC_CALL`` (``-mtls-dialect=gnu2``).
+  (`D114416 <https://reviews.llvm.org/D114416>`_)
 * For x86-64, ``--no-relax`` now suppresses ``R_X86_64_GOTPCRELX`` and
   ``R_X86_64_REX_GOTPCRELX`` GOT optimization
   (`D113615 <https://reviews.llvm.org/D113615>`_)
 
 Breaking changes
 ----------------
 
 * ...

lld/test/ELF/invalid/x86-64-tlsdesc-gd.s

 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
 # RUN: echo '.tbss; .globl a; a:' | llvm-mc -filetype=obj -triple=x86_64 - -o %t1.o
 # RUN: ld.lld -shared %t1.o -o %t1.so
 
 ## GD to LE relaxation.
 # RUN: not ld.lld %t.o %t1.o -o /dev/null 2>&1 | FileCheck -DINPUT=%t.o %s
 ## GD to IE relaxation.
 # RUN: not ld.lld %t.o %t1.so -o /dev/null 2>&1 | FileCheck -DINPUT=%t.o %s
 
-# CHECK: error: [[INPUT]]:(.text+0x0): R_X86_64_GOTPC32_TLSDESC must be used in callq *x@tlsdesc(%rip), %rax
+# CHECK: error: [[INPUT]]:(.text+0x0): R_X86_64_GOTPC32_TLSDESC must be used in leaq x@tlsdesc(%rip), %REG
 
-leaq a@tlsdesc(%rip), %rdx
+leaq a@tlsdesc(%rbx), %rdx
 call *a@tlscall(%rdx)
 movl %fs:(%rax), %eax

lld/test/ELF/x86-64-tlsdesc-gd.s

 # RUN: llvm-readelf -r %t | FileCheck --check-prefix=NOREL %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=LE %s
 
 # RUN: ld.lld %t.o %t1.so -o %t
 # RUN: llvm-readobj -r %t | FileCheck --check-prefix=IE-REL %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=IE %s
 
 # GD-RELA:      .rela.dyn {
-# GD-RELA-NEXT:   0x23B8 R_X86_64_TLSDESC - 0xB
-# GD-RELA-NEXT:   0x23A8 R_X86_64_TLSDESC a 0x0
-# GD-RELA-NEXT:   0x23C8 R_X86_64_TLSDESC c 0x0
+# GD-RELA-NEXT:   0x23C0 R_X86_64_TLSDESC - 0xB
+# GD-RELA-NEXT:   0x23B0 R_X86_64_TLSDESC a 0x0
+# GD-RELA-NEXT:   0x23D0 R_X86_64_TLSDESC c 0x0
 # GD-RELA-NEXT: }
 # GD-RELA:      Hex dump of section '.got':
-# GD-RELA-NEXT: 0x000023a8 00000000 00000000 00000000 00000000
-# GD-RELA-NEXT: 0x000023b8 00000000 00000000 00000000 00000000
-# GD-RELA-NEXT: 0x000023c8 00000000 00000000 00000000 00000000
+# GD-RELA-NEXT: 0x000023b0 00000000 00000000 00000000 00000000
+# GD-RELA-NEXT: 0x000023c0 00000000 00000000 00000000 00000000
+# GD-RELA-NEXT: 0x000023d0 00000000 00000000 00000000 00000000
 
 # GD-REL:       .rel.dyn {
-# GD-REL-NEXT:    0x23A0 R_X86_64_TLSDESC -
-# GD-REL-NEXT:    0x2390 R_X86_64_TLSDESC a
-# GD-REL-NEXT:    0x23B0 R_X86_64_TLSDESC c
+# GD-REL-NEXT:    0x23A8 R_X86_64_TLSDESC -
+# GD-REL-NEXT:    0x2398 R_X86_64_TLSDESC a
+# GD-REL-NEXT:    0x23B8 R_X86_64_TLSDESC c
 # GD-REL-NEXT:  }
 # GD-REL:       Hex dump of section '.got':
-# GD-REL-NEXT:  0x00002390 00000000 00000000 00000000 00000000
-# GD-REL-NEXT:  0x000023a0 00000000 00000000 0b000000 00000000
-# GD-REL-NEXT:  0x000023b0 00000000 00000000 00000000 00000000
+# GD-REL-NEXT:  0x00002398 00000000 00000000 00000000 00000000
+# GD-REL-NEXT:  0x000023a8 00000000 00000000 0b000000 00000000
+# GD-REL-NEXT:  0x000023b8 00000000 00000000 00000000 00000000
 
-## &.rela.dyn[a]-pc = 0x23A8-0x12e7 = 4289
-# GD:            leaq 4289(%rip), %rax
+## &.rela.dyn[a]-pc = 0x23B0-0x12e7 = 4297
+# GD:            leaq 4297(%rip), %rax
 # GD-NEXT: 12e7: callq *(%rax)
 # GD-NEXT:       movl %fs:(%rax), %eax
 
-## &.rela.dyn[b]-pc = 0x23B8-0x12f3 = 4293
-# GD-NEXT:       leaq 4293(%rip), %rax
-# GD-NEXT: 12f3: callq *(%rax)
+## &.rela.dyn[b]-pc = 0x23C0-0x12f3 = 4301
+# GD-NEXT:       leaq 4301(%rip), %rcx
+# GD-NEXT: 12f3: movq %rcx, %rax
+# GD-NEXT:       callq *(%rax)
 # GD-NEXT:       movl %fs:(%rax), %eax
 
-## &.rela.dyn[c]-pc = 0x23C8-0x12f3 = 4297
-# GD-NEXT:       leaq 4297(%rip), %rax
-# GD-NEXT: 12ff: callq *(%rax)
+## &.rela.dyn[c]-pc = 0x23D0-0x1302 = 4302
+# GD-NEXT:       leaq 4302(%rip), %r15
+# GD-NEXT: 1302: movq %r15, %rax
+# GD-NEXT:       callq *(%rax)
 # GD-NEXT:       movl %fs:(%rax), %eax
 
 # NOREL: no relocations
 
 ## tpoff(a) = st_value(a) - tls_size = -8
 # LE:      movq $-8, %rax
 # LE-NEXT: nop
 # LE-NEXT: movl %fs:(%rax), %eax
 ## tpoff(b) = st_value(b) - tls_size = -5
-# LE:      movq $-5, %rax
+# LE:      movq $-5, %rcx
+# LE-NEXT: movq %rcx, %rax
 # LE-NEXT: nop
 # LE-NEXT: movl %fs:(%rax), %eax
 ## tpoff(c) = st_value(c) - tls_size = -4
-# LE:      movq $-4, %rax
+# LE:      movq $-4, %r15
+# LE-NEXT: movq %r15, %rax
 # LE-NEXT: nop
 # LE-NEXT: movl %fs:(%rax), %eax
 
 # IE-REL:      .rela.dyn {
-# IE-REL-NEXT:   0x202370 R_X86_64_TPOFF64 c 0x0
+# IE-REL-NEXT:   0x202378 R_X86_64_TPOFF64 c 0x0
 # IE-REL-NEXT: }
 
 ## a is relaxed to use LE.
 # IE:              movq $-4, %rax
 # IE-NEXT:         nop
 # IE-NEXT:         movl %fs:(%rax), %eax
-# IE-NEXT:         movq $-1, %rax
+# IE-NEXT:         movq $-1, %rcx
+# IE-NEXT:         movq %rcx, %rax
 # IE-NEXT:         nop
 # IE-NEXT:         movl %fs:(%rax), %eax
-## &.rela.dyn[c]-pc = 0x202370 - 0x2012a7 = 4297
-# IE-NEXT:         movq 4297(%rip), %rax
-# IE-NEXT: 2012a7: nop
+## &.rela.dyn[c]-pc = 0x202378 - 0x2012aa = 4302
+# IE-NEXT:         movq 4302(%rip), %r15
+# IE-NEXT: 2012aa: movq %r15, %rax
+# IE-NEXT:         nop
 # IE-NEXT:         movl %fs:(%rax), %eax
 
 leaq a@tlsdesc(%rip), %rax
 call *a@tlscall(%rax)
 movl %fs:(%rax), %eax
 
-leaq b@tlsdesc(%rip), %rax
+## leaq/call may not be adjacent:  https://gitlab.freedesktop.org/mesa/mesa/-/issues/5665
+## Test non-RAX registers as well.
+leaq b@tlsdesc(%rip), %rcx
+movq %rcx, %rax
 call *b@tlscall(%rax)
 movl %fs:(%rax), %eax
 
-leaq c@tlsdesc(%rip), %rax
+leaq c@tlsdesc(%rip), %r15
+movq %r15, %rax
 call *c@tlscall(%rax)
 movl %fs:(%rax), %eax
 
 .section .tbss
 .globl a
 .zero 8
 a:
 .zero 3
 b:
 .zero 1