[BOLT] Support input binaries that use R_X86_GOTPC64

[rafauler]

Summary

In large code model, the address of GOT is calculated by the
static linker via R_X86_GOTPC64 reloc applied against a MOVABSQ
instruction. In the final binary, it can be disassembled as a regular
immediate, but because such immediate is the result of PC-relative
pointer arithmetic, we need to parse this relocation and update this
calculation whenever we move code, otherwise we break the code trying
to read GOT.

A test case showing how GOT is accessed was provided.

[maksfb]

bolt/lib/Target/X86/X86MCSymbolizer.cpp

164

Can you encapsulate GOT MCSymbol lookup in a BinaryContext method?

[rafauler]

bolt/lib/Target/X86/X86MCSymbolizer.cpp

164

Absolutely

[rafauler]

Reviewer suggestions

[rafauler]

In this solution, we don't need to mark any symbols as globals and therefore I've removed the dependency on the parent diff "[BOLT] Local hidden should be global syms."

The consequence is that we need to special-case the recognition of the GOT symbol in two more places, in addition to the X86MCSymbolizer: analyzeRelocation() and in JITLinkLinker when resolving the address of symbols.

[rafauler]

Triggering build tests again after removing parent revs

[rafauler]

Fix stack alignment in testcases, which caused libc's puts() stack to
crash in ubuntu systems because libc was using movaps in malloc
initialization.

[maksfb]

bolt/include/bolt/Core/BinaryContext.h

876

nit:

879

[maksfb]

Thanks for adding one more test case.

bolt/lib/Rewrite/JITLinkLinker.cpp

147

When can GOT be a jump table?

bolt/lib/Target/X86/X86MCSymbolizer.cpp

167

176

183–184

Should we just always take this path instead of searching for the matching name?

bolt/test/runtime/X86/gotoff-large-code-model-2.s

11

bolt/test/runtime/X86/gotoff-large-code-model.s

8

nit:

[rafauler]

Suggestions

[rafauler]

@maksfb thanks for the review

[rafauler]

Forgot to update references to renamed functions

[maksfb]

LGTM

bolt/lib/Rewrite/JITLinkLinker.cpp

147

bolt/include/bolt/Core/BinaryContext.h

const BinaryData *getPLTBinaryDataByName(StringRef Name) const {
     // without it.
     size_t At = Name.find("@");
     if (At != std::string::npos)
       return getBinaryDataByName(Name.str().substr(0, At) + "@PLT");
 
     return nullptr;
   }
 
+  /// Retrieves a reference to ELF's _GLOBAL_OFFSET_TABLE_ symbol, which points
+  /// at GOT, or null if it is not present in the input binary symtab.
+  BinaryData *getGOTSymbol();
+
+  /// Checks if symbol name refers to ELF's _GLOBAL_OFFSET_TABLE_ symbol
+  bool isGOTSymbol(StringRef SymName) const {
+    return SymName == "_GLOBAL_OFFSET_TABLE_";
+  }
+
   /// Return true if \p SymbolName was generated internally and was not present
   /// in the input binary.
   bool isInternalSymbolName(const StringRef Name) {
     return Name.startswith("SYMBOLat") || Name.startswith("DATAat") ||
            Name.startswith("HOLEat");
   }
 
   MCSymbol *getHotTextStartSymbol() const {

bolt/include/bolt/Core/Relocation.h

struct Relocation {
   /// Check if \p Type is a supported relocation type.
   static bool isSupported(uint64_t Type);
 
   /// Return true if relocation type implies the creation of a GOT entry
   static bool isGOT(uint64_t Type);
 
   /// Special relocation type that allows the linker to modify the instruction.
   static bool isX86GOTPCRELX(uint64_t Type);
+  static bool isX86GOTPC64(uint64_t Type);
 
   /// Return true if relocation type is NONE
   static bool isNone(uint64_t Type);
 
   /// Return true if relocation type is RELATIVE
   static bool isRelative(uint64_t Type);
 
   /// Return true if relocation type is IRELATIVE

bolt/lib/Core/BinaryContext.cpp

while (BD) {
       if (BD->containsAddress(Address))
         return BD;
       BD = BD->getParent();
     }
   }
   return nullptr;
 }
 
+BinaryData *BinaryContext::getGOTSymbol() {
+  // First tries to find a global symbol with that name
+  BinaryData *GOTSymBD = getBinaryDataByName("_GLOBAL_OFFSET_TABLE_");
+  if (GOTSymBD)
+    return GOTSymBD;
+
+  // This symbol might be hidden from run-time link, so fetch the local
+  // definition if available.
+  GOTSymBD = getBinaryDataByName("_GLOBAL_OFFSET_TABLE_/1");
+  if (!GOTSymBD)
+    return nullptr;
+
+  // If the local symbol is not unique, fail
+  unsigned Index = 2;
+  SmallString<30> Storage;
+  while (const BinaryData *BD =
+             getBinaryDataByName(Twine("_GLOBAL_OFFSET_TABLE_/")
+                                     .concat(Twine(Index++))
+                                     .toStringRef(Storage)))
+    if (BD->getAddress() != GOTSymBD->getAddress())
+      return nullptr;
+
+  return GOTSymBD;
+}
+
 bool BinaryContext::setBinaryDataSize(uint64_t Address, uint64_t Size) {
   auto NI = BinaryDataMap.find(Address);
   assert(NI != BinaryDataMap.end());
   if (NI == BinaryDataMap.end())
     return false;
   // TODO: it's possible that a jump table starts at the same address
   // as a larger blob of private data.  When we set the size of the
   // jump table, it might be smaller than the total blob size.  In this

bolt/lib/Core/Relocation.cpp

static bool isSupportedX86(uint64_t Type) {
   case ELF::R_X86_64_16:
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
   case ELF::R_X86_64_64:
   case ELF::R_X86_64_PC8:
   case ELF::R_X86_64_PC32:
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_PLT32:
+  case ELF::R_X86_64_GOTPC64:
   case ELF::R_X86_64_GOTPCREL:
   case ELF::R_X86_64_GOTTPOFF:
   case ELF::R_X86_64_TPOFF32:
   case ELF::R_X86_64_GOTPCRELX:
   case ELF::R_X86_64_REX_GOTPCRELX:
     return true;
   }
 }
static size_t getSizeForTypeX86(uint64_t Type) {
   case ELF::R_X86_64_GOTPCREL:
   case ELF::R_X86_64_GOTTPOFF:
   case ELF::R_X86_64_TPOFF32:
   case ELF::R_X86_64_GOTPCRELX:
   case ELF::R_X86_64_REX_GOTPCRELX:
     return 4;
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_64:
+  case ELF::R_X86_64_GOTPC64:
     return 8;
   }
 }
 
 static size_t getSizeForTypeAArch64(uint64_t Type) {
   switch (Type) {
   default:
     errs() << object::getELFRelocationTypeName(ELF::EM_AARCH64, Type) << '\n';
case ELF::R_X86_64_TPOFF32:
     return false;
   case ELF::R_X86_64_PC8:
   case ELF::R_X86_64_PC32:
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_GOTPCREL:
   case ELF::R_X86_64_PLT32:
   case ELF::R_X86_64_GOTOFF64:
   case ELF::R_X86_64_GOTPC32:
+  case ELF::R_X86_64_GOTPC64:
   case ELF::R_X86_64_GOTTPOFF:
   case ELF::R_X86_64_GOTPCRELX:
   case ELF::R_X86_64_REX_GOTPCRELX:
     return true;
   }
 }
 
 static bool isPCRelativeAArch64(uint64_t Type) {

 }
 
 bool Relocation::isX86GOTPCRELX(uint64_t Type) {
   if (Arch != Triple::x86_64)
     return false;
   return Type == ELF::R_X86_64_GOTPCRELX || Type == ELF::R_X86_64_REX_GOTPCRELX;
 }
 
+bool Relocation::isX86GOTPC64(uint64_t Type) {
+  if (Arch != Triple::x86_64)
+    return false;
+  return Type == ELF::R_X86_64_GOTPC64;
+}
+
 bool Relocation::isNone(uint64_t Type) { return Type == getNone(); }
 
 bool Relocation::isRelative(uint64_t Type) {
   if (Arch == Triple::aarch64)
     return Type == ELF::R_AARCH64_RELATIVE;
   if (Arch == Triple::riscv64)
     return Type == ELF::R_RISCV_RELATIVE;
   return Type == ELF::R_X86_64_RELATIVE;

bolt/lib/Rewrite/JITLinkLinker.cpp

for (const auto &Symbol : Symbols) {
                                ? I->getOutputAddress()
                                : I->getAddress();
         LLVM_DEBUG(dbgs() << "Resolved to address 0x"
                           << Twine::utohexstr(Address) << "\n");
         AllResults[Symbol.first] = orc::ExecutorSymbolDef(
             orc::ExecutorAddr(Address), JITSymbolFlags());
         continue;
       }
+
+      if (Linker.BC.isGOTSymbol(SymName)) {
+        if (const BinaryData *I = Linker.BC.getGOTSymbol()) {
+          uint64_t Address =
+              I->isMoved() ? I->getOutputAddress() : I->getAddress();
+          LLVM_DEBUG(dbgs() << "Resolved to address 0x"
+                            << Twine::utohexstr(Address) << "\n");
+          AllResults[Symbol.first] = orc::ExecutorSymbolDef(
+              orc::ExecutorAddr(Address), JITSymbolFlags());
+          continue;
+        }
+      }
+
       LLVM_DEBUG(dbgs() << "Resolved to address 0x0\n");
       AllResults[Symbol.first] =
           orc::ExecutorSymbolDef(orc::ExecutorAddr(0), JITSymbolFlags());
     }
 
     LC->run(std::move(AllResults));
   }
 

bolt/lib/Rewrite/RewriteInstance.cpp

if (!IsAArch64 && !ContainingBF->containsAddress(Rel.getOffset())) {
                           *ContainingBF);
       ContainingBF->setSize(ContainingBF->getMaxSize());
       ContainingBF->setSimple(false);
       return;
     }
   }
 
   MCSymbol *ReferencedSymbol = nullptr;
-  if (!IsSectionRelocation)
+  if (!IsSectionRelocation) {
     if (BinaryData *BD = BC->getBinaryDataByName(SymbolName))
       ReferencedSymbol = BD->getSymbol();
+    else if (BC->isGOTSymbol(SymbolName))
+      if (BinaryData *BD = BC->getGOTSymbol())
+        ReferencedSymbol = BD->getSymbol();
+  }
 
   ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};
   symbol_iterator SymbolIter = Rel.getSymbol();
   if (SymbolIter != InputFile->symbol_end()) {
     SymbolRef Symbol = *SymbolIter;
     section_iterator Section =
         cantFail(Symbol.getSection(), "cannot get symbol section");
     if (Section != InputFile->section_end()) {

bolt/lib/Target/X86/X86MCPlusBuilder.cpp

bool shouldRecordCodeRelocation(uint64_t RelType) const override {
     case ELF::R_X86_64_8:
     case ELF::R_X86_64_16:
     case ELF::R_X86_64_32:
     case ELF::R_X86_64_32S:
     case ELF::R_X86_64_64:
     case ELF::R_X86_64_PC8:
     case ELF::R_X86_64_PC32:
     case ELF::R_X86_64_PC64:
+    case ELF::R_X86_64_GOTPC64:
     case ELF::R_X86_64_GOTPCRELX:
     case ELF::R_X86_64_REX_GOTPCRELX:
       return true;
     case ELF::R_X86_64_PLT32:
     case ELF::R_X86_64_GOTPCREL:
     case ELF::R_X86_64_TPOFF32:
     case ELF::R_X86_64_GOTTPOFF:
       return false;

bolt/lib/Target/X86/X86MCSymbolizer.h

 namespace llvm {
 namespace bolt {
 
 class X86MCSymbolizer : public MCSymbolizer {
 protected:
   BinaryFunction &Function;
   bool CreateNewSymbols{true};
 
+  std::pair<MCSymbol *, uint64_t> handleGOTPC64(const Relocation &R,
+                                                uint64_t InstrAddr);
+
 public:
   X86MCSymbolizer(BinaryFunction &Function, bool CreateNewSymbols = true)
       : MCSymbolizer(*Function.getBinaryContext().Ctx.get(), nullptr),
         Function(Function), CreateNewSymbols(CreateNewSymbols) {}
 
   X86MCSymbolizer(const X86MCSymbolizer &) = delete;
   X86MCSymbolizer &operator=(const X86MCSymbolizer &) = delete;
   virtual ~X86MCSymbolizer();

bolt/lib/Target/X86/X86MCSymbolizer.cpp

bool X86MCSymbolizer::tryAddingSymbolicOperand(
 
   // Check for relocations against the operand.
   if (!Relocation || Relocation->Offset != InstOffset + ImmOffset)
     Relocation = Function.getRelocationAt(InstOffset + ImmOffset);
 
   if (!Relocation)
     return processPCRelOperandNoRel();
 
+  // GOTPC64 is special because the X86 Assembler doesn't know how to emit
+  // a PC-relative 8-byte fixup, which is what we need to cover this. The
+  // only way to do this is to use the symbol name _GLOBAL_OFFSET_TABLE_.
+  if (Relocation::isX86GOTPC64(Relocation->Type)) {
+    auto [Sym, Addend] = handleGOTPC64(*Relocation, InstAddress);
+    addOperand(Sym, Addend);
+    return true;
+  }
+
   uint64_t SymbolValue = Relocation->Value - Relocation->Addend;
   if (Relocation->isPCRelative())
     SymbolValue += InstAddress + ImmOffset;
 
   // Process reference to the symbol.
   if (CreateNewSymbols)
     BC.handleAddressRef(SymbolValue, Function, Relocation->isPCRelative());
 
   uint64_t Addend = Relocation->Addend;
   // Real addend for pc-relative targets is adjusted with a delta from
   // the relocation placement to the next instruction.
   if (Relocation->isPCRelative())
     Addend += InstOffset + InstSize - Relocation->Offset;
 
   addOperand(Relocation->Symbol, Addend);
 
   return true;
 }
 
+std::pair<MCSymbol *, uint64_t>
+X86MCSymbolizer::handleGOTPC64(const Relocation &R, uint64_t InstrAddr) {
+  BinaryContext &BC = Function.getBinaryContext();
+  const BinaryData *GOTSymBD = BC.getGOTSymbol();
+  if (!GOTSymBD || !GOTSymBD->getAddress()) {
+    errs() << "BOLT-ERROR: R_X86_GOTPC64 relocation is present but we did "
+              "not detect a valid  _GLOBAL_OFFSET_TABLE_ in symbol table\n";
+    exit(1);
+  }
+  // R_X86_GOTPC64 are not relative to the Reloc nor end of instruction,
+  // but the start of the MOVABSQ instruction. So the Target Address is
+  // whatever is encoded in the original operand when we disassembled
+  // the binary (here, R.Value) plus MOVABSQ address (InstrAddr).
+  // Here we extract the intended Addend by subtracting the real
+  // GOT addr.
+  const int64_t Addend = R.Value + InstrAddr - GOTSymBD->getAddress();
+  return std::make_pair(BC.Ctx->getOrCreateSymbol("_GLOBAL_OFFSET_TABLE_"),
+                        Addend);
+}
+
 void X86MCSymbolizer::tryAddingPcLoadReferenceComment(raw_ostream &CStream,
                                                       int64_t Value,
                                                       uint64_t Address) {}
 
 } // namespace bolt
 } // namespace llvm

bolt/test/runtime/X86/gotoff-large-code-model-2.s

+# A variation of gotoff-large-code-model.s that accesses GOT value
+# with a slightly different code sequence.
+
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown \
+# RUN:   %s -o %t.o
+# RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q
+
+# RUN: llvm-bolt %t.exe --funcs init_impls --lite \
+# RUN:   -o %t.bolted
+# RUN: %t.bolted | FileCheck %s
+
+  .section  .rodata.str1.1,"aMS",@progbits,1
+.LC2:
+  .string "Hello, world\n"
+  .text
+  .p2align 4
+  .globl  init_impls
+  .type init_impls, @function
+init_impls:
+  .cfi_startproc
+  push   %rbp
+  mov    %rsp,%rbp
+  push   %r15
+  push   %rbx
+  sub    $0x8,%rsp
+  lea    1f(%rip),%rbx
+  #  R_X86_64_GOTPC64  _GLOBAL_OFFSET_TABLE_+0x2
+1: movabsq $_GLOBAL_OFFSET_TABLE_, %r11
+  add    %r11,%rbx
+  #  R_X86_64_GOTOFF64 .LC2
+  movabs $.LC2@gotoff,%rax
+  lea    (%rbx,%rax,1),%rax
+  mov    %rax,%rdi
+  mov    %rbx,%r15
+  #  R_X86_64_PLTOFF64 puts
+  movabs $puts@pltoff,%rax
+  add    %rbx,%rax
+  call   *%rax
+  add    $0x8,%rsp
+  pop    %rbx
+  pop    %r15
+  pop    %rbp
+  retq
+  .cfi_endproc
+  .size init_impls, .-init_impls
+
+  .globl main
+  .type main, @function
+  .p2align 4
+main:
+  callq init_impls
+  xorq  %rax, %rax
+  ret
+
+# CHECK: Hello, world

bolt/test/runtime/X86/gotoff-large-code-model.s

+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown \
+# RUN:   %s -o %t.o
+# RUN: %clang %cflags -no-pie %t.o -o %t.exe -Wl,-q
+
+# RUN: llvm-bolt %t.exe --funcs init_impls --lite \
+# RUN:   -o %t.bolted
+# RUN: %t.bolted | FileCheck %s
+
+  .section  .rodata.str1.1,"aMS",@progbits,1
+.LC2:
+  .string "Hello, world\n"
+  .text
+  .p2align 4
+  .globl  init_impls
+  .type init_impls, @function
+init_impls:
+  .cfi_startproc
+  push   %rbp
+  mov    %rsp,%rbp
+  push   %r15
+  push   %rbx
+  sub    $0x8,%rsp
+1:
+  lea    1b(%rip),%rbx
+  #  R_X86_64_GOTPC64  _GLOBAL_OFFSET_TABLE_+0x9
+  movabsq $_GLOBAL_OFFSET_TABLE_-1b, %r11
+  add    %r11,%rbx
+  #  R_X86_64_GOTOFF64 .LC2
+  movabs $.LC2@gotoff,%rax
+  lea    (%rbx,%rax,1),%rax
+  mov    %rax,%rdi
+  mov    %rbx,%r15
+  #  R_X86_64_PLTOFF64 puts
+  movabs $puts@pltoff,%rax
+  add    %rbx,%rax
+  call   *%rax
+  add    $0x8,%rsp
+  pop    %rbx
+  pop    %r15
+  pop    %rbp
+  retq
+  .cfi_endproc
+  .size init_impls, .-init_impls
+
+  .globl main
+  .type main, @function
+  .p2align 4
+main:
+  callq init_impls
+  xorq  %rax, %rax
+  ret
+
+# CHECK: Hello, world