[X86] Selective relocation relaxation for +tagged-globals

[morehouse]

Summary

For tagged-globals, we only need to disable relaxation for globals that
we actually tag. With this patch function pointer relocations, which
we do not instrument, can be relaxed.

This patch also makes tagged-globals work properly with LTO, as
-Wa,-mrelax-relocations=no doesn't work with LTO.

[pcc]

I think it ought to be possible to write these non-relaxable relocations from assembly, so that the behavior with -c is equivalent to the behavior with -S and then assembling the output. For example, you should be able to assemble

mov foo@gotpcrel_norelax(%rip), %rax

and the -S output from the compiler should look like that as well.

[morehouse]

In D113220#3122771, @pcc wrote:

I think it ought to be possible to write these non-relaxable relocations from assembly, so that the behavior with -c is equivalent to the behavior with -S and then assembling the output. For example, you should be able to assemble

mov foo@gotpcrel_norelax(%rip), %rax

and the -S output from the compiler should look like that as well.

Won't this cause incompatibility with all other assemblers?

[pcc]

In D113220#3122916, @morehouse wrote:

In D113220#3122771, @pcc wrote:

I think it ought to be possible to write these non-relaxable relocations from assembly, so that the behavior with -c is equivalent to the behavior with -S and then assembling the output. For example, you should be able to assemble

mov foo@gotpcrel_norelax(%rip), %rax

and the -S output from the compiler should look like that as well.

Won't this cause incompatibility with all other assemblers?

We are not required to remain compatible with other assemblers in our assembly output unless -no-integrated-as is specified, but any extensions should be documented here: https://llvm.org/docs/Extensions.html

Indeed, LLVM will use a number of its own extensions by default in its assembly output, e.g. .addrsig.

[morehouse]

• Add @GOTPCREL_NORELAX as an assembly extension.

[pcc]

LGTM

[MaskRay]

GNU ld's behavior and LLD's are not the same, and we may not end up on the same page. However, for courtesy it would be nice to ask [e-mail] whether they may adopt a similar notation.
We can still proceed if they don't do anything, but this makes for a possible common ground in case they have an opinion on the notation.

[kstoimenov]

llvm/lib/Target/X86/X86AsmPrinter.cpp

193–195

Maybe format the rest of the lines to align like they did before your change? Either that or format them to have a single space after the ";".

[kstoimenov]

llvm/lib/Target/X86/X86AsmPrinter.cpp

193–195

Correction: after ":".

llvm/include/llvm/MC/MCExpr.h

enum VariantKind : uint16_t {
     VK_None,
     VK_Invalid,
 
     VK_GOT,
     VK_GOTOFF,
     VK_GOTREL,
     VK_PCREL,
     VK_GOTPCREL,
+    VK_GOTPCREL_NORELAX,
     VK_GOTTPOFF,
     VK_INDNTPOFF,
     VK_NTPOFF,
     VK_GOTNTPOFF,
     VK_PLT,
     VK_TLSGD,
     VK_TLSLD,
     VK_TLSLDM,

llvm/lib/MC/ELFObjectWriter.cpp

bool ELFObjectWriter::shouldRelocateWithSymbol(const MCAssembler &Asm,
 
   // These VariantKind cause the relocation to refer to something other than
   // the symbol itself, like a linker generated table. Since the address of
   // symbol is not relevant, we cannot replace the symbol with the
   // section and patch the difference in the addend.
   case MCSymbolRefExpr::VK_GOT:
   case MCSymbolRefExpr::VK_PLT:
   case MCSymbolRefExpr::VK_GOTPCREL:
+  case MCSymbolRefExpr::VK_GOTPCREL_NORELAX:
   case MCSymbolRefExpr::VK_PPC_GOT_LO:
   case MCSymbolRefExpr::VK_PPC_GOT_HI:
   case MCSymbolRefExpr::VK_PPC_GOT_HA:
     return true;
   }
 
   // An undefined symbol is not in any section, so the relocation has to point
   // to the symbol itself.

llvm/lib/MC/MCExpr.cpp

StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_None: return "<<none>>";
 
   case VK_DTPOFF: return "DTPOFF";
   case VK_DTPREL: return "DTPREL";
   case VK_GOT: return "GOT";
   case VK_GOTOFF: return "GOTOFF";
   case VK_GOTREL: return "GOTREL";
   case VK_PCREL: return "PCREL";
+  case VK_GOTPCREL_NORELAX:
   case VK_GOTPCREL: return "GOTPCREL";
   case VK_GOTTPOFF: return "GOTTPOFF";
   case VK_INDNTPOFF: return "INDNTPOFF";
   case VK_NTPOFF: return "NTPOFF";
   case VK_GOTNTPOFF: return "GOTNTPOFF";
   case VK_PLT: return "PLT";
   case VK_TLSGD: return "TLSGD";
   case VK_TLSLD: return "TLSLD";

llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h

 
 } // end namespace X86;
 
 /// X86II - This namespace holds all of the target specific flags that
 /// instruction info tracks.
 ///
 namespace X86II {
   /// Target Operand Flag enum.
   enum TOF {
     //===------------------------------------------------------------------===//
     // X86 Specific MachineOperand flags.
 
     MO_NO_FLAG,
 
     /// MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a
     /// relocation of:
     ///    SYMBOL_LABEL + [. - PICBASELABEL]
enum TOF {
     /// MO_GOTPCREL - On a symbol operand this indicates that the immediate is
     /// offset to the GOT entry for the symbol name from the current code
     /// location.
     ///
     /// See the X86-64 ELF ABI supplement for more details.
     ///    SYMBOL_LABEL @GOTPCREL
     MO_GOTPCREL,
 
+    /// MO_GOTPCREL_NORELAX - Same as MO_GOTPCREL except that R_X86_64_GOTPCREL
+    /// relocations are guaranteed to be emitted by the integrated assembler
+    /// instead of the relaxable R_X86_64[_REX]_GOTPCRELX relocations.
+    ///
+    /// See the X86-64 ELF ABI supplement for more details.
+    ///    SYMBOL_LABEL @GOTPCREL
+    MO_GOTPCREL_NORELAX,
+
     /// MO_PLT - On a symbol operand this indicates that the immediate is
     /// offset to the PLT entry of symbol name from the current code location.
     ///
     /// See the X86-64 ELF ABI supplement for more details.
     ///    SYMBOL_LABEL @PLT
     MO_PLT,
 
     /// MO_TLSGD - On a symbol operand this indicates that the immediate is
enum TOF {
     MO_ABS8,
 
     /// MO_COFFSTUB - On a symbol operand "FOO", this indicates that the
     /// reference is actually to the ".refptr.FOO" symbol.  This is used for
     /// stub symbols on windows.
     MO_COFFSTUB,
   };
 
   enum : uint64_t {
     //===------------------------------------------------------------------===//
     // Instruction encodings.  These are the standard/most common forms for X86
     // instructions.
     //
 
     // PseudoFrm - This represents an instruction that is a pseudo instruction
     // or one that has not been implemented yet.  It is illegal to code generate
     // it, but tolerated for intermediate implementation stages.
enum : uint64_t {
     NoTrackShift = EVEX_RCShift + 1,
     NOTRACK = 1ULL << NoTrackShift,
 
     // Force VEX encoding
     ExplicitVEXShift = NoTrackShift + 1,
     ExplicitVEXPrefix = 1ULL << ExplicitVEXShift
   };
 
   /// \returns true if the instruction with given opcode is a prefix.
   inline bool isPrefix(uint64_t TSFlags) {
     return (TSFlags & X86II::FormMask) == PrefixByte;
   }
 
   /// \returns true if the instruction with given opcode is a pseudo.
   inline bool isPseudo(uint64_t TSFlags) {
     return (TSFlags & X86II::FormMask) == Pseudo;
   }

llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp

default:
       return ELF::R_X86_64_GOTPCREL;
     case X86::reloc_riprel_4byte_relax:
       return ELF::R_X86_64_GOTPCRELX;
     case X86::reloc_riprel_4byte_relax_rex:
     case X86::reloc_riprel_4byte_movq_load:
       return ELF::R_X86_64_REX_GOTPCRELX;
     }
     llvm_unreachable("unexpected relocation type!");
+  case MCSymbolRefExpr::VK_GOTPCREL_NORELAX:
+    checkIs32(Ctx, Loc, Type);
+    return ELF::R_X86_64_GOTPCREL;
   case MCSymbolRefExpr::VK_X86_PLTOFF:
     checkIs64(Ctx, Loc, Type);
     return ELF::R_X86_64_PLTOFF64;
   }
 }
 
 enum X86_32RelType { RT32_NONE, RT32_32, RT32_16, RT32_8 };
 

llvm/lib/Target/X86/X86.td

: SubtargetFeature<
           "lvi-load-hardening", "UseLVILoadHardening", "true",
           "Insert LFENCE instructions to prevent data speculatively injected "
           "into loads from being used maliciously.">;
 
 def FeatureTaggedGlobals
     : SubtargetFeature<
           "tagged-globals", "AllowTaggedGlobals", "true",
           "Use an instruction sequence for taking the address of a global "
-          "that allows a memory tag in the upper address bits. Currently "
-          "also requires -mrelax-relocations=no to work properly.">;
+          "that allows a memory tag in the upper address bits.">;
 
 //===----------------------------------------------------------------------===//
 // X86 Subtarget Tuning features
 //===----------------------------------------------------------------------===//
 
 def TuningSlowSHLD : SubtargetFeature<"slow-shld", "IsSHLDSlow", "true",
                                        "SHLD instruction is slow">;
 

llvm/lib/Target/X86/X86AsmPrinter.cpp

void X86AsmPrinter::PrintSymbolOperand(const MachineOperand &MO,
   case X86II::MO_TLSLD:     O << "@TLSLD";     break;
   case X86II::MO_TLSLDM:    O << "@TLSLDM";    break;
   case X86II::MO_GOTTPOFF:  O << "@GOTTPOFF";  break;
   case X86II::MO_INDNTPOFF: O << "@INDNTPOFF"; break;
   case X86II::MO_TPOFF:     O << "@TPOFF";     break;
   case X86II::MO_DTPOFF:    O << "@DTPOFF";    break;
   case X86II::MO_NTPOFF:    O << "@NTPOFF";    break;
   case X86II::MO_GOTNTPOFF: O << "@GOTNTPOFF"; break;
+  case X86II::MO_GOTPCREL_NORELAX:
   case X86II::MO_GOTPCREL:  O << "@GOTPCREL";  break;
   case X86II::MO_GOT:       O << "@GOT";       break;
   case X86II::MO_GOTOFF:    O << "@GOTOFF";    break;
   case X86II::MO_PLT:       O << "@PLT";       break;
   case X86II::MO_TLVP:      O << "@TLVP";      break;
   case X86II::MO_TLVP_PIC_BASE:
     O << "@TLVP" << '-';
     MF->getPICBaseSymbol()->print(O, MAI);
     break;
   case X86II::MO_SECREL:    O << "@SECREL32";  break;
   }

llvm/lib/Target/X86/X86FastISel.cpp

if (!Subtarget->isPICStyleRIPRel() ||
         if (TLI.getPointerTy(DL) == MVT::i64) {
           Opc = X86::MOV64rm;
           RC  = &X86::GR64RegClass;
         } else {
           Opc = X86::MOV32rm;
           RC  = &X86::GR32RegClass;
         }
 
-        if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL)
+        if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL ||
+            GVFlags == X86II::MO_GOTPCREL_NORELAX)
           StubAM.Base.Reg = X86::RIP;
 
         LoadReg = createResultReg(RC);
         MachineInstrBuilder LoadMI =
           BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), LoadReg);
         addFullAddress(LoadMI, StubAM);
 
         // Ok, back to normal mode.
bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   } else {
     // Direct call.
     assert(GV && "Not a direct call");
     // See if we need any target-specific flags on the GV operand.
     unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV);
 
     // This will be a direct call, or an indirect call through memory for
     // NonLazyBind calls or dllimport calls.
     bool NeedLoad = OpFlags == X86II::MO_DLLIMPORT ||
                     OpFlags == X86II::MO_GOTPCREL ||
+                    OpFlags == X86II::MO_GOTPCREL_NORELAX ||
                     OpFlags == X86II::MO_COFFSTUB;
     unsigned CallOpc = NeedLoad
                            ? (Is64Bit ? X86::CALL64m : X86::CALL32m)
                            : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
 
     MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CallOpc));
     if (NeedLoad)
       MIB.addReg(Is64Bit ? X86::RIP : 0).addImm(1).addReg(0);

llvm/lib/Target/X86/X86ISelLowering.cpp

if (GV && GV->isAbsoluteSymbolRef())
     return X86ISD::Wrapper;
 
   CodeModel::Model M = getTargetMachine().getCodeModel();
   if (Subtarget.isPICStyleRIPRel() &&
       (M == CodeModel::Small || M == CodeModel::Kernel))
     return X86ISD::WrapperRIP;
 
   // GOTPCREL references must always use RIP.
-  if (OpFlags == X86II::MO_GOTPCREL)
+  if (OpFlags == X86II::MO_GOTPCREL || OpFlags == X86II::MO_GOTPCREL_NORELAX)
     return X86ISD::WrapperRIP;
 
   return X86ISD::Wrapper;
 }
 
 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
 // their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
 // one of the above mentioned nodes. It has to be wrapped because otherwise

llvm/lib/Target/X86/X86InstrInfo.h

 } // namespace X86
 
 /// isGlobalStubReference - Return true if the specified TargetFlag operand is
 /// a reference to a stub for a global, not the global itself.
 inline static bool isGlobalStubReference(unsigned char TargetFlag) {
   switch (TargetFlag) {
   case X86II::MO_DLLIMPORT:               // dllimport stub.
   case X86II::MO_GOTPCREL:                // rip-relative GOT reference.
+  case X86II::MO_GOTPCREL_NORELAX:        // rip-relative GOT reference.
   case X86II::MO_GOT:                     // normal GOT reference.
   case X86II::MO_DARWIN_NONLAZY_PIC_BASE: // Normal $non_lazy_ptr ref.
   case X86II::MO_DARWIN_NONLAZY:          // Normal $non_lazy_ptr ref.
   case X86II::MO_COFFSTUB:                // COFF .refptr stub.
     return true;
   default:
     return false;
   }

llvm/lib/Target/X86/X86InstrInfo.cpp

 X86InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
   using namespace X86II;
   static const std::pair<unsigned, const char *> TargetFlags[] = {
       {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"},
       {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"},
       {MO_GOT, "x86-got"},
       {MO_GOTOFF, "x86-gotoff"},
       {MO_GOTPCREL, "x86-gotpcrel"},
+      {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"},
       {MO_PLT, "x86-plt"},
       {MO_TLSGD, "x86-tlsgd"},
       {MO_TLSLD, "x86-tlsld"},
       {MO_TLSLDM, "x86-tlsldm"},
       {MO_GOTTPOFF, "x86-gottpoff"},
       {MO_INDNTPOFF, "x86-indntpoff"},
       {MO_TPOFF, "x86-tpoff"},
       {MO_DTPOFF, "x86-dtpoff"},

llvm/lib/Target/X86/X86MCInstLower.cpp

case X86II::MO_NTPOFF:
     RefKind = MCSymbolRefExpr::VK_NTPOFF;
     break;
   case X86II::MO_GOTNTPOFF:
     RefKind = MCSymbolRefExpr::VK_GOTNTPOFF;
     break;
   case X86II::MO_GOTPCREL:
     RefKind = MCSymbolRefExpr::VK_GOTPCREL;
     break;
+  case X86II::MO_GOTPCREL_NORELAX:
+    RefKind = MCSymbolRefExpr::VK_GOTPCREL_NORELAX;
+    break;
   case X86II::MO_GOT:
     RefKind = MCSymbolRefExpr::VK_GOT;
     break;
   case X86II::MO_GOTOFF:
     RefKind = MCSymbolRefExpr::VK_GOTOFF;
     break;
   case X86II::MO_PLT:
     RefKind = MCSymbolRefExpr::VK_PLT;

llvm/lib/Target/X86/X86Subtarget.cpp

 
 unsigned char
 X86Subtarget::classifyLocalReference(const GlobalValue *GV) const {
   // Tagged globals have non-zero upper bits, which makes direct references
   // require a 64-bit immediate.  On the small code model this causes relocation
   // errors, so we go through the GOT instead.
   if (AllowTaggedGlobals && TM.getCodeModel() == CodeModel::Small && GV &&
       !isa<Function>(GV))
-    return X86II::MO_GOTPCREL;
+    return X86II::MO_GOTPCREL_NORELAX;
 
   // If we're not PIC, it's not very interesting.
   if (!isPositionIndependent())
     return X86II::MO_NO_FLAG;
 
   if (is64Bit()) {
     // 64-bit ELF PIC local references may use GOTOFF relocations.
     if (isTargetELF()) {
if (isOSWindows())
     return X86II::MO_NO_FLAG;
 
   if (is64Bit()) {
     // ELF supports a large, truly PIC code model with non-PC relative GOT
     // references. Other object file formats do not. Use the no-flag, 64-bit
     // reference for them.
     if (TM.getCodeModel() == CodeModel::Large)
       return isTargetELF() ? X86II::MO_GOT : X86II::MO_NO_FLAG;
+    // Tagged globals have non-zero upper bits, which makes direct references
+    // require a 64-bit immediate. So we can't let the linker relax the
+    // relocation to a 32-bit RIP-relative direct reference.
+    if (AllowTaggedGlobals && GV && !isa<Function>(GV))
+      return X86II::MO_GOTPCREL_NORELAX;
     return X86II::MO_GOTPCREL;
   }
 
   if (isTargetDarwin()) {
     if (!isPositionIndependent())
       return X86II::MO_DARWIN_NONLAZY;
     return X86II::MO_DARWIN_NONLAZY_PIC_BASE;
   }

llvm/test/CodeGen/X86/tagged-globals-pic.ll

 ; RUN: llc --relocation-model=pic < %s | FileCheck %s
+; RUN: llc --relocation-model=pic --relax-elf-relocations --filetype=obj -o - < %s | llvm-objdump -d -r - | FileCheck %s --check-prefix=OBJ
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @global = external global i32
 declare void @func()
 
 define i32* @global_addr() #0 {
   ; CHECK-LABEL: global_addr:
   ; CHECK: movq global@GOTPCREL(%rip), %rax
   ; CHECK: retq
 
+  ; OBJ-LABEL: <global_addr>:
+  ; OBJ: movq (%rip),
+  ; OBJ-NEXT: R_X86_64_GOTPCREL global
+
   ret i32* @global
 }
 
 define i32 @global_load() #0 {
   ; CHECK-LABEL: global_load:
   ; CHECK: movq global@GOTPCREL(%rip), [[REG:%r[0-9a-z]+]]
   ; CHECK: movl ([[REG]]), %eax
   ; CHECK: retq
 
+  ; OBJ-LABEL: <global_load>:
+  ; OBJ: movq (%rip),
+  ; OBJ-NEXT: R_X86_64_GOTPCREL global
+
   %load = load i32, i32* @global
   ret i32 %load
 }
 
 define void @global_store() #0 {
   ; CHECK-LABEL: global_store:
   ; CHECK: movq global@GOTPCREL(%rip), [[REG:%r[0-9a-z]+]]
   ; CHECK: movl $0, ([[REG]])
   ; CHECK: retq
 
+  ; OBJ-LABEL: <global_store>:
+  ; OBJ: movq (%rip),
+  ; OBJ-NEXT: R_X86_64_GOTPCREL global
+
   store i32 0, i32* @global
   ret void
 }
 
 define void ()* @func_addr() #0 {
   ; CHECK-LABEL: func_addr:
   ; CHECK: movq func@GOTPCREL(%rip), %rax
   ; CHECK: retq
 
+  ; OBJ-LABEL: <func_addr>:
+  ; OBJ: movq (%rip),
+  ; OBJ-NEXT: R_X86_64_REX_GOTPCRELX func
+
   ret void ()* @func
 }
 
 ; Jump tables shouldn't go through the GOT.
 define i32 @jump_table(i32 %x) #0 {
   ; CHECK-LABEL: jump_table:
   ; CHECK-NOT: @GOT