[JITLink][ELF/AARCH64] Implement R_AARCH64_LDST*_ABS_LO12_NC relocation types

[sunho]

Summary

Implement R_AARCH64_LDST*_ABS_LO12_NC relocaiton entries by reusing PageOffset21 generic relocation edge. The difference between macho backend is that in ELF, the shift value is explicitly given by relocation type. lld generates the relocation type that matches with instruction bitwidth, so getting the shift value implicitly from instruction bytes should be fine in typical use cases.

Another way to achieve this is create explicit edges like PageOffset21Shift0, PageOffset21Shift1, PageOffset21Shift2, ... and macho backend explicitly lower to that reading instructions bytes, if that's more preferred way, I'll change to that.

[sunho]

Merge testcase

[sgraenitz]

The difference between macho backend is that in ELF, the shift value is explicitly given by relocation type. lld generates the relocation type that matches with instruction bitwidth, so getting the shift value implicitly from instruction bytes should be fine in typical use cases.

Good explanation, thanks. Yes, I'd agree that's the best solution.

Merge testcase

Thanks!

llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp

153

Alignment can not be smaller then one byte right? Can we skip the check in this case?

164

Can we factor this out into a helper function like checkAlignment(Instr, N)? I guess it's sufficient for the error message to say something like "not aligned to N byte boundary".

[sunho]

llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp

153

I realized the I wrote a very confusing error message here. The more clear message would be "R_AARCH64_LDST8_ABS_LO12_NC target is not a LDRB/STRB (imm12) instruction." I'll change to that.

We still need to check here because technically R_AARCH64_LDST8_ABS_LO12_NC relocation needs the address to be not shifted at all, but if instruction is like LDRH, we're going to shift it left by 1 since getPageOffset12Shift(Instr) returns 1.

I belive such a relocation should not be generated if codegen is doing sane thing, but we still need to check just in case.

[sunho]

Change error messages

[sunho]

Rebase

[sunho]

Rebase

[sunho]

Fix MSVC build error

[sgraenitz]

LGTM, thanks for working on this!

[sunho]

Rebase

[lhames]

Very nice. LGTM!

[lhames]

Committed in ab472c9615447c8a036d198df7debd4fd9b2e8cc.

llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp

 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h"
 #include "ELFLinkGraphBuilder.h"
 #include "JITLinkGeneric.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/MathExtras.h"
 
 #define DEBUG_TYPE "jitlink"
 
 using namespace llvm;
 using namespace llvm::jitlink;
 
 namespace llvm {

 
 template <typename ELFT>
 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> {
 private:
   enum ELFAArch64RelocationKind : Edge::Kind {
     ELFCall26 = Edge::FirstRelocation,
     ELFAdrPage21,
     ELFAddAbs12,
+    ELFLdSt8Abs12,
+    ELFLdSt16Abs12,
+    ELFLdSt32Abs12,
+    ELFLdSt64Abs12,
+    ELFLdSt128Abs12,
   };
 
   static Expected<ELFAArch64RelocationKind>
   getRelocationKind(const uint32_t Type) {
     using namespace aarch64;
     switch (Type) {
     case ELF::R_AARCH64_CALL26:
       return ELFCall26;
     case ELF::R_AARCH64_ADR_PREL_PG_HI21:
       return ELFAdrPage21;
     case ELF::R_AARCH64_ADD_ABS_LO12_NC:
       return ELFAddAbs12;
+    case ELF::R_AARCH64_LDST8_ABS_LO12_NC:
+      return ELFLdSt8Abs12;
+    case ELF::R_AARCH64_LDST16_ABS_LO12_NC:
+      return ELFLdSt16Abs12;
+    case ELF::R_AARCH64_LDST32_ABS_LO12_NC:
+      return ELFLdSt32Abs12;
+    case ELF::R_AARCH64_LDST64_ABS_LO12_NC:
+      return ELFLdSt64Abs12;
+    case ELF::R_AARCH64_LDST128_ABS_LO12_NC:
+      return ELFLdSt128Abs12;
     }
 
     return make_error<JITLinkError>("Unsupported aarch64 relocation:" +
                                     formatv("{0:d}", Type));
   }
 
   Error addRelocations() override {
     LLVM_DEBUG(dbgs() << "Processing relocations:\n");
 
     using Base = ELFLinkGraphBuilder<ELFT>;
     using Self = ELFLinkGraphBuilder_aarch64<ELFT>;
     for (const auto &RelSect : Base::Sections)
       if (Error Err = Base::forEachRelocation(RelSect, this,
                                               &Self::addSingleRelocation))
         return Err;
 
     return Error::success();
   }
 
   Error addSingleRelocation(const typename ELFT::Rela &Rel,
                             const typename ELFT::Shdr &FixupSect,
                             Block &BlockToFix) {
+    using support::ulittle32_t;
     using Base = ELFLinkGraphBuilder<ELFT>;
 
     uint32_t SymbolIndex = Rel.getSymbol(false);
     auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
     if (!ObjSymbol)
       return ObjSymbol.takeError();
 
     Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
Error addSingleRelocation(const typename ELFT::Rela &Rel,
     if (!RelocKind)
       return RelocKind.takeError();
 
     int64_t Addend = Rel.r_addend;
     orc::ExecutorAddr FixupAddress =
         orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;
     Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
 
+    // Get a pointer to the fixup content.
+    const void *FixupContent = BlockToFix.getContent().data() +
+                               (FixupAddress - BlockToFix.getAddress());
+
     Edge::Kind Kind = Edge::Invalid;
 
     switch (*RelocKind) {
     case ELFCall26: {
       Kind = aarch64::Branch26;
       break;
     }
     case ELFAdrPage21: {
       Kind = aarch64::Page21;
       break;
     }
     case ELFAddAbs12: {
       Kind = aarch64::PageOffset12;
       break;
     }
+    case ELFLdSt8Abs12: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLoadStoreImm12(Instr) ||
+          aarch64::getPageOffset12Shift(Instr) != 0)
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDST8_ABS_LO12_NC target is not a "
+            "LDRB/STRB (imm12) instruction");
+
+      Kind = aarch64::PageOffset12;
+      break;
+    }
+    case ELFLdSt16Abs12: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLoadStoreImm12(Instr) ||
+          aarch64::getPageOffset12Shift(Instr) != 1)
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDST16_ABS_LO12_NC target is not a "
+            "LDRH/STRH (imm12) instruction");
+
+      Kind = aarch64::PageOffset12;
+      break;
+    }
+    case ELFLdSt32Abs12: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLoadStoreImm12(Instr) ||
+          aarch64::getPageOffset12Shift(Instr) != 2)
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDST32_ABS_LO12_NC target is not a "
+            "LDR/STR (imm12, 32 bit) instruction");
+
+      Kind = aarch64::PageOffset12;
+      break;
+    }
+    case ELFLdSt64Abs12: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLoadStoreImm12(Instr) ||
+          aarch64::getPageOffset12Shift(Instr) != 3)
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDST64_ABS_LO12_NC target is not a "
+            "LDR/STR (imm12, 64 bit) instruction");
+
+      Kind = aarch64::PageOffset12;
+      break;
+    }
+    case ELFLdSt128Abs12: {
+      uint32_t Instr = *(const ulittle32_t *)FixupContent;
+      if (!aarch64::isLoadStoreImm12(Instr) ||
+          aarch64::getPageOffset12Shift(Instr) != 4)
+        return make_error<JITLinkError>(
+            "R_AARCH64_LDST128_ABS_LO12_NC target is not a "
+            "LDR/STR (imm12, 128 bit) instruction");
+
+      Kind = aarch64::PageOffset12;
+      break;
+    }
     };
 
     Edge GE(Kind, Offset, *GraphSymbol, Addend);
     LLVM_DEBUG({
       dbgs() << "    ";
       printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind));
       dbgs() << "\n";
     });
 
     BlockToFix.addEdge(std::move(GE));
     return Error::success();
   }
 
   /// Return the string name of the given ELF aarch64 edge kind.
   const char *getELFAArch64RelocationKindName(Edge::Kind R) {
     switch (R) {
     case ELFCall26:
       return "ELFCall26";
     case ELFAdrPage21:
       return "ELFAdrPage21";
     case ELFAddAbs12:
       return "ELFAddAbs12";
+    case ELFLdSt8Abs12:
+      return "ELFLdSt8Abs12";
+    case ELFLdSt16Abs12:
+      return "ELFLdSt16Abs12";
+    case ELFLdSt32Abs12:
+      return "ELFLdSt32Abs12";
+    case ELFLdSt64Abs12:
+      return "ELFLdSt64Abs12";
+    case ELFLdSt128Abs12:
+      return "ELFLdSt128Abs12";
     default:
       return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
     }
   }
 
 public:
   ELFLinkGraphBuilder_aarch64(StringRef FileName,
                               const object::ELFFile<ELFT> &Obj, const Triple T)

llvm/test/ExecutionEngine/JITLink/AArch64/ELF_aarch64_relocations.s

test_adr_prel:
         .size test_adr_prel, .-test_adr_prel
 
         .globl  test_add_abs_lo12
         .p2align  2
 test_add_abs_lo12:
         add x0, x0, :lo12:named_data
         .size test_add_abs_lo12, .-test_add_abs_lo12
 
+# Check R_AARCH64_LDST*_ABS_LO12_NC relocation of a local symbol
+#
+# The immediate value should be the symbol address right shifted according to its instruction bitwidth.
+#
+# jitlink-check: decode_operand(test_ldrb, 2) = named_data[11:0]
+# jitlink-check: decode_operand(test_ldrsb, 2) = (named_data + 0)[11:0]
+# jitlink-check: decode_operand(test_ldrh, 2) = (named_data + 0)[11:1]
+# jitlink-check: decode_operand(test_ldrsh, 2) = (named_data + 0)[11:1]
+# jitlink-check: decode_operand(test_ldr_32bit, 2) = (named_data + 0)[11:2]
+# jitlink-check: decode_operand(test_ldr_64bit, 2) = (named_data + 0)[11:3]
+# jitlink-check: decode_operand(test_strb, 2) = named_data[11:0]
+# jitlink-check: decode_operand(test_strh, 2) = (named_data + 0)[11:1]
+# jitlink-check: decode_operand(test_str_32bit, 2) = (named_data + 0)[11:2]
+# jitlink-check: decode_operand(test_str_64bit, 2) = (named_data + 0)[11:3]
+
+        .globl  test_ldrb
+test_ldrb:
+        ldrb  w0, [x1, :lo12:named_data]
+        .size test_ldrb, .-test_ldrb
+
+        .globl  test_ldrsb
+test_ldrsb:
+        ldrsb  w0, [x1, :lo12:named_data]
+        .size test_ldrsb, .-test_ldrsb
+
+        .globl  test_ldrh
+test_ldrh:
+        ldrh  w0, [x1, :lo12:named_data]
+        .size test_ldrh, .-test_ldrh
+
+        .globl  test_ldrsh
+test_ldrsh:
+        ldrsh  w0, [x1, :lo12:named_data]
+        .size test_ldrsh, .-test_ldrsh
+
+        .globl  test_ldr_32bit
+test_ldr_32bit:
+        ldr  w0, [x1, :lo12:named_data]
+        .size test_ldr_32bit, .-test_ldr_32bit
+
+        .globl  test_ldr_64bit
+test_ldr_64bit:
+        ldr  x0, [x1, :lo12:named_data]
+        .size test_ldr_64bit, .-test_ldr_64bit
+
+        .globl  test_strb
+test_strb:
+        strb  w0, [x1, :lo12:named_data]
+        .size test_strb, .-test_strb
+
+        .globl  test_strh
+test_strh:
+        strh  w0, [x1, :lo12:named_data]
+        .size test_strh, .-test_strh
+
+        .globl  test_str_32bit
+test_str_32bit:
+        str  w0, [x1, :lo12:named_data]
+        .size test_str_32bit, .-test_str_32bit
+
+        .globl  test_str_64bit
+test_str_64bit:
+        str  x0, [x1, :lo12:named_data]
+        .size test_str_64bit, .-test_str_64bit
+
         .globl  named_data
         .p2align  4
         .type   named_data,@object
 named_data:
         .quad   0x2222222222222222
         .quad   0x3333333333333333
         .size   named_data, .-named_data