[JITLink][AArch64] Lift fixup functions from aarch64.cpp to aarch64.h. (NFC)

[sunho]

Summary

Lift fixup functions from aarch64.cpp to aarch64.h so that they have better chance of getting inlined. Also, adds some comments documenting the purpose of functions.

[sgraenitz]

LGTM!

Just to add some context for other reviewers: We want this code in the header to make sure it's inlined into the backends.

llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h

enum EdgeKind_aarch64 : Edge::Kind {
   NegDelta32,
   NegDelta64,
 };
 
 /// Returns a string name for the given aarch64 edge. For debugging purposes
 /// only
 const char *getEdgeKindName(Edge::Kind K);
 
-bool isLoadStoreImm12(uint32_t Instr);
+// Returns whether the Instr is LD/ST (imm12)
+inline bool isLoadStoreImm12(uint32_t Instr) {
+  constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
+  return (Instr & LoadStoreImm12Mask) == 0x39000000;
+}
+
+// Returns the amount the address operand of LD/ST (imm12)
+// should be shifted right by.
+//
+// The shift value varies by the data size of LD/ST instruction.
+// For instance, LDH instructoin needs the address to be shifted
+// right by 1.
+inline unsigned getPageOffset12Shift(uint32_t Instr) {
+  constexpr uint32_t Vec128Mask = 0x04800000;
+
+  if (isLoadStoreImm12(Instr)) {
+    uint32_t ImplicitShift = Instr >> 30;
+    if (ImplicitShift == 0)
+      if ((Instr & Vec128Mask) == Vec128Mask)
+        ImplicitShift = 4;
+
+    return ImplicitShift;
+  }
+
+  return 0;
+}
 
-unsigned getPageOffset12Shift(uint32_t Instr);
+/// Apply fixup expression for edge to block content.
+inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
+  using namespace support;
 
-Error applyFixup(LinkGraph &G, Block &B, const Edge &E);
+  char *BlockWorkingMem = B.getAlreadyMutableContent().data();
+  char *FixupPtr = BlockWorkingMem + E.getOffset();
+  orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset();
+
+  switch (E.getKind()) {
+  case Branch26: {
+    assert((FixupAddress.getValue() & 0x3) == 0 &&
+           "Branch-inst is not 32-bit aligned");
+
+    int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
+
+    if (static_cast<uint64_t>(Value) & 0x3)
+      return make_error<JITLinkError>("Branch26 target is not 32-bit "
+                                      "aligned");
+
+    if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
+      return makeTargetOutOfRangeError(G, B, E);
+
+    uint32_t RawInstr = *(little32_t *)FixupPtr;
+    assert((RawInstr & 0x7fffffff) == 0x14000000 &&
+           "RawInstr isn't a B or BR immediate instruction");
+    uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2;
+    uint32_t FixedInstr = RawInstr | Imm;
+    *(little32_t *)FixupPtr = FixedInstr;
+    break;
+  }
+  case Pointer32: {
+    uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
+    if (Value > std::numeric_limits<uint32_t>::max())
+      return makeTargetOutOfRangeError(G, B, E);
+    *(ulittle32_t *)FixupPtr = Value;
+    break;
+  }
+  case Pointer64:
+  case Pointer64Anon: {
+    uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
+    *(ulittle64_t *)FixupPtr = Value;
+    break;
+  }
+  case Page21: {
+    assert((E.getKind() != GOTPage21 || E.getAddend() == 0) &&
+           "GOTPAGE21 with non-zero addend");
+    uint64_t TargetPage =
+        (E.getTarget().getAddress().getValue() + E.getAddend()) &
+        ~static_cast<uint64_t>(4096 - 1);
+    uint64_t PCPage =
+        FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1);
+
+    int64_t PageDelta = TargetPage - PCPage;
+    if (!isInt<33>(PageDelta))
+      return makeTargetOutOfRangeError(G, B, E);
+
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    assert((RawInstr & 0xffffffe0) == 0x90000000 &&
+           "RawInstr isn't an ADRP instruction");
+    uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;
+    uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;
+    uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
+  case PageOffset12: {
+    uint64_t TargetOffset =
+        (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff;
+
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    unsigned ImmShift = getPageOffset12Shift(RawInstr);
+
+    if (TargetOffset & ((1 << ImmShift) - 1))
+      return make_error<JITLinkError>("PAGEOFF12 target is not aligned");
+
+    uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
+    uint32_t FixedInstr = RawInstr | EncodedImm;
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
+  case LDRLiteral19: {
+    assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");
+    assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
+    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+    assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
+    int64_t Delta = E.getTarget().getAddress() - FixupAddress;
+    if (Delta & 0x3)
+      return make_error<JITLinkError>("LDR literal target is not 32-bit "
+                                      "aligned");
+    if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
+      return makeTargetOutOfRangeError(G, B, E);
+
+    uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
+    uint32_t FixedInstr = RawInstr | EncodedImm;
+    *(ulittle32_t *)FixupPtr = FixedInstr;
+    break;
+  }
+  case Delta32:
+  case Delta64:
+  case NegDelta32:
+  case NegDelta64: {
+    int64_t Value;
+    if (E.getKind() == Delta32 || E.getKind() == Delta64)
+      Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
+    else
+      Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
+
+    if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
+      if (Value < std::numeric_limits<int32_t>::min() ||
+          Value > std::numeric_limits<int32_t>::max())
+        return makeTargetOutOfRangeError(G, B, E);
+      *(little32_t *)FixupPtr = Value;
+    } else
+      *(little64_t *)FixupPtr = Value;
+    break;
+  }
+  case TLVPage21:
+  case GOTPage21:
+  case TLVPageOffset12:
+  case GOTPageOffset12:
+  case PointerToGOT: {
+    return make_error<JITLinkError>(
+        "In graph " + G.getName() + ", section " + B.getSection().getName() +
+        "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind()));
+  }
+  default:
+    return make_error<JITLinkError>(
+        "In graph " + G.getName() + ", section " + B.getSection().getName() +
+        "unsupported edge kind" + getEdgeKindName(E.getKind()));
+  }
+
+  return Error::success();
+}
 
 /// AArch64 null pointer content.
 extern const uint8_t NullGOTEntryContent[8];
 
 /// AArch64 PLT stub content.
 extern const uint8_t StubContent[8];
 
 /// Global Offset Table Builder.

llvm/lib/ExecutionEngine/JITLink/aarch64.cpp

 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
 
 #define DEBUG_TYPE "jitlink"
 
 namespace llvm {
 namespace jitlink {
 namespace aarch64 {
 
-bool isLoadStoreImm12(uint32_t Instr) {
-  constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
-  return (Instr & LoadStoreImm12Mask) == 0x39000000;
-}
-
-unsigned getPageOffset12Shift(uint32_t Instr) {
-  constexpr uint32_t Vec128Mask = 0x04800000;
-
-  if (isLoadStoreImm12(Instr)) {
-    uint32_t ImplicitShift = Instr >> 30;
-    if (ImplicitShift == 0)
-      if ((Instr & Vec128Mask) == Vec128Mask)
-        ImplicitShift = 4;
-
-    return ImplicitShift;
-  }
-
-  return 0;
-}
-
-Error applyFixup(LinkGraph &G, Block &B, const Edge &E) {
-  using namespace support;
-
-  char *BlockWorkingMem = B.getAlreadyMutableContent().data();
-  char *FixupPtr = BlockWorkingMem + E.getOffset();
-  orc::ExecutorAddr FixupAddress = B.getAddress() + E.getOffset();
-
-  switch (E.getKind()) {
-  case Branch26: {
-    assert((FixupAddress.getValue() & 0x3) == 0 &&
-           "Branch-inst is not 32-bit aligned");
-
-    int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
-
-    if (static_cast<uint64_t>(Value) & 0x3)
-      return make_error<JITLinkError>("Branch26 target is not 32-bit "
-                                      "aligned");
-
-    if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
-      return makeTargetOutOfRangeError(G, B, E);
-
-    uint32_t RawInstr = *(little32_t *)FixupPtr;
-    assert((RawInstr & 0x7fffffff) == 0x14000000 &&
-           "RawInstr isn't a B or BR immediate instruction");
-    uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2;
-    uint32_t FixedInstr = RawInstr | Imm;
-    *(little32_t *)FixupPtr = FixedInstr;
-    break;
-  }
-  case Pointer32: {
-    uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
-    if (Value > std::numeric_limits<uint32_t>::max())
-      return makeTargetOutOfRangeError(G, B, E);
-    *(ulittle32_t *)FixupPtr = Value;
-    break;
-  }
-  case Pointer64:
-  case Pointer64Anon: {
-    uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
-    *(ulittle64_t *)FixupPtr = Value;
-    break;
-  }
-  case Page21: {
-    assert((E.getKind() != GOTPage21 || E.getAddend() == 0) &&
-           "GOTPAGE21 with non-zero addend");
-    uint64_t TargetPage =
-        (E.getTarget().getAddress().getValue() + E.getAddend()) &
-        ~static_cast<uint64_t>(4096 - 1);
-    uint64_t PCPage =
-        FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1);
-
-    int64_t PageDelta = TargetPage - PCPage;
-    if (!isInt<33>(PageDelta))
-      return makeTargetOutOfRangeError(G, B, E);
-
-    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
-    assert((RawInstr & 0xffffffe0) == 0x90000000 &&
-           "RawInstr isn't an ADRP instruction");
-    uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;
-    uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;
-    uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
-    *(ulittle32_t *)FixupPtr = FixedInstr;
-    break;
-  }
-  case PageOffset12: {
-    uint64_t TargetOffset =
-        (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff;
-
-    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
-    unsigned ImmShift = getPageOffset12Shift(RawInstr);
-
-    if (TargetOffset & ((1 << ImmShift) - 1))
-      return make_error<JITLinkError>("PAGEOFF12 target is not aligned");
-
-    uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
-    uint32_t FixedInstr = RawInstr | EncodedImm;
-    *(ulittle32_t *)FixupPtr = FixedInstr;
-    break;
-  }
-  case LDRLiteral19: {
-    assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");
-    assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
-    uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
-    assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
-    int64_t Delta = E.getTarget().getAddress() - FixupAddress;
-    if (Delta & 0x3)
-      return make_error<JITLinkError>("LDR literal target is not 32-bit "
-                                      "aligned");
-    if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
-      return makeTargetOutOfRangeError(G, B, E);
-
-    uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;
-    uint32_t FixedInstr = RawInstr | EncodedImm;
-    *(ulittle32_t *)FixupPtr = FixedInstr;
-    break;
-  }
-  case Delta32:
-  case Delta64:
-  case NegDelta32:
-  case NegDelta64: {
-    int64_t Value;
-    if (E.getKind() == Delta32 || E.getKind() == Delta64)
-      Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
-    else
-      Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
-
-    if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
-      if (Value < std::numeric_limits<int32_t>::min() ||
-          Value > std::numeric_limits<int32_t>::max())
-        return makeTargetOutOfRangeError(G, B, E);
-      *(little32_t *)FixupPtr = Value;
-    } else
-      *(little64_t *)FixupPtr = Value;
-    break;
-  }
-  case TLVPage21:
-  case GOTPage21:
-  case TLVPageOffset12:
-  case GOTPageOffset12:
-  case PointerToGOT: {
-    return make_error<JITLinkError>(
-        "In graph " + G.getName() + ", section " + B.getSection().getName() +
-        "GOT/TLV edge kinds not lowered: " + getEdgeKindName(E.getKind()));
-  }
-  default:
-    return make_error<JITLinkError>(
-        "In graph " + G.getName() + ", section " + B.getSection().getName() +
-        "unsupported edge kind" + getEdgeKindName(E.getKind()));
-  }
-
-  return Error::success();
-}
-
 const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00,
                                         0x00, 0x00, 0x00, 0x00};
 
 const uint8_t StubContent[8] = {
     0x10, 0x00, 0x00, 0x58, // LDR x16, <literal>
     0x00, 0x02, 0x1f, 0xd6  // BR  x16
 };