Index: lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp =================================================================== --- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -14,6 +14,7 @@ #include "Targets/RuntimeDyldCOFFI386.h" #include "Targets/RuntimeDyldCOFFThumb.h" #include "Targets/RuntimeDyldCOFFX86_64.h" +#include "Targets/RuntimeDyldCOFFAArch64.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Object/ObjectFile.h" @@ -55,6 +56,8 @@ return std::make_unique(MemMgr, Resolver); case Triple::x86_64: return std::make_unique(MemMgr, Resolver); + case Triple::aarch64: + return std::make_unique(MemMgr, Resolver); } } Index: lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h =================================================================== --- /dev/null +++ lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h @@ -0,0 +1,329 @@ +//===-- RuntimeDyldCOFFAArch64.h --- COFF/AArch64 specific code ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// COFF AArch64 support for MC-JIT runtime dynamic linker. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H +#define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H + +#include "../RuntimeDyldCOFF.h" +#include "llvm/BinaryFormat/COFF.h" +#include "llvm/Object/COFF.h" +#include "llvm/Support/Endian.h" + +#define DEBUG_TYPE "dyld" + +using namespace llvm::support::endian; + +namespace llvm { + +static void or32le(void *P, int32_t V) { write32le(P, read32le(P) | V); } + +static void or32AArch64Imm(void *L, uint64_t Imm) { + or32le(L, (Imm & 0xFFF) << 10); +} + +static void write32AArch64Addr(void *L, uint64_t Imm) { + uint32_t ImmLo = (Imm & 0x3) << 29; + uint32_t ImmHi = (Imm & 0x1FFFFC) << 3; + uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3); + write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi); +} + +// Return the bits [Start, End] from Val shifted Start bits. +// For instance, getBits(0xF0, 4, 8) returns 0xF. +static uint64_t getBits(uint64_t Val, int Start, int End) { + uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1; + return (Val >> Start) & Mask; +} + +class RuntimeDyldCOFFAArch64 : public RuntimeDyldCOFF { + +private: + // When a module is loaded we save the SectionID of the unwind + // sections in a table until we receive a request to register all + // unregisteredEH frame sections with the memory manager. + SmallVector UnregisteredEHFrameSections; + SmallVector RegisteredEHFrameSections; + uint64_t ImageBase; + +public: + RuntimeDyldCOFFAArch64(RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver) + : RuntimeDyldCOFF(MM, Resolver), ImageBase(0) {} + + unsigned getStubAlignment() override { return 8; } + + unsigned getMaxStubSize() const override { return 20; } + + std::tuple + generateRelocationStub(unsigned SectionID, StringRef TargetName, + uint64_t Offset, uint64_t RelType, uint64_t Addend, + StubMap &Stubs) { + uintptr_t StubOffset; + SectionEntry &Section = Sections[SectionID]; + + RelocationValueRef OriginalRelValueRef; + OriginalRelValueRef.SectionID = SectionID; + OriginalRelValueRef.Offset = Offset; + OriginalRelValueRef.Addend = Addend; + OriginalRelValueRef.SymbolName = TargetName.data(); + + auto Stub = Stubs.find(OriginalRelValueRef); + if (Stub == Stubs.end()) { + LLVM_DEBUG(dbgs() << " Create a new stub function for " + << TargetName.data() << "\n"); + + StubOffset = Section.getStubOffset(); + Stubs[OriginalRelValueRef] = StubOffset; + createStubFunction(Section.getAddressWithOffset(StubOffset)); + Section.advanceStubOffset(getMaxStubSize()); + } else { + LLVM_DEBUG(dbgs() << " Stub function found for " << TargetName.data() + << "\n"); + StubOffset = Stub->second; + } + + // Resolve original relocation to stub function. + const RelocationEntry RE(SectionID, Offset, RelType, Addend); + resolveRelocation(RE, Section.getLoadAddressWithOffset(StubOffset)); + + // adjust relocation info so resolution writes to the stub function + Addend = 0; + Offset = StubOffset; + RelType = COFF::IMAGE_REL_ARM64_ADDR64; + + return std::make_tuple(Offset, RelType, Addend); + } + + Expected + processRelocationRef(unsigned SectionID, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, + ObjSectionToIDMap &ObjSectionToID, + StubMap &Stubs) override { + + auto Symbol = RelI->getSymbol(); + if (Symbol == Obj.symbol_end()) + report_fatal_error("Unknown symbol in relocation"); + + Expected TargetNameOrErr = Symbol->getName(); + if (!TargetNameOrErr) + return TargetNameOrErr.takeError(); + StringRef TargetName = *TargetNameOrErr; + + auto SectionOrErr = Symbol->getSection(); + if (!SectionOrErr) + return SectionOrErr.takeError(); + auto Section = *SectionOrErr; + + uint64_t RelType = RelI->getType(); + uint64_t Offset = RelI->getOffset(); + + // If there is no section, this must be an external reference. + const bool IsExtern = Section == Obj.section_end(); + + // Determine the Addend used to adjust the relocation value. + uint64_t Addend = 0; + SectionEntry &AddendSection = Sections[SectionID]; + uintptr_t ObjTarget = AddendSection.getObjAddress() + Offset; + uint8_t *Displacement = (uint8_t *)ObjTarget; + + switch (RelType) { + case COFF::IMAGE_REL_ARM64_ADDR32: + case COFF::IMAGE_REL_ARM64_ADDR32NB: + case COFF::IMAGE_REL_ARM64_SECREL: + Addend = readBytesUnaligned(Displacement, 4); + break; + case COFF::IMAGE_REL_ARM64_BRANCH26: { + auto *p = reinterpret_cast(Displacement); + assert(((*p & 0xFC000000) == 0x14000000 || + (*p & 0xFC000000) == 0x94000000) && + "Expected branch instruction."); + + // Get the 26 bit addend encoded in the branch instruction and sign-extend + // to 64 bit. The lower 2 bits are always zeros and are therefore implicit + // (<< 2). + Addend = (*p & 0x03FFFFFF) << 2; + Addend = SignExtend64(Addend, 28); + + if (IsExtern) + std::tie(Offset, RelType, Addend) = generateRelocationStub( + SectionID, TargetName, Offset, RelType, Addend, Stubs); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: { + auto *p = reinterpret_cast(Displacement); + assert((*p & 0x9F000000) == 0x90000000 && "Expected adrp instruction."); + + // Get the 21 bit addend encoded in the adrp instruction and sign-extend + // to 64 bit. The lower 12 bits (4096 byte page) are always zeros and are + // therefore implicit (<< 12). + Addend = ((*p & 0x60000000) >> 29) | ((*p & 0x01FFFFE0) >> 3) << 12; + Addend = SignExtend64(Addend, 33); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: { + // Verify that the relocation points to one of the expected load / store + // or add / sub instructions. + auto *p = reinterpret_cast(Displacement); + assert((((*p & 0x3B000000) == 0x39000000) || + ((*p & 0x11C00000) == 0x11000000) ) && + "Expected load / store or add/sub instruction."); + + // Get the 12 bit addend encoded in the instruction. + Addend = (*p & 0x003FFC00) >> 10; + + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + if (ImplicitShift == 0) { + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) + ImplicitShift = 4; + } + } + // Compensate for implicit shift. + Addend <<= ImplicitShift; + } + default: + break; + } + +#if !defined(NDEBUG) + SmallString<32> RelTypeName; + RelI->getTypeName(RelTypeName); +#endif + LLVM_DEBUG(dbgs() << "\t\tIn Section " << SectionID << " Offset " << Offset + << " RelType: " << RelTypeName << " TargetName: " + << TargetName << " Addend " << Addend << "\n"); + + unsigned TargetSectionID = -1; + if (IsExtern) { + RelocationEntry RE(SectionID, Offset, RelType, Addend); + addRelocationForSymbol(RE, TargetName); + } else { + if (auto TargetSectionIDOrErr = + findOrEmitSection(Obj, *Section, Section->isText(), ObjSectionToID)) { + TargetSectionID = *TargetSectionIDOrErr; + } + else + return TargetSectionIDOrErr.takeError(); + + // This relocation is ignored. + if (RelType != COFF::IMAGE_REL_ARM64_ABSOLUTE) { + uint64_t TargetOffset = getSymbolOffset(*Symbol); + RelocationEntry RE(SectionID, Offset, RelType, TargetOffset + Addend); + addRelocationForSection(RE, TargetSectionID); + } + } + return ++RelI; + } + + void resolveRelocation(const RelocationEntry &RE, uint64_t Value) override + { + const auto Section = Sections[RE.SectionID]; + uint8_t *Target = Section.getAddressWithOffset(RE.Offset); + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); + + switch (RE.RelType) { + default: llvm_unreachable("unsupported relocation type"); + case COFF::IMAGE_REL_ARM64_ADDR64: { + or32le(Target + 12, ((Value + RE.Addend) & 0xFFFF) << 5); + or32le(Target + 8, ((Value + RE.Addend) & 0xFFFF0000) >> 11); + or32le(Target + 4, ((Value + RE.Addend) & 0xFFFF00000000) >> 27); + or32le(Target + 0, ((Value + RE.Addend) & 0xFFFF000000000000) >> 43); + break; + } + case COFF::IMAGE_REL_ARM64_PAGEBASE_REL21: { + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); + int64_t PCRelVal = + ((Value + RE.Addend) & (-4096)) - (FinalAddress & (-4096)); + write32AArch64Addr(Target, PCRelVal >> 12); + break; + } + + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12A: + case COFF::IMAGE_REL_ARM64_PAGEOFFSET_12L: { + Value += RE.Addend; + Value &= 0xFFF; + auto *p = reinterpret_cast(Target); + // Check which instruction we are decoding to obtain the implicit shift + // factor of the instruction and verify alignment. + int ImplicitShift = 0; + if ((*p & 0x3B000000) == 0x39000000) { // << load / store + // For load / store instructions the size is encoded in bits 31:30. + ImplicitShift = ((*p >> 30) & 0x3); + switch (ImplicitShift) { + case 0: + // Check if this a vector op to get the correct shift value. + if ((*p & 0x04800000) == 0x04800000) { + ImplicitShift = 4; + assert(((Value & 0xF) == 0) && + "128-bit LDR/STR not 16-byte aligned."); + } + break; + case 1: + assert(((Value & 0x1) == 0) && "16-bit LDR/STR not 2-byte aligned."); + break; + case 2: + assert(((Value & 0x3) == 0) && "32-bit LDR/STR not 4-byte aligned."); + break; + case 3: + assert(((Value & 0x7) == 0) && "64-bit LDR/STR not 8-byte aligned."); + break; + } + } + // Compensate for implicit shift. + Value >>= ImplicitShift; + assert(isUInt<12>(Value) && "Addend cannot be encoded."); + *p = (*p & 0xFFC003FF) | ((uint32_t)(Value << 10) & 0x003FFC00); + break; + } + case COFF::IMAGE_REL_ARM64_ADDR32NB: { + // The target's 32-bit RVA. + // NOTE: use Section[0].getLoadAddress() as an approximation of ImageBase + uint64_t Result = FinalAddress - + Sections[0].getLoadAddress() + RE.Addend; + writeBytesUnaligned(Result, Target, 4); + break; + } + case COFF::IMAGE_REL_ARM64_BRANCH26: { + uint64_t FinalAddress = Section.getLoadAddressWithOffset(RE.Offset); + int64_t PCRelVal = Value - FinalAddress + RE.Addend; + auto *p = reinterpret_cast(Target); + // Verify that the relocation points to the expected branch instruction. + assert(((*p & 0xFC000000) == 0x14000000 || + (*p & 0xFC000000) == 0x94000000) && + "Expected branch instruction."); + + // Verify addend value. + assert((PCRelVal & 0x3) == 0 && "Branch target is not aligned"); + assert(isInt<28>(PCRelVal) && "Branch target is out of range."); + + // Encode the addend as 26 bit immediate in the branch instruction. + or32le(p, (PCRelVal & 0x0FFFFFFC) >> 2); +// *p = (*p & 0xFC000000) | ((uint32_t)(PCRelVal >> 2) & 0x03FFFFFF); + break; + } + } + } + + + void registerEHFrames() override {} +}; + +} // End namespace llvm + +#endif \ No newline at end of file Index: lib/Target/AArch64/AArch64TargetMachine.cpp =================================================================== --- lib/Target/AArch64/AArch64TargetMachine.cpp +++ lib/Target/AArch64/AArch64TargetMachine.cpp @@ -247,7 +247,10 @@ // The default MCJIT memory managers make no guarantees about where they can // find an executable page; JITed code needs to be able to refer to globals // no matter how far away they are. - if (JIT) + // We should set the CodeModel::Small for Windows ARM64 in JIT mode, + // since with large code model LLVM generating 4 MOV instructions, and + // Windows doesn't support relocating these long branch (4 MOVs). + if (JIT && !TT.isOSWindows()) return CodeModel::Large; return CodeModel::Small; }