Index: llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h =================================================================== --- llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h +++ llvm/include/llvm/ExecutionEngine/JITLink/x86_64.h @@ -343,7 +343,9 @@ /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup /// phase will result in an assert/unreachable during the fixup phase. /// - RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable + RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable, + // First platform specific relocation. + FirstPlatformRelocation }; /// Returns a string name for the given x86-64 edge. For debugging purposes Index: llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp =================================================================== --- llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp +++ llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp @@ -18,13 +18,19 @@ namespace llvm { namespace jitlink { +static Triple createTripleWithCOFFFormat(Triple T) { + T.setObjectFormat(Triple::COFF); + return T; +} + COFFLinkGraphBuilder::COFFLinkGraphBuilder( const object::COFFObjectFile &Obj, Triple TT, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : Obj(Obj), - G(std::make_unique( - Obj.getFileName().str(), Triple(std::move(TT)), getPointerSize(Obj), - getEndianness(Obj), std::move(GetEdgeKindName))) { + G(std::make_unique(Obj.getFileName().str(), + createTripleWithCOFFFormat(TT), + getPointerSize(Obj), getEndianness(Obj), + std::move(GetEdgeKindName))) { LLVM_DEBUG({ dbgs() << "Created COFFLinkGraphBuilder for \"" << Obj.getFileName() << "\"\n"; @@ -128,16 +134,6 @@ if (Expected SecNameOrErr = Obj.getSectionName(*Sec)) SectionName = *SecNameOrErr; - bool IsDiscardable = - (*Sec)->Characteristics & - (COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_LNK_INFO); - if (IsDiscardable) { - LLVM_DEBUG(dbgs() << " " << SecIndex << ": \"" << SectionName - << "\" is discardable: " - "No graph section will be created.\n"); - continue; - } - // FIXME: Skip debug info sections LLVM_DEBUG({ @@ -325,6 +321,8 @@ SecIndex <= static_cast(Obj.getNumberOfSections()); SecIndex++) { auto &SymbolSet = SymbolSets[SecIndex]; + if (SymbolSet.empty()) + continue; jitlink::Block *B = getGraphBlock(SecIndex); orc::ExecutorAddrDiff LastOffset = B->getSize(); orc::ExecutorAddrDiff LastDifferentOffset = B->getSize(); @@ -394,6 +392,17 @@ formatv("{0:d}", SymIndex)); Block *B = getGraphBlock(Symbol.getSectionNumber()); + if (!B) { + LLVM_DEBUG({ + dbgs() << " " << SymIndex + << ": Skipping graph symbol since section was not created for " + "COFF symbol \"" + << SymbolName << "\" in section " << Symbol.getSectionNumber() + << "\n"; + }); + return nullptr; + } + if (Symbol.isExternal()) { // This is not a comdat sequence, export the symbol as it is if (!isComdatSection(Section)) { Index: llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp =================================================================== --- llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp +++ llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp @@ -26,6 +26,13 @@ namespace { +enum EdgeKind_coff_x86_64 : Edge::Kind { + PCRel32 = x86_64::FirstPlatformRelocation, + Pointer32NB, + PCRel32ToPLT, + Pointer32NBToPLT, +}; + class COFFJITLinker_x86_64 : public JITLinker { friend class JITLinker; @@ -43,10 +50,10 @@ class COFFLinkGraphBuilder_x86_64 : public COFFLinkGraphBuilder { private: - uint64_t ImageBase = 0; enum COFFX86RelocationKind { COFFAddr32NB, COFFRel32, + COFFRel32_1, }; static Expected @@ -56,6 +63,8 @@ return COFFAddr32NB; case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32: return COFFRel32; + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32_1: + return COFFRel32_1; } return make_error("Unsupported x86_64 relocation:" + @@ -74,17 +83,6 @@ return Error::success(); } - uint64_t getImageBase() { - if (!ImageBase) { - ImageBase = std::numeric_limits::max(); - for (const auto &Block : getGraph().blocks()) { - if (Block->getAddress().getValue()) - ImageBase = std::min(ImageBase, Block->getAddress().getValue()); - } - } - return ImageBase; - } - Error addSingleRelocation(const object::RelocationRef &Rel, const object::SectionRef &FixupSect, Block &BlockToFix) { @@ -121,15 +119,22 @@ Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); Edge::Kind Kind = Edge::Invalid; + const char *FixupPtr = BlockToFix.getContent().data() + Offset; switch (*RelocKind) { case COFFAddr32NB: { - Kind = x86_64::Pointer32; - Offset -= getImageBase(); + Kind = EdgeKind_coff_x86_64::Pointer32NB; + Addend = *reinterpret_cast(FixupPtr); break; } case COFFRel32: { - Kind = x86_64::BranchPCRel32; + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast(FixupPtr); + break; + } + case COFFRel32_1: { + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast(FixupPtr) - 1; break; } }; @@ -137,42 +142,263 @@ Edge GE(Kind, Offset, *GraphSymbol, Addend); LLVM_DEBUG({ dbgs() << " "; - printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind)); + printEdge(dbgs(), BlockToFix, GE, getCOFFX86RelocationKindName(Kind)); dbgs() << "\n"; }); BlockToFix.addEdge(std::move(GE)); + return Error::success(); } - /// Return the string name of the given COFF x86_64 edge kind. - const char *getCOFFX86RelocationKindName(COFFX86RelocationKind R) { - switch (R) { - case COFFAddr32NB: - return "COFFAddr32NB"; - case COFFRel32: - return "COFFRel32"; +public: + COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj, const Triple T) + : COFFLinkGraphBuilder(Obj, std::move(T), getCOFFX86RelocationKindName) {} +}; + +// Create PLT stubs for every potential out-of-reach target. +// This will invalidly generate stubs for lea/cmp/mov instruction targets. +// Invalid stubs will be cleaned up by upcoming PLT stubs optimization pass +// where it removes PLT stub back to original target when target is not +// out-of-scope from the callsite. In COFF, real external symbols exported +// by dllexport can only be functions. So, as long as referenced static symbols +// are correctly materialized within the same jitdylib, this should work out. +class PLTTableManager_COFF_x86_64 + : public TableManager { +public: + PLTTableManager_COFF_x86_64(x86_64::GOTTableManager &GOT) : GOT(GOT) {} + + static StringRef getSectionName() { return "$__STUBS"; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + if ((E.getKind() != EdgeKind_coff_x86_64::PCRel32 && + E.getKind() != EdgeKind_coff_x86_64::Pointer32NB) || + E.getTarget().isDefined()) { + return false; + } + if (E.getKind() == EdgeKind_coff_x86_64::PCRel32) { + E.setKind(EdgeKind_coff_x86_64::PCRel32ToPLT); + } else if (E.getKind() == EdgeKind_coff_x86_64::Pointer32NB) { + E.setKind(EdgeKind_coff_x86_64::Pointer32NBToPLT); } + E.setTarget(getEntryForTarget(G, E.getTarget())); + return true; + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + return x86_64::createAnonymousPointerJumpStub( + G, getStubsSection(G), GOT.getEntryForTarget(G, Target)); } public: - COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj, const Triple T) - : COFFLinkGraphBuilder(Obj, std::move(T), x86_64::getEdgeKindName) {} + Section &getStubsSection(LinkGraph &G) { + if (!PLTSection) + PLTSection = + &G.createSection(getSectionName(), MemProt::Read | MemProt::Exec); + return *PLTSection; + } + + x86_64::GOTTableManager &GOT; + Section *PLTSection = nullptr; +}; + +class COFFLinkGraphLowering_x86_64 { +public: + // Optimize out PLT stub back to original target if target is within the + // reach. + Error optimizePLTStubs(LinkGraph &G, JITLinkContext &Ctx) { + // In order to guarantee the unique address of external symbol, (whether + // it would be stub or original) edges are optimized if and only + // if every other edge pointing to the same PLT stub can be optimized. + // + // Hence, we collect the edges and related information first and iterate + // through the PLT stub list to optimize edges if possible. + // + // Uniqueness is needed in order to keep unwind/exception table valid. + std::map> + PLTEntryEdgeMap; // Collected edges per PLT block symbol. + DenseMap + OriginalSymbolMap; // PLT Block Sybmol to original symbol map + DenseSet CantOptimize; // Symbols that can't be optimized as at + // least one edge is out-of-reach. + DenseSet ShouldOptimize; // Symbols that *must* be optimized + // because it's data instruction. + + // Collect the edges by PLT stub symbol. + for (auto *B : G.blocks()) { + for (auto &E : B->edges()) { + // Skip unrelated edges. + if (E.getKind() != EdgeKind_coff_x86_64::Pointer32NBToPLT && + E.getKind() != EdgeKind_coff_x86_64::PCRel32ToPLT) + continue; + + const uint8_t *FixupPtr = + reinterpret_cast(B->getContent().data()) + + E.getOffset(); + + auto &PLTEntryBlock = E.getTarget().getBlock(); + assert(PLTEntryBlock.edges_size() == 1 && + "PLT entry should only have one outgoing edge"); + auto &GOTEntry = PLTEntryBlock.edges().begin()->getTarget(); + auto &GOTEntryBlock = GOTEntry.getBlock(); + assert(GOTEntryBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + auto &OrigTarget = GOTEntryBlock.edges().begin()->getTarget(); + orc::ExecutorAddr TargetAddr = OrigTarget.getAddress(); + orc::ExecutorAddr EdgeAddr = B->getFixupAddress(E); + + OriginalSymbolMap[&E.getTarget()] = &OrigTarget; + PLTEntryEdgeMap[&E.getTarget()].push_back(&E); + + if (E.getKind() == EdgeKind_coff_x86_64::PCRel32ToPLT) { + // 0xe8 and 0xe9 are both invalid RIP relative addressing ModR/M + // Hence, we're allowed to check if instruction is jmp or call like + // this. + bool IsCallOrJmp = FixupPtr[-1] == 0xe8 || FixupPtr[-1] == 0xe9; + + // If instruction is not call or jmp, it's invalid to use PLT stub. + // This symbol is supposed to come from the same jitdylib and should + // be within the reach. If we can't optimize this stub entry because + // original symbol is not within the reach, the user is likely to be + // not providing the proper static library file. (i.e. not providing + // msvcrt.lib separately for jitlink, but using process symbol) We + // should abort and report an error in this case. + if (!IsCallOrJmp) + ShouldOptimize.insert(&E.getTarget()); + + int64_t Value = TargetAddr - (EdgeAddr + 4) + E.getAddend(); + if (!isInt<32>(Value)) + CantOptimize.insert(&E.getTarget()); + } else if (E.getKind() == EdgeKind_coff_x86_64::Pointer32NBToPLT) { + auto ImageBase = getImageBaseAddress(Ctx); + if (!ImageBase) + return ImageBase.takeError(); + int64_t Value = TargetAddr.getValue() + E.getAddend() - *ImageBase; + if (!isUInt<32>(Value)) + CantOptimize.insert(&E.getTarget()); + } + } + } + + // Iterate through PLT stub symbols and optimize the edges. + for (auto &KeyValue : PLTEntryEdgeMap) { + Symbol *PLTEntry = KeyValue.first; + std::vector &Edges = KeyValue.second; + Symbol *OrigSymbol = OriginalSymbolMap[PLTEntry]; + if (CantOptimize.count(PLTEntry) && ShouldOptimize.count(PLTEntry)) + return make_error( + "PLT stub can't be used for symbol (name: " + + OrigSymbol->getName() + ")"); + if (CantOptimize.count(PLTEntry)) + continue; + + for (auto *E : Edges) { + E->setTarget(*OrigSymbol); + if (E->getKind() == EdgeKind_coff_x86_64::PCRel32ToPLT) { + E->setKind(EdgeKind_coff_x86_64::PCRel32); + } else if (E->getKind() == EdgeKind_coff_x86_64::Pointer32NBToPLT) { + E->setKind(EdgeKind_coff_x86_64::Pointer32NB); + } + } + } + + return Error::success(); + } + + // Lowers COFF x86_64 specific edges to generic x86_64 edges. + Error lowerCOFFRelocationEdges(LinkGraph &G, JITLinkContext &Ctx) { + for (auto *B : G.blocks()) { + for (auto &E : B->edges()) { + switch (E.getKind()) { + case EdgeKind_coff_x86_64::PCRel32: + case EdgeKind_coff_x86_64::PCRel32ToPLT: { + E.setAddend(E.getAddend() - 4); + E.setKind(x86_64::Delta32); + break; + } + case EdgeKind_coff_x86_64::Pointer32NB: + case EdgeKind_coff_x86_64::Pointer32NBToPLT: { + auto ImageBase = getImageBaseAddress(Ctx); + if (!ImageBase) + return ImageBase.takeError(); + E.setAddend(E.getAddend() - *ImageBase); + E.setKind(x86_64::Pointer32); + break; + } + } + } + } + return Error::success(); + } + +private: + Expected getImageBaseAddress(JITLinkContext &Ctx) { + if (this->ImageBase) + return this->ImageBase; + JITLinkContext::LookupMap Symbols; + Symbols["__ImageBase"] = SymbolLookupFlags::RequiredSymbol; + JITTargetAddress ImageBase; + Error Err = Error::success(); + Ctx.lookup(Symbols, + createLookupContinuation([&](Expected LR) { + ErrorAsOutParameter EAO(&Err); + if (!LR) { + Err = LR.takeError(); + return; + } + auto &ImageBaseSymbol = LR->begin()->second; + ImageBase = ImageBaseSymbol.getAddress(); + })); + if (Err) + return std::move(Err); + this->ImageBase = ImageBase; + return ImageBase; + } + JITTargetAddress ImageBase = 0; }; Error buildTables_COFF_x86_64(LinkGraph &G) { LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); x86_64::GOTTableManager GOT; - x86_64::PLTTableManager PLT(GOT); + PLTTableManager_COFF_x86_64 PLT(GOT); visitExistingEdges(G, GOT, PLT); return Error::success(); } + +Error optimizeAndLowerEdges_COFF_x86_64(LinkGraph &G, JITLinkContext *Ctx) { + LLVM_DEBUG(dbgs() << "Optimizing and lowering COFF x86_64 edges:\n"); + COFFLinkGraphLowering_x86_64 GraphLowering; + + if (auto Err = GraphLowering.optimizePLTStubs(G, *Ctx)) + return Err; + + if (auto Err = GraphLowering.lowerCOFFRelocationEdges(G, *Ctx)) + return Err; + + return Error::success(); +} } // namespace namespace llvm { namespace jitlink { +/// Return the string name of the given COFF x86_64 edge kind. +const char *getCOFFX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Pointer32NB: + return "Pointer32NB"; + case PCRel32: + return "PCRel32"; + case PCRel32ToPLT: + return "PCRel32ToPLT"; + case Pointer32NBToPLT: + return "Pointer32NBToPLT"; + default: + return x86_64::getEdgeKindName(R); + } +} + Expected> createLinkGraphFromCOFFObject_x86_64(MemoryBufferRef ObjectBuffer) { LLVM_DEBUG({ @@ -202,8 +428,11 @@ // Add an in-place GOT/Stubs/TLSInfoEntry build pass. Config.PostPrunePasses.push_back(buildTables_COFF_x86_64); - // Add GOT/Stubs optimizer pass. - Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses); + // Add Stubs optimizer / COFF edge lowering passes. + JITLinkContext *CtxPtr = Ctx.get(); + Config.PreFixupPasses.push_back([CtxPtr](LinkGraph &G) { + return optimizeAndLowerEdges_COFF_x86_64(G, CtxPtr); + }); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) Index: llvm/test/ExecutionEngine/JITLink/X86/COFF_addr32nb_reloc.test =================================================================== --- /dev/null +++ llvm/test/ExecutionEngine/JITLink/X86/COFF_addr32nb_reloc.test @@ -0,0 +1,85 @@ +# RUN: yaml2obj %s -o %t +# RUN: llvm-jitlink -noexec -abs __ImageBase=0xfff00000 \ +# RUN: -slab-allocate 100Kb -slab-address 0xfff00000 -slab-page-size 4096 \ +# RUN: -check %s %t +# +# Check IMAGE_REL_AMD64_ADDR32NB relocation properly sets the delta of target +# from imagebase. +# +# jitlink-check: *{4}(pdata) = func - __ImageBase +--- !COFF +header: + Machine: IMAGE_FILE_MACHINE_AMD64 + Characteristics: [ ] +sections: + - Name: .text + Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] + Alignment: 16 + SectionData: '0000000000000000' + - Name: .func + Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] + Alignment: 16 + SectionData: '0000000000000000' + - Name: .pdata + Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] + Alignment: 4 + SectionData: '0000000000000000' + Relocations: + - VirtualAddress: 0 + SymbolTableIndex: 7 + Type: IMAGE_REL_AMD64_ADDR32NB +symbols: + - Name: .text + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 8 + NumberOfRelocations: 0 + CheckSum: 0 + NumberOfLinenumbers: 0 + Number: 1 + - Name: .func + Value: 0 + SectionNumber: 2 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 8 + NumberOfRelocations: 0 + CheckSum: 0 + NumberOfLinenumbers: 0 + Number: 2 + - Name: .pdata + Value: 0 + SectionNumber: 3 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_NULL + StorageClass: IMAGE_SYM_CLASS_STATIC + SectionDefinition: + Length: 8 + NumberOfRelocations: 1 + CheckSum: 0 + NumberOfLinenumbers: 0 + Number: 3 + - Name: main + Value: 0 + SectionNumber: 1 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL + - Name: func + Value: 0 + SectionNumber: 2 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL + - Name: pdata + Value: 0 + SectionNumber: 3 + SimpleType: IMAGE_SYM_TYPE_NULL + ComplexType: IMAGE_SYM_DTYPE_FUNCTION + StorageClass: IMAGE_SYM_CLASS_EXTERNAL Index: llvm/test/ExecutionEngine/JITLink/X86/COFF_basic.s =================================================================== --- llvm/test/ExecutionEngine/JITLink/X86/COFF_basic.s +++ /dev/null @@ -1,29 +0,0 @@ -# RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc %s -o %t -# RUN: llvm-jitlink -noexec %t -# -# Check a basic COFF object file loads successfully. - - .text - .def @feat.00; - .scl 3; - .type 0; - .endef - .globl @feat.00 -.set @feat.00, 0 - .file "main.c" - .def main; - .scl 2; - .type 32; - .endef - .globl main - .p2align 4, 0x90 -main: - .seh_proc main - pushq %rax - .seh_stackalloc 8 - .seh_endprologue - movl $0, 4(%rsp) - xorl %eax, %eax - popq %rcx - retq - .seh_endproc Index: llvm/test/ExecutionEngine/JITLink/X86/COFF_external_var.s =================================================================== --- llvm/test/ExecutionEngine/JITLink/X86/COFF_external_var.s +++ llvm/test/ExecutionEngine/JITLink/X86/COFF_external_var.s @@ -1,11 +1,9 @@ -# REQUIRES: asserts # RUN: llvm-mc -filetype=obj -triple=x86_64-windows-msvc %s -o %t -# RUN: llvm-jitlink -abs var=0xcafef00d --debug-only=jitlink -noexec %t 2>&1 | FileCheck %s +# RUN: not llvm-jitlink -slab-allocate 100Kb -slab-address 0xfff00000 -slab-page-size 4096 \ +# RUN: -abs var=0x7fff00000000 -noexec %t # -# Check an external symbol to a variable is created. +# Check data access to a external variable out of reach causes an error. # -# CHECK: Creating graph symbols... -# CHECK: 7: Creating external graph symbol for COFF symbol "var" in (external) (index: 0) .text Index: llvm/test/ExecutionEngine/JITLink/X86/COFF_label.test =================================================================== --- llvm/test/ExecutionEngine/JITLink/X86/COFF_label.test +++ llvm/test/ExecutionEngine/JITLink/X86/COFF_label.test @@ -19,7 +19,7 @@ Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] Alignment: 16 SectionData: 4883EC28488D542438E8000000004883C428C34883EC28488D54243033C9E8000000004883C428C3 - - Name: '.text$mn' + - Name: '.func' Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ] Alignment: 16 SectionData: 4883EC28488D542438E8000000004883C428C34883EC28488D54243033C9E8000000004883C428C3