diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -82,3 +82,7 @@ if (UNIX) set(ALL_ORC_SUPPORTED_ARCH ${X86_64} ${ARM64} ${ARM32}) endif() + +if (WIN32) + set(ALL_ORC_SUPPORTED_ARCH ${X86_64}) +endif() diff --git a/compiler-rt/lib/orc/CMakeLists.txt b/compiler-rt/lib/orc/CMakeLists.txt --- a/compiler-rt/lib/orc/CMakeLists.txt +++ b/compiler-rt/lib/orc/CMakeLists.txt @@ -1,46 +1,59 @@ # Build for all components of the ORC runtime support library. -# ORC runtime library implementation files. -set(ORC_SOURCES +# ORC runtime library common implementation files. +set(ORC_COMMON_SOURCES debug.cpp extensible_rtti.cpp log_error_to_stderr.cpp - macho_ehframe_registration.cpp - macho_platform.cpp - elfnix_platform.cpp run_program_wrapper.cpp dlfcn_wrapper.cpp ) +# ORC runtime library implementation files for all ORC architectures.s +set(ALL_ORC_SOURCES + ${ORC_COMMON_SOURCES} + coff_platform.cpp + elfnix_platform.cpp + macho_ehframe_registration.cpp + macho_platform.cpp + ) + # Implementation files for all ORC architectures. set(ALL_ORC_ASM_SOURCES macho_tlv.x86-64.S macho_tlv.arm64.S elfnix_tls.x86-64.S -) + elfnix_tls.aarch64.S + ) -set(ORC_IMPL_HEADERS -# Implementation headers will go here. +# Common implementation headers will go here. +set(ORC_COMMON_IMPL_HEADERS adt.h - c_api.h common.h compiler.h endianness.h error.h executor_address.h extensible_rtti.h - macho_platform.h simple_packed_serialization.h stl_extras.h wrapper_function_utils.h -) + ) + +# Implementation headers for all ORC architectures. +set(ALL_ORC_IMPL_HEADERS + ${ORC_COMMON_IMPL_HEADERS} + macho_platform.h + coff_platform.h + elfnix_platform.h + ) # Create list of all source files for # consumption by tests. set(ORC_ALL_SOURCE_FILES - ${ORC_SOURCES} + ${ALL_ORC_SOURCES} ${ALL_ORC_ASM_SOURCES} - ${ORC_IMPL_HEADERS} + ${ALL_ORC_IMPL_HEADERS} ) list(REMOVE_DUPLICATES ORC_ALL_SOURCE_FILES) @@ -74,6 +87,17 @@ macho_tlv.x86-64.S macho_tlv.arm64.S ) + + set(ORC_IMPL_HEADERS + ${ORC_COMMON_IMPL_HEADERS} + macho_platform.h + ) + + set(ORC_SOURCES + ${ORC_COMMON_SOURCES} + macho_ehframe_registration.cpp + macho_platform.cpp + ) add_compiler_rt_object_libraries(RTOrc OS ${ORC_SUPPORTED_OS} @@ -93,10 +117,36 @@ LINK_LIBS ${ORC_LINK_LIBS} PARENT_TARGET orc) else() # not Apple - add_asm_sources(ORC_ASM_SOURCES - elfnix_tls.x86-64.S - elfnix_tls.aarch64.S - ) + if (WIN32) + set(ORC_BUILD_TYPE STATIC) + + set(ORC_IMPL_HEADERS + ${ORC_COMMON_IMPL_HEADERS} + coff_platform.h + ) + + set(ORC_SOURCES + ${ORC_COMMON_SOURCES} + coff_platform.cpp + ) + else() + set(ORC_BUILD_TYPE STATIC) + + set(ORC_IMPL_HEADERS + ${ORC_COMMON_IMPL_HEADERS} + elfnix_platform.h + ) + + set(ORC_SOURCES + ${ORC_COMMON_SOURCES} + elfnix_platform.cpp + ) + + add_asm_sources(ORC_ASM_SOURCES + elfnix_tls.x86-64.S + elfnix_tls.aarch64.S + ) + endif() foreach(arch ${ORC_SUPPORTED_ARCH}) if(NOT CAN_TARGET_${arch}) @@ -112,13 +162,13 @@ # Common ORC archive for instrumented binaries. add_compiler_rt_runtime(clang_rt.orc - STATIC - ARCHS ${arch} - CFLAGS ${ORC_CFLAGS} - LINK_FLAGS ${ORC_LINK_FLAGS} - LINK_LIBS ${ORC_LINK_LIBS} - OBJECT_LIBS ${ORC_COMMON_RUNTIME_OBJECT_LIBS} RTOrc - PARENT_TARGET orc) + ${ORC_BUILD_TYPE} + ARCHS ${arch} + CFLAGS ${ORC_CFLAGS} + LINK_FLAGS ${ORC_LINK_FLAGS} + LINK_LIBS ${ORC_LINK_LIBS} + OBJECT_LIBS ${ORC_COMMON_RUNTIME_OBJECT_LIBS} RTOrc + PARENT_TARGET orc) endforeach() endif() # not Apple diff --git a/compiler-rt/lib/orc/coff_platform.h b/compiler-rt/lib/orc/coff_platform.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/orc/coff_platform.h @@ -0,0 +1,39 @@ +//===- coff_platform.h -------------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC Runtime support for dynamic loading features on COFF-based platforms. +// +//===----------------------------------------------------------------------===// + +#ifndef ORC_RT_COFF_PLATFORM_H +#define ORC_RT_COFF_PLATFORM_H + +#include "common.h" +#include "executor_address.h" + +// dlfcn functions. +ORC_RT_INTERFACE const char *__orc_rt_coff_jit_dlerror(); +ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlopen(const char *path, int mode); +ORC_RT_INTERFACE int __orc_rt_coff_jit_dlclose(void *header); +ORC_RT_INTERFACE void *__orc_rt_coff_jit_dlsym(void *header, + const char *symbol); + +namespace __orc_rt { +namespace coff { + +enum dlopen_mode : int { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 +}; + +} // end namespace coff +} // end namespace __orc_rt + +#endif \ No newline at end of file diff --git a/compiler-rt/lib/orc/coff_platform.cpp b/compiler-rt/lib/orc/coff_platform.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/orc/coff_platform.cpp @@ -0,0 +1,724 @@ +//===- coff_platform.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains code required to load the rest of the COFF runtime. +// +//===----------------------------------------------------------------------===// + +#define NOMINMAX +#include + +#include "coff_platform.h" + +#include "debug.h" +#include "error.h" +#include "wrapper_function_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +#define DEBUG_TYPE "coff_platform" + +using namespace __orc_rt; + +namespace __orc_rt { + +using COFFJITDylibDepInfo = std::vector; +using COFFJITDylibDepInfoMap = + std::unordered_map; + +using SPSCOFFObjectSectionsMap = + SPSSequence>; + +using SPSCOFFJITDylibDepInfo = SPSSequence; + +using SPSCOFFJITDylibDepInfoMap = + SPSSequence>; + +} // namespace __orc_rt + +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_symbol_lookup_tag) +ORC_RT_JIT_DISPATCH_TAG(__orc_rt_coff_push_initializers_tag) + +namespace { +class COFFPlatformRuntimeState { +private: + // Ctor/dtor section. + // Manage lists of *tor functions sorted by the last character of subsection + // name. + struct XtorSection { + void Register(char SubsectionChar, span Xtors) { + Subsections[SubsectionChar - 'A'].push_back(Xtors); + SubsectionsNew[SubsectionChar - 'A'].push_back(Xtors); + } + + void RegisterNoRun(char SubsectionChar, span Xtors) { + Subsections[SubsectionChar - 'A'].push_back(Xtors); + } + + void Reset() { SubsectionsNew = Subsections; } + + void RunAllNewAndFlush(); + + private: + std::array>, 26> Subsections; + std::array>, 26> SubsectionsNew; + }; + + struct JITDylibState { + std::string Name; + void *Header = nullptr; + size_t LinkedAgainstRefCount = 0; + size_t DlRefCount = 0; + std::vector Deps; + XtorSection CInitSection; // XIA~XIZ + XtorSection CXXInitSection; // XCA~XCZ + XtorSection CPreTermSection; // XPA~XPZ + XtorSection CTermSection; // XTA~XTZ + + bool referenced() const { + return LinkedAgainstRefCount != 0 || DlRefCount != 0; + } + }; + +public: + static void initialize(); + static COFFPlatformRuntimeState &get(); + static void destroy(); + + COFFPlatformRuntimeState() = default; + + // Delete copy and move constructors. + COFFPlatformRuntimeState(const COFFPlatformRuntimeState &) = delete; + COFFPlatformRuntimeState & + operator=(const COFFPlatformRuntimeState &) = delete; + COFFPlatformRuntimeState(COFFPlatformRuntimeState &&) = delete; + COFFPlatformRuntimeState &operator=(COFFPlatformRuntimeState &&) = delete; + + const char *dlerror(); + void *dlopen(std::string_view Name, int Mode); + int dlclose(void *Header); + void *dlsym(void *Header, std::string_view Symbol); + + Error registerJITDylib(std::string Name, void *Header); + Error deregisterJITDylib(void *Header); + + Error registerObjectSections( + ExecutorAddr HeaderAddr, + std::vector> Secs, + bool RunInitializers); + Error deregisterObjectSections( + ExecutorAddr HeaderAddr, + std::vector> Secs); + + void *findJITDylibBaseByPC(uint64_t PC); + +private: + Error registerBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); + Error deregisterBlockRange(ExecutorAddr HeaderAddr, ExecutorAddrRange Range); + + Error registerSEHFrames(ExecutorAddr HeaderAddr, + ExecutorAddrRange SEHFrameRange); + Error deregisterSEHFrames(ExecutorAddr HeaderAddr, + ExecutorAddrRange SEHFrameRange); + + Expected dlopenImpl(std::string_view Path, int Mode); + Error dlopenFull(JITDylibState &JDS); + Error dlopenInitialize(JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo); + + Error dlcloseImpl(void *DSOHandle); + Error dlcloseDeinitialize(JITDylibState &JDS); + + JITDylibState *getJITDylibStateByHeader(void *DSOHandle); + JITDylibState *getJITDylibStateByName(std::string_view Path); + Expected lookupSymbolInJITDylib(void *DSOHandle, + std::string_view Symbol); + + static COFFPlatformRuntimeState *CPS; + + std::recursive_mutex JDStatesMutex; + std::map JDStates; + struct BlockRange { + void *Header; + size_t Size; + }; + std::map BlockRanges; + std::unordered_map JDNameToHeader; + std::string DLFcnError; +}; + +} // namespace + +COFFPlatformRuntimeState *COFFPlatformRuntimeState::CPS = nullptr; + +COFFPlatformRuntimeState::JITDylibState * +COFFPlatformRuntimeState::getJITDylibStateByHeader(void *Header) { + auto I = JDStates.find(Header); + if (I == JDStates.end()) + return nullptr; + return &I->second; +} + +COFFPlatformRuntimeState::JITDylibState * +COFFPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { + // FIXME: Avoid creating string copy here. + auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); + if (I == JDNameToHeader.end()) + return nullptr; + void *H = I->second; + auto J = JDStates.find(H); + assert(J != JDStates.end() && + "JITDylib has name map entry but no header map entry"); + return &J->second; +} + +Error COFFPlatformRuntimeState::registerJITDylib(std::string Name, + void *Header) { + ORC_RT_DEBUG({ + printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); + }); + std::lock_guard Lock(JDStatesMutex); + if (JDStates.count(Header)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (name = " << Name << ")"; + return make_error(ErrStream.str()); + } + if (JDNameToHeader.count(Name)) { + std::ostringstream ErrStream; + ErrStream << "Duplicate JITDylib registration for header " << Header + << " (header = " << Header << ")"; + return make_error(ErrStream.str()); + } + + auto &JDS = JDStates[Header]; + JDS.Name = std::move(Name); + JDS.Header = Header; + JDNameToHeader[JDS.Name] = Header; + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterJITDylib(void *Header) { + std::lock_guard Lock(JDStatesMutex); + auto I = JDStates.find(Header); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to deregister unrecognized header " << Header; + return make_error(ErrStream.str()); + } + + // Remove std::string construction once we can use C++20. + auto J = JDNameToHeader.find( + std::string(I->second.Name.data(), I->second.Name.size())); + assert(J != JDNameToHeader.end() && + "Missing JDNameToHeader entry for JITDylib"); + + ORC_RT_DEBUG({ + printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), + Header); + }); + + JDNameToHeader.erase(J); + JDStates.erase(I); + return Error::success(); +} + +void COFFPlatformRuntimeState::XtorSection::RunAllNewAndFlush() { + for (auto &Subsection : SubsectionsNew) { + for (auto &XtorGroup : Subsection) + for (auto &Xtor : XtorGroup) + if (Xtor) + Xtor(); + Subsection.clear(); + } +} + +const char *COFFPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } + +void *COFFPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { + ORC_RT_DEBUG({ + std::string S(Path.data(), Path.size()); + printdbg("COFFPlatform::dlopen(\"%s\")\n", S.c_str()); + }); + std::lock_guard Lock(JDStatesMutex); + if (auto H = dlopenImpl(Path, Mode)) + return *H; + else { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(H.takeError()); + return nullptr; + } +} + +int COFFPlatformRuntimeState::dlclose(void *DSOHandle) { + ORC_RT_DEBUG({ + auto *JDS = getJITDylibStateByHeader(DSOHandle); + std::string DylibName; + if (JDS) { + std::string S; + printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); + } else + printdbg("COFFPlatform::dlclose(%p) (%s)\n", DSOHandle, "invalid handle"); + }); + std::lock_guard Lock(JDStatesMutex); + if (auto Err = dlcloseImpl(DSOHandle)) { + // FIXME: Make dlerror thread safe. + DLFcnError = toString(std::move(Err)); + return -1; + } + return 0; +} + +void *COFFPlatformRuntimeState::dlsym(void *Header, std::string_view Symbol) { + auto Addr = lookupSymbolInJITDylib(Header, Symbol); + if (!Addr) { + return 0; + } + + return Addr->toPtr(); +} + +Expected COFFPlatformRuntimeState::dlopenImpl(std::string_view Path, + int Mode) { + // Try to find JITDylib state by name. + auto *JDS = getJITDylibStateByName(Path); + + if (!JDS) + return make_error("No registered JTIDylib for path " + + std::string(Path.data(), Path.size())); + + if (auto Err = dlopenFull(*JDS)) + return std::move(Err); + + // Bump the ref-count on this dylib. + ++JDS->DlRefCount; + + // Return the header address. + return JDS->Header; +} + +Error COFFPlatformRuntimeState::dlopenFull(JITDylibState &JDS) { + // Call back to the JIT to push the initializers. + Expected DepInfoMap((COFFJITDylibDepInfoMap())); + if (auto Err = WrapperFunction( + SPSExecutorAddr)>::call(&__orc_rt_coff_push_initializers_tag, + DepInfoMap, + ExecutorAddr::fromPtr(JDS.Header))) + return Err; + if (!DepInfoMap) + return DepInfoMap.takeError(); + + if (auto Err = dlopenInitialize(JDS, *DepInfoMap)) + return Err; + + if (!DepInfoMap->empty()) { + ORC_RT_DEBUG({ + printdbg("Unrecognized dep-info key headers in dlopen of %s\n", + JDS.Name.c_str()); + }); + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep-info key headers " + "while processing dlopen of " + << JDS.Name; + return make_error(ErrStream.str()); + } + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlopenInitialize( + JITDylibState &JDS, COFFJITDylibDepInfoMap &DepInfo) { + ORC_RT_DEBUG({ + printdbg("COFFPlatformRuntimeState::dlopenInitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // Skip visited dependency. + auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); + if (I == DepInfo.end()) + return Error::success(); + + auto DI = std::move(I->second); + DepInfo.erase(I); + + // Run initializers of dependencies in proper order by depth-first traversal + // of dependency graph. + std::vector OldDeps; + std::swap(JDS.Deps, OldDeps); + JDS.Deps.reserve(DI.size()); + for (auto DepHeaderAddr : DI) { + auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr()); + if (!DepJDS) { + std::ostringstream ErrStream; + ErrStream << "Encountered unrecognized dep header " + << DepHeaderAddr.toPtr() << " while initializing " + << JDS.Name; + return make_error(ErrStream.str()); + } + ++DepJDS->LinkedAgainstRefCount; + if (auto Err = dlopenInitialize(*DepJDS, DepInfo)) + return Err; + } + + // Run static initializers. + JDS.CInitSection.RunAllNewAndFlush(); + JDS.CXXInitSection.RunAllNewAndFlush(); + + // Decrement old deps. + for (auto *DepJDS : OldDeps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; + } + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { + // Try to find JITDylib state by header. + auto *JDS = getJITDylibStateByHeader(DSOHandle); + + if (!JDS) { + std::ostringstream ErrStream; + ErrStream << "No registered JITDylib for " << DSOHandle; + return make_error(ErrStream.str()); + } + + // Bump the ref-count. + --JDS->DlRefCount; + + if (!JDS->referenced()) + return dlcloseDeinitialize(*JDS); + + return Error::success(); +} + +Error COFFPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) { + ORC_RT_DEBUG({ + printdbg("COFFPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", + JDS.Name.c_str()); + }); + + // FIXME: call atexits. + + // Run static terminators. + JDS.CPreTermSection.RunAllNewAndFlush(); + JDS.CTermSection.RunAllNewAndFlush(); + + // Queue all xtors as new again. + JDS.CInitSection.Reset(); + JDS.CXXInitSection.Reset(); + JDS.CPreTermSection.Reset(); + JDS.CTermSection.Reset(); + + // Deinitialize any dependencies. + for (auto *DepJDS : JDS.Deps) { + --DepJDS->LinkedAgainstRefCount; + if (!DepJDS->referenced()) + if (auto Err = dlcloseDeinitialize(*DepJDS)) + return Err; + } + + return Error::success(); +} + +Expected +COFFPlatformRuntimeState::lookupSymbolInJITDylib(void *header, + std::string_view Sym) { + Expected Result((ExecutorAddr())); + if (auto Err = WrapperFunction( + SPSExecutorAddr, SPSString)>::call(&__orc_rt_coff_symbol_lookup_tag, + Result, + ExecutorAddr::fromPtr(header), + Sym)) + return std::move(Err); + return Result; +} + +Error COFFPlatformRuntimeState::registerObjectSections( + ExecutorAddr HeaderAddr, + std::vector> Secs, + bool RunInitializers) { + std::lock_guard Lock(JDStatesMutex); + auto I = JDStates.find(HeaderAddr.toPtr()); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to register unrecognized header " + << HeaderAddr.getValue(); + return make_error(ErrStream.str()); + } + auto &JDState = I->second; + for (auto &KV : Secs) { + if (auto Err = registerBlockRange(HeaderAddr, KV.second)) + return Err; + if (KV.first.empty()) + continue; + char LastChar = KV.first.data()[KV.first.size() - 1]; + if (KV.first == ".pdata") { + if (auto Err = registerSEHFrames(HeaderAddr, KV.second)) + return Err; + } else if (KV.first >= ".CRT$XIA" && KV.first <= ".CRT$XIZ") { + if (RunInitializers) + JDState.CInitSection.Register(LastChar, + KV.second.toSpan()); + else + JDState.CInitSection.RegisterNoRun(LastChar, + KV.second.toSpan()); + } else if (KV.first >= ".CRT$XCA" && KV.first <= ".CRT$XCZ") { + if (RunInitializers) + JDState.CXXInitSection.Register(LastChar, + KV.second.toSpan()); + else + JDState.CXXInitSection.RegisterNoRun( + LastChar, KV.second.toSpan()); + } else if (KV.first >= ".CRT$XPA" && KV.first <= ".CRT$XPZ") + JDState.CPreTermSection.Register(LastChar, + KV.second.toSpan()); + else if (KV.first >= ".CRT$XTA" && KV.first <= ".CRT$XTZ") + JDState.CTermSection.Register(LastChar, + KV.second.toSpan()); + } + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterObjectSections( + ExecutorAddr HeaderAddr, + std::vector> Secs) { + std::lock_guard Lock(JDStatesMutex); + auto I = JDStates.find(HeaderAddr.toPtr()); + if (I == JDStates.end()) { + std::ostringstream ErrStream; + ErrStream << "Attempted to register unrecognized header " + << HeaderAddr.getValue(); + return make_error(ErrStream.str()); + } + auto &JDState = I->second; + for (auto &KV : Secs) { + if (auto Err = deregisterBlockRange(HeaderAddr, KV.second)) + return Err; + if (KV.first == ".pdata") + if (auto Err = deregisterSEHFrames(HeaderAddr, KV.second)) + return Err; + } + return Error::success(); +} + +Error COFFPlatformRuntimeState::registerSEHFrames( + ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { + int N = (SEHFrameRange.End.getValue() - SEHFrameRange.Start.getValue()) / + sizeof(RUNTIME_FUNCTION); + auto Func = SEHFrameRange.Start.toPtr(); + if (!RtlAddFunctionTable(Func, N, + static_cast(HeaderAddr.getValue()))) + return make_error("Failed to register SEH frames"); + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterSEHFrames( + ExecutorAddr HeaderAddr, ExecutorAddrRange SEHFrameRange) { + if (!RtlDeleteFunctionTable(SEHFrameRange.Start.toPtr())) + return make_error("Failed to deregister SEH frames"); + return Error::success(); +} + +Error COFFPlatformRuntimeState::registerBlockRange(ExecutorAddr HeaderAddr, + ExecutorAddrRange Range) { + assert(!BlockRanges.count(Range.Start.toPtr()) && + "Block range address already registered"); + BlockRange B = {HeaderAddr.toPtr(), Range.size().getValue()}; + BlockRanges.emplace(Range.Start.toPtr(), B); + return Error::success(); +} + +Error COFFPlatformRuntimeState::deregisterBlockRange(ExecutorAddr HeaderAddr, + ExecutorAddrRange Range) { + assert(BlockRanges.count(Range.Start.toPtr()) && + "Block range address not registered"); + BlockRanges.erase(Range.Start.toPtr()); + return Error::success(); +} + +void COFFPlatformRuntimeState::initialize() { + assert(!CPS && "COFFPlatformRuntimeState should be null"); + CPS = new COFFPlatformRuntimeState(); +} + +COFFPlatformRuntimeState &COFFPlatformRuntimeState::get() { + assert(CPS && "COFFPlatformRuntimeState not initialized"); + return *CPS; +} + +void COFFPlatformRuntimeState::destroy() { + assert(CPS && "COFFPlatformRuntimeState not initialized"); + delete CPS; +} + +void *COFFPlatformRuntimeState::findJITDylibBaseByPC(uint64_t PC) { + std::lock_guard Lock(JDStatesMutex); + auto It = BlockRanges.upper_bound(reinterpret_cast(PC)); + if (It == BlockRanges.begin()) + return nullptr; + --It; + auto &Range = It->second; + if (PC >= reinterpret_cast(It->first) + Range.Size) + return nullptr; + return Range.Header; +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_platform_bootstrap(char *ArgData, size_t ArgSize) { + COFFPlatformRuntimeState::initialize(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_platform_shutdown(char *ArgData, size_t ArgSize) { + COFFPlatformRuntimeState::destroy(); + return WrapperFunctionResult().release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_register_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](std::string &Name, ExecutorAddr HeaderAddr) { + return COFFPlatformRuntimeState::get().registerJITDylib( + std::move(Name), HeaderAddr.toPtr()); + }) + .release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_deregister_jitdylib(char *ArgData, size_t ArgSize) { + return WrapperFunction::handle( + ArgData, ArgSize, + [](ExecutorAddr HeaderAddr) { + return COFFPlatformRuntimeState::get().deregisterJITDylib( + HeaderAddr.toPtr()); + }) + .release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_register_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector> + &Secs, + bool RunInitializers) { + return COFFPlatformRuntimeState::get().registerObjectSections( + HeaderAddr, std::move(Secs), RunInitializers); + }) + .release(); +} + +ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult +__orc_rt_coff_deregister_object_sections(char *ArgData, size_t ArgSize) { + return WrapperFunction:: + handle(ArgData, ArgSize, + [](ExecutorAddr HeaderAddr, + std::vector> + &Secs) { + return COFFPlatformRuntimeState::get().deregisterObjectSections( + HeaderAddr, std::move(Secs)); + }) + .release(); +} +//------------------------------------------------------------------------------ +// JIT'd dlfcn alternatives. +//------------------------------------------------------------------------------ + +const char *__orc_rt_coff_jit_dlerror() { + return COFFPlatformRuntimeState::get().dlerror(); +} + +void *__orc_rt_coff_jit_dlopen(const char *path, int mode) { + return COFFPlatformRuntimeState::get().dlopen(path, mode); +} + +int __orc_rt_coff_jit_dlclose(void *header) { + return COFFPlatformRuntimeState::get().dlclose(header); +} + +void *__orc_rt_coff_jit_dlsym(void *header, const char *symbol) { + return COFFPlatformRuntimeState::get().dlsym(header, symbol); +} + +//------------------------------------------------------------------------------ +// COFF SEH exception support +//------------------------------------------------------------------------------ + +struct ThrowInfo { + uint32_t attributes; + void *data; +}; + +ORC_RT_INTERFACE void __stdcall __orc_rt_coff_cxx_throw_exception( + void *pExceptionObject, ThrowInfo *pThrowInfo) { + constexpr uint32_t EH_EXCEPTION_NUMBER = 'msc' | 0xE0000000; + constexpr uint32_t EH_MAGIC_NUMBER1 = 0x19930520; + auto BaseAddr = COFFPlatformRuntimeState::get().findJITDylibBaseByPC( + reinterpret_cast(pThrowInfo)); + if (!BaseAddr) { + // This is not from JIT'd region. + // FIXME: Use the default implementation like below when alias api is + // capable. _CxxThrowException(pExceptionObject, pThrowInfo); + fprintf(stderr, "Throwing exception from compiled callback into JIT'd " + "exception handler not supported yet.\n"); + abort(); + return; + } + const ULONG_PTR parameters[] = { + EH_MAGIC_NUMBER1, + reinterpret_cast(pExceptionObject), + reinterpret_cast(pThrowInfo), + reinterpret_cast(BaseAddr), + }; + RaiseException(EH_EXCEPTION_NUMBER, EXCEPTION_NONCONTINUABLE, + _countof(parameters), parameters); +} + +//------------------------------------------------------------------------------ +// COFF Run Program +//------------------------------------------------------------------------------ + +ORC_RT_INTERFACE int64_t __orc_rt_coff_run_program(const char *JITDylibName, + const char *EntrySymbolName, + int argc, char *argv[]) { + using MainTy = int (*)(int, char *[]); + + void *H = + __orc_rt_coff_jit_dlopen(JITDylibName, __orc_rt::coff::ORC_RT_RTLD_LAZY); + if (!H) { + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + return -1; + } + + auto *Main = + reinterpret_cast(__orc_rt_coff_jit_dlsym(H, EntrySymbolName)); + + if (!Main) { + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + return -1; + } + + int Result = Main(argc, argv); + + if (__orc_rt_coff_jit_dlclose(H) == -1) + __orc_rt_log_error(__orc_rt_coff_jit_dlerror()); + + return Result; +} diff --git a/compiler-rt/lib/orc/common.h b/compiler-rt/lib/orc/common.h --- a/compiler-rt/lib/orc/common.h +++ b/compiler-rt/lib/orc/common.h @@ -34,14 +34,14 @@ /// This is declared for use by the runtime, but should be implemented in the /// executor or provided by a definition added to the JIT before the runtime /// is loaded. -extern "C" __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT; +ORC_RT_IMPORT __orc_rt_Opaque __orc_rt_jit_dispatch_ctx ORC_RT_WEAK_IMPORT; /// For dispatching calls to the JIT object. /// /// This is declared for use by the runtime, but should be implemented in the /// executor or provided by a definition added to the JIT before the runtime /// is loaded. -extern "C" __orc_rt_CWrapperFunctionResult +ORC_RT_IMPORT __orc_rt_CWrapperFunctionResult __orc_rt_jit_dispatch(__orc_rt_Opaque *DispatchCtx, const void *FnTag, const char *Data, size_t Size) ORC_RT_WEAK_IMPORT; diff --git a/compiler-rt/lib/orc/compiler.h b/compiler-rt/lib/orc/compiler.h --- a/compiler-rt/lib/orc/compiler.h +++ b/compiler-rt/lib/orc/compiler.h @@ -15,8 +15,15 @@ #ifndef ORC_RT_COMPILER_H #define ORC_RT_COMPILER_H +#if defined(_WIN32) +#define ORC_RT_INTERFACE extern "C" +#define ORC_RT_HIDDEN +#define ORC_RT_IMPORT extern "C" __declspec(dllimport) +#else #define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default"))) #define ORC_RT_HIDDEN __attribute__((visibility("hidden"))) +#define ORC_RT_IMPORT extern "C" +#endif #ifndef __has_builtin # define __has_builtin(x) 0 @@ -56,8 +63,10 @@ #define ORC_RT_UNLIKELY(EXPR) (EXPR) #endif -#ifdef __APPLE__ +#if defined(__APPLE__) #define ORC_RT_WEAK_IMPORT __attribute__((weak_import)) +#elif defined(_WIN32) +#define ORC_RT_WEAK_IMPORT #else #define ORC_RT_WEAK_IMPORT __attribute__((weak)) #endif diff --git a/compiler-rt/lib/orc/error.h b/compiler-rt/lib/orc/error.h --- a/compiler-rt/lib/orc/error.h +++ b/compiler-rt/lib/orc/error.h @@ -113,9 +113,7 @@ bool isChecked() const { return ErrPtr & 0x1; } - void setChecked(bool Checked) { - ErrPtr = (reinterpret_cast(ErrPtr) & ~uintptr_t(1)) | Checked; - } + void setChecked(bool Checked) { ErrPtr = (ErrPtr & ~uintptr_t(1)) | Checked; } template std::unique_ptr takePayload() { static_assert(std::is_base_of::value, diff --git a/compiler-rt/test/orc/TestCases/Windows/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/Windows/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/lit.local.cfg.py @@ -0,0 +1,2 @@ +if config.root.host_os != 'Windows': + config.unsupported = True diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/Inputs/standalone-dylib.c b/compiler-rt/test/orc/TestCases/Windows/x86-64/Inputs/standalone-dylib.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/Inputs/standalone-dylib.c @@ -0,0 +1,9 @@ +#include + +int Ctor() { + printf("constructor\n"); + return 0; +} + +#pragma section(".CRT$XIV", long, read) +__declspec(allocate(".CRT$XIV")) int (*i1)(void) = Ctor; diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.c b/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.c @@ -0,0 +1,10 @@ +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t 2>&1 | FileCheck %s +// CHECK: Hello, world! + +#include +int main(int argc, char *argv[]) { + printf("Hello, world!\n"); + fflush(stdout); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.cpp b/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.cpp @@ -0,0 +1,9 @@ +// RUN: %clangxx -c -o %t %s +// RUN: %llvm_jitlink %t 2>&1 | FileCheck %s +// CHECK: Hello, world! + +#include +int main(int argc, char *argv[]) { + std::cout << "Hello, world!" << std::endl; + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/lit.local.cfg.py b/compiler-rt/test/orc/TestCases/Windows/x86-64/lit.local.cfg.py new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/lit.local.cfg.py @@ -0,0 +1,5 @@ +if config.root.host_arch not in ['AMD64','x86_64']: + config.unsupported = True + +if config.target_arch not in ['AMD64','x86_64']: + config.unsupported = True diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer-three.c b/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer-three.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer-three.c @@ -0,0 +1,36 @@ +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t 2>&1 | FileCheck %s +// CHECK: init1 +// CHECK-NEXT: init2 +// CHECK-NEXT: init3 + +#include + +int init1() { + printf("init1\n"); + return 0; +} + +int init2() { + printf("init2\n"); + return 0; +} + +int init3() { + printf("init3\n"); + return 0; +} + +#pragma section(".CRT$XIX", long, read) +__declspec(allocate(".CRT$XIX")) int (*i3)(void) = init3; + +#pragma section(".CRT$XIV", long, read) +__declspec(allocate(".CRT$XIV")) int (*i1)(void) = init1; + +#pragma section(".CRT$XIW", long, read) +__declspec(allocate(".CRT$XIW")) int (*i2)(void) = init2; + +int main(int argc, char *argv[]) { + fflush(stdout); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer.c b/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer.c @@ -0,0 +1,27 @@ +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t 2>&1 | FileCheck %s +// CHECK: init1 +// CHECK-NEXT: init2 + +#include + +int init1() { + printf("init1\n"); + return 0; +} + +int init2() { + printf("init2\n"); + return 0; +} + +#pragma section(".CRT$XIW", long, read) +__declspec(allocate(".CRT$XIW")) int (*i2)(void) = init2; + +#pragma section(".CRT$XIV", long, read) +__declspec(allocate(".CRT$XIV")) int (*i1)(void) = init1; + +int main(int argc, char *argv[]) { + fflush(stdout); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/sehframe-handling.cpp b/compiler-rt/test/orc/TestCases/Windows/x86-64/sehframe-handling.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/sehframe-handling.cpp @@ -0,0 +1,15 @@ +// RUN: %clangxx -c -o %t %s +// RUN: %llvm_jitlink %t + +extern "C" __declspec(dllimport) void llvm_jitlink_setTestResultOverride( + long Value); + +int main(int argc, char *argv[]) { + llvm_jitlink_setTestResultOverride(1); + try { + throw 0; + } catch (int X) { + llvm_jitlink_setTestResultOverride(X); + } + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.S b/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.S new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.S @@ -0,0 +1,135 @@ +// Test that basic COFF static initializers work. The main function in this +// test returns the expression a + b + c + 1. a is initialzed as -1 and c is +// initialized as -2 by static initializers. b is initialized as 2 by constant +// value. The test will succeeds only if static initializers are ran successfully. +// +// RUN: %clang -c -o %t %s +// RUN: %llvm_jitlink %t + .text + .def @feat.00; + .scl 3; + .type 0; + .endef + .globl @feat.00 +.set @feat.00, 0 + .file "static-initializer.cpp" + .def init; + .scl 2; + .type 32; + .endef + .globl init # -- Begin function init + .p2align 4, 0x90 +init: # @init +# %bb.0: + movl $4294967295, %eax # imm = 0xFFFFFFFF + retq + # -- End function + .def init2; + .scl 2; + .type 32; + .endef + .globl init2 # -- Begin function init2 + .p2align 4, 0x90 +init2: # @init2 +# %bb.0: + movl $4294967294, %eax # imm = 0xFFFFFFFE + retq + # -- End function + .def "??__Ea@@YAXXZ"; + .scl 3; + .type 32; + .endef + .p2align 4, 0x90 # -- Begin function ??__Ea@@YAXXZ +"??__Ea@@YAXXZ": # @"??__Ea@@YAXXZ" +.seh_proc "??__Ea@@YAXXZ" +# %bb.0: + subq $40, %rsp + .seh_stackalloc 40 + .seh_endprologue + callq init + movl %eax, a(%rip) + addq $40, %rsp + retq + .seh_endproc + # -- End function + .def "??__Ec@@YAXXZ"; + .scl 3; + .type 32; + .endef + .p2align 4, 0x90 # -- Begin function ??__Ec@@YAXXZ +"??__Ec@@YAXXZ": # @"??__Ec@@YAXXZ" +.seh_proc "??__Ec@@YAXXZ" +# %bb.0: + subq $40, %rsp + .seh_stackalloc 40 + .seh_endprologue + callq init2 + movl %eax, c(%rip) + addq $40, %rsp + retq + .seh_endproc + # -- End function + .def main; + .scl 2; + .type 32; + .endef + .globl main # -- Begin function main + .p2align 4, 0x90 +main: # @main +.seh_proc main +# %bb.0: + subq $56, %rsp + .seh_stackalloc 56 + .seh_endprologue + movl $0, 52(%rsp) + movq %rdx, 40(%rsp) + movl %ecx, 36(%rsp) + movl a(%rip), %ecx + addl b(%rip), %ecx + addl c(%rip), %ecx + addl $1, %ecx + callq *__imp_llvm_jitlink_setTestResultOverride(%rip) + xorl %eax, %eax + addq $56, %rsp + retq + .seh_endproc + # -- End function + .def _GLOBAL__sub_I_static_initializer.cpp; + .scl 3; + .type 32; + .endef + .p2align 4, 0x90 # -- Begin function _GLOBAL__sub_I_static_initializer.cpp +_GLOBAL__sub_I_static_initializer.cpp: # @_GLOBAL__sub_I_static_initializer.cpp +.seh_proc _GLOBAL__sub_I_static_initializer.cpp +# %bb.0: + subq $40, %rsp + .seh_stackalloc 40 + .seh_endprologue + callq "??__Ea@@YAXXZ" + callq "??__Ec@@YAXXZ" + nop + addq $40, %rsp + retq + .seh_endproc + # -- End function + .bss + .globl a # @a + .p2align 2 +a: + .long 0 # 0x0 + + .data + .globl b # @b + .p2align 2 +b: + .long 2 # 0x2 + + .bss + .globl c # @c + .p2align 2 +c: + .long 0 # 0x0 + + .section .CRT$XCU,"dr" + .p2align 3 + .quad _GLOBAL__sub_I_static_initializer.cpp diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.cpp b/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.cpp @@ -0,0 +1,17 @@ +// RUN: %clangxx -c -o %t %s +// RUN: %llvm_jitlink %t + +extern "C" __declspec(dllimport) void llvm_jitlink_setTestResultOverride( + long Value); + +extern "C" int init() { return -1; } +extern "C" int init2() { return -2; } + +extern "C" int a = init(); +extern "C" int b = 2; +extern "C" int c = init2(); + +int main(int argc, char *argv[]) { + llvm_jitlink_setTestResultOverride(a + b + c + 1); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Windows/x86-64/trivial-jit-dlopen.c b/compiler-rt/test/orc/TestCases/Windows/x86-64/trivial-jit-dlopen.c new file mode 100644 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Windows/x86-64/trivial-jit-dlopen.c @@ -0,0 +1,34 @@ +// XFAIL: * +// Test that __orc_rt_coff_jit_dlopen and __orc_rt_coff_jit_dlclose work as +// expected for a straightforward dlopen; dlclose sequence: first the +// constructors should be run. +// +// RUN: %clang -c -o %t.inits.o %p/Inputs/standalone-dylib.c +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias dlopen=__orc_rt_coff_jit_dlopen \ +// RUN: -alias dlclose=__orc_rt_coff_jit_dlclose \ +// RUN: %t.test.o -jd inits %t.inits.o -lmain | FileCheck %s + +// CHECK: entering main +// CHECK-NEXT: constructor +// CHECK-NEXT: leaving main + +#include +__declspec(dllimport) void *dlopen(const char *path, int mode); +__declspec(dllimport) int dlclose(void *handle); + +int main(int argc, char *argv[]) { + printf("entering main\n"); + void *H = dlopen("inits", 0); + if (!H) { + printf("failed\n"); + return -1; + } + if (dlclose(H) == -1) { + printf("failed\n"); + return -1; + } + printf("leaving main\n"); + return 0; +} diff --git a/compiler-rt/test/orc/lit.cfg.py b/compiler-rt/test/orc/lit.cfg.py --- a/compiler-rt/test/orc/lit.cfg.py +++ b/compiler-rt/test/orc/lit.cfg.py @@ -29,8 +29,13 @@ config.substitutions.append( ('%clangxx ', build_invocation(config.cxx_mode_flags + [config.target_cflags]))) -config.substitutions.append( - ('%llvm_jitlink', (llvm_jitlink + ' -orc-runtime=' + orc_rt_path))) +if config.host_os == 'Windows': + config.substitutions.append( + ('%llvm_jitlink', (llvm_jitlink + ' -orc-runtime=' + + orc_rt_path + ' -no-process-syms=true -slab-allocate=64MB'))) +else: + config.substitutions.append( + ('%llvm_jitlink', (llvm_jitlink + ' -orc-runtime=' + orc_rt_path))) config.substitutions.append( ('%lli_orc_jitlink', (lli + ' -jit-kind=orc -jit-linker=jitlink -orc-runtime=' + orc_rt_path))) @@ -40,5 +45,5 @@ # Exclude Inputs directories. config.excludes = ['Inputs'] -if config.host_os not in ['Darwin', 'FreeBSD', 'Linux']: +if config.host_os not in ['Darwin', 'FreeBSD', 'Linux', 'Windows']: config.unsupported = True diff --git a/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h new file mode 100644 --- /dev/null +++ b/llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h @@ -0,0 +1,211 @@ +//===--- COFFPlatform.h -- Utilities for executing COFF in Orc --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities for executing JIT'd COFF in Orc. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_COFFPLATFORM_H +#define LLVM_EXECUTIONENGINE_ORC_COFFPLATFORM_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +#include +#include +#include +#include + +namespace llvm { +namespace orc { + +/// Mediates between COFF initialization and ExecutionSession state. +class COFFPlatform : public Platform { +public: + /// A function that will be called with the name of dll file that must be + /// loaded. + using LoadDynamicLibrary = + unique_function; + + /// Try to create a COFFPlatform instance, adding the ORC runtime to the + /// given JITDylib. + static Expected> + Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath = nullptr, + Optional RuntimeAliases = None); + + Error bootstrap(JITDylib &PlatformJD); + + ExecutionSession &getExecutionSession() const { return ES; } + ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + + Error setupJITDylib(JITDylib &JD) override; + Error teardownJITDylib(JITDylib &JD) override; + Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) override; + Error notifyRemoving(ResourceTracker &RT) override; + + /// Returns an AliasMap containing the default aliases for the COFFPlatform. + /// This can be modified by clients when constructing the platform to add + /// or remove aliases. + static SymbolAliasMap standardPlatformAliases(ExecutionSession &ES); + + /// Returns the array of required CXX aliases. + static ArrayRef> requiredCXXAliases(); + + /// Returns the array of standard runtime utility aliases for COFF. + static ArrayRef> + standardRuntimeUtilityAliases(); + + static bool isInitializerSection(StringRef Name) { + return Name.startswith(".CRT"); + } + + static StringRef getSEHFrameSectionName() { return ".pdata"; } + +private: + using COFFJITDylibDepInfo = std::vector; + using COFFJITDylibDepInfoMap = + std::vector>; + using COFFObjectSectionsMap = + SmallVector>; + using PushInitializersSendResultFn = + unique_function)>; + using SendSymbolAddressFn = unique_function)>; + using JITDylibDepMap = DenseMap>; + + // The COFFPlatformPlugin scans/modifies LinkGraphs to support COFF + // platform features including initializers, exceptions, and language + // runtime registration. + class COFFPlatformPlugin : public ObjectLinkingLayer::Plugin { + public: + COFFPlatformPlugin(COFFPlatform &CP) : CP(CP) {} + + void modifyPassConfig(MaterializationResponsibility &MR, + jitlink::LinkGraph &G, + jitlink::PassConfiguration &Config) override; + + SyntheticSymbolDependenciesMap + getSyntheticSymbolDependencies(MaterializationResponsibility &MR) override; + + // FIXME: We should be tentatively tracking scraped sections and discarding + // if the MR fails. + Error notifyFailed(MaterializationResponsibility &MR) override { + return Error::success(); + } + + Error notifyRemovingResources(ResourceKey K) override { + return Error::success(); + } + + void notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) override {} + + private: + using InitSymbolDepMap = + DenseMap; + + Error associateJITDylibHeaderSymbol(jitlink::LinkGraph &G, + MaterializationResponsibility &MR, + bool Bootstrap); + + Error preserveInitializerSections(jitlink::LinkGraph &G, + MaterializationResponsibility &MR); + Error registerObjectPlatformSections(jitlink::LinkGraph &G, JITDylib &JD); + Error registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G, + JITDylib &JD); + + std::mutex PluginMutex; + COFFPlatform &CP; + InitSymbolDepMap InitSymbolDeps; + }; + + struct JDBootstrapState { + JITDylib *JD = nullptr; + std::string JDName; + ExecutorAddr HeaderAddr; + std::list ObjectSectionsMaps; + SmallVector> Initializers; + }; + + static bool supportedTarget(const Triple &TT); + + COFFPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, + LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath, Error &Err); + + // Associate COFFPlatform JIT-side runtime support functions with handlers. + Error associateRuntimeSupportFunctions(JITDylib &PlatformJD); + + // Records the addresses of runtime symbols used by the platform. + Error bootstrapCOFFRuntime(JITDylib &PlatformJD); + + // Run a specific void function if it exists. + Error runSymbolIfExists(JITDylib &PlatformJD, StringRef SymbolName); + + // Run collected initializers in boostrap stage. + Error runBootstrapInitializers(JDBootstrapState &BState); + Error runBootstrapSubsectionInitializers(JDBootstrapState &BState, + StringRef Start, StringRef End); + + // Build dependency graph of a JITDylib + Expected buildJDDepMap(JITDylib &JD); + + // Implements rt_pushInitializers by making repeat async lookups for + // initializer symbols (each lookup may spawn more initializer symbols if + // it pulls in new materializers, e.g. from objects in a static library). + void pushInitializersLoop(PushInitializersSendResultFn SendResult, + JITDylibSP JD, JITDylibDepMap &JDDepMap); + + void rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr); + + void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, + StringRef SymbolName); + + ExecutionSession &ES; + ObjectLinkingLayer &ObjLinkingLayer; + + LoadDynamicLibrary LoadDynLibrary; + std::unique_ptr VCRuntimeBootstrap; + + SymbolStringPtr COFFHeaderStartSymbol; + + // State of bootstrap in progress + std::map JDBootstrapStates; + std::atomic Bootstrapping; + + ExecutorAddr orc_rt_coff_platform_bootstrap; + ExecutorAddr orc_rt_coff_platform_shutdown; + ExecutorAddr orc_rt_coff_register_object_sections; + ExecutorAddr orc_rt_coff_deregister_object_sections; + ExecutorAddr orc_rt_coff_register_jitdylib; + ExecutorAddr orc_rt_coff_deregister_jitdylib; + + DenseMap JITDylibToHeaderAddr; + DenseMap HeaderAddrToJITDylib; + + DenseMap RegisteredInitSymbols; + + std::set DylibsToPreload; + + std::mutex PlatformMutex; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_COFFPLATFORM_H diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -4,6 +4,7 @@ add_llvm_component_library(LLVMOrcJIT COFFVCRuntimeSupport.cpp + COFFPlatform.cpp CompileOnDemandLayer.cpp CompileUtils.cpp Core.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -0,0 +1,839 @@ +//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" + +#include "llvm/Object/COFF.h" + +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { +namespace shared { + +using SPSCOFFJITDylibDepInfo = SPSSequence; +using SPSCOFFJITDylibDepInfoMap = + SPSSequence>; +using SPSCOFFObjectSectionsMap = + SPSSequence>; +using SPSCOFFRegisterObjectSectionsArgs = + SPSArgList; +using SPSCOFFDeregisterObjectSectionsArgs = + SPSArgList; + +} // namespace shared +} // namespace orc +} // namespace llvm +namespace { + +class COFFHeaderMaterializationUnit : public MaterializationUnit { +public: + COFFHeaderMaterializationUnit(COFFPlatform &CP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)), + CP(CP) {} + + StringRef getName() const override { return "COFFHeaderMU"; } + + void materialize(std::unique_ptr R) override { + unsigned PointerSize; + support::endianness Endianness; + const auto &TT = + CP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + + switch (TT.getArch()) { + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto G = std::make_unique( + "", TT, PointerSize, Endianness, + jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", jitlink::MemProt::Read); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is __ImageBase symbol. + auto &ImageBaseSymbol = G->addDefinedSymbol( + HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, true); + + addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol); + + CP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + struct NTHeader { + support::ulittle32_t PEMagic; + object::coff_file_header FileHeader; + struct PEHeader { + object::pe32plus_header Header; + object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1]; + } OptionalHeader; + }; + + struct HeaderBlockContent { + object::dos_header DOSHeader; + NTHeader NTHeader; + }; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + HeaderBlockContent Hdr = {}; + + // Set up magic + Hdr.DOSHeader.Magic[0] = 'M'; + Hdr.DOSHeader.Magic[1] = 'Z'; + Hdr.DOSHeader.AddressOfNewExeHeader = + offsetof(HeaderBlockContent, NTHeader); + uint32_t PEMagic = *reinterpret_cast(COFF::PEMagic); + Hdr.NTHeader.PEMagic = PEMagic; + Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS; + + switch (G.getTargetTriple().getArch()) { + case Triple::x86_64: + Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast(&Hdr), sizeof(Hdr))); + + return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, + 0); + } + + static void addImageBaseRelocationEdge(jitlink::Block &B, + jitlink::Symbol &ImageBase) { + auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) + + offsetof(NTHeader, OptionalHeader) + + offsetof(object::pe32plus_header, ImageBase); + B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0); + } + + static MaterializationUnit::Interface + createHeaderInterface(COFFPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + + return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), + HeaderStartSymbol); + } + + COFFPlatform &CP; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected> +COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, + const char *VCRuntimePath, + Optional RuntimeAliases) { + auto &EPC = ES.getExecutorProcessControl(); + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error("Unsupported COFFPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr(new COFFPlatform( + ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), + std::move(LoadDynLibrary), VCRuntimePath, Err)); + if (Err) + return std::move(Err); + return std::move(P); +} + +Error COFFPlatform::setupJITDylib(JITDylib &JD) { + if (auto Err = JD.define(std::make_unique( + *this, COFFHeaderStartSymbol))) + return Err; + + if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError()) + return Err; + + if (!Bootstrapping) { + auto ImportedLibs = VCRuntimeBootstrap->loadStaticVCRuntime(JD); + if (!ImportedLibs) + return ImportedLibs.takeError(); + for (auto &Lib : *ImportedLibs) + if (auto Err = LoadDynLibrary(JD, Lib)) + return Err; + if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD)) + return Err; + } + + return Error::success(); +} + +Error COFFPlatform::teardownJITDylib(JITDylib &JD) { + std::lock_guard Lock(PlatformMutex); + auto I = JITDylibToHeaderAddr.find(&JD); + if (I != JITDylibToHeaderAddr.end()) { + assert(HeaderAddrToJITDylib.count(I->second) && + "HeaderAddrToJITDylib missing entry"); + HeaderAddrToJITDylib.erase(I->second); + JITDylibToHeaderAddr.erase(I); + } + return Error::success(); +} + +Error COFFPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + RegisteredInitSymbols[&JD].add(InitSym, + SymbolLookupFlags::WeaklyReferencedSymbol); + + LLVM_DEBUG({ + dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU " + << MU.getName() << "\n"; + }); + return Error::success(); +} + +Error COFFPlatform::notifyRemoving(ResourceTracker &RT) { + llvm_unreachable("Not supported yet"); +} + +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} + +SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} + +ArrayRef> +COFFPlatform::requiredCXXAliases() { + static const std::pair RequiredCXXAliases[] = { + {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"}, + }; + + return ArrayRef>(RequiredCXXAliases); +} + +ArrayRef> +COFFPlatform::standardRuntimeUtilityAliases() { + static const std::pair + StandardRuntimeUtilityAliases[] = { + {"__orc_rt_run_program", "__orc_rt_coff_run_program"}, + {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"}, + {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"}, + {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"}, + {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"}, + {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}}; + + return ArrayRef>( + StandardRuntimeUtilityAliases); +} + +bool COFFPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} + +COFFPlatform::COFFPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr OrcRuntimeGenerator, + LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + LoadDynLibrary(std::move(LoadDynLibrary)), + COFFHeaderStartSymbol(ES.intern("__ImageBase")) { + ErrorAsOutParameter _(&Err); + + Bootstrapping.store(true); + ObjLinkingLayer.addPlugin(std::make_unique(*this)); + + // Load vc runtime + auto VCRT = + COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath); + if (!VCRT) { + Err = std::move(VCRT.takeError()); + return; + } + VCRuntimeBootstrap = std::move(*VCRT); + + for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries()) + DylibsToPreload.insert(Lib); + + auto ImportedLibs = VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD); + if (!ImportedLibs) { + Err = ImportedLibs.takeError(); + return; + } + + for (auto &Lib : *ImportedLibs) + DylibsToPreload.insert(Lib); + + // PlatformJD hasn't been set up by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; + } + + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); +} + +Error COFFPlatform::bootstrap(JITDylib &PlatformJD) { + for (auto &Lib : DylibsToPreload) + if (auto Err = LoadDynLibrary(PlatformJD, Lib)) + return Err; + + if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) + return Err; + + // Associate wrapper function tags with JIT-side function implementations. + if (auto Err = associateRuntimeSupportFunctions(PlatformJD)) { + return Err; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto Err = bootstrapCOFFRuntime(PlatformJD)) { + return Err; + } + + Bootstrapping.store(false); + JDBootstrapStates.clear(); + + return Error::success(); +} + +Expected +COFFPlatform::buildJDDepMap(JITDylib &JD) { + return ES.runSessionLocked([&]() -> Expected { + JITDylibDepMap JDDepMap; + + SmallVector Worklist({&JD}); + while (!Worklist.empty()) { + auto CurJD = Worklist.back(); + Worklist.pop_back(); + + auto &DM = JDDepMap[CurJD]; + CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) { + DM.reserve(O.size()); + for (auto &KV : O) { + if (KV.first == CurJD) + continue; + { + // Bare jitdylibs not known to the platform + std::lock_guard Lock(PlatformMutex); + if (!JITDylibToHeaderAddr.count(KV.first)) { + LLVM_DEBUG({ + dbgs() << "JITDylib unregistered to COFFPlatform detected in " + "LinkOrder: " + << CurJD->getName() << "\n"; + }); + continue; + } + } + DM.push_back(KV.first); + // Push unvisited entry. + if (!JDDepMap.count(KV.first)) { + Worklist.push_back(KV.first); + JDDepMap[KV.first] = {}; + } + } + }); + } + return std::move(JDDepMap); + }); +} + +void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult, + JITDylibSP JD, + JITDylibDepMap &JDDepMap) { + SmallVector Worklist({JD.get()}); + DenseSet Visited({JD.get()}); + DenseMap NewInitSymbols; + ES.runSessionLocked([&]() { + while (!Worklist.empty()) { + auto CurJD = Worklist.back(); + Worklist.pop_back(); + + auto RISItr = RegisteredInitSymbols.find(CurJD); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[CurJD] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + + for (auto DepJD : JDDepMap[CurJD]) + if (!Visited.count(DepJD)) { + Worklist.push_back(DepJD); + Visited.insert(DepJD); + } + } + }); + + // If there are no further init symbols to look up then send the link order + // (as a list of header addresses) to the caller. + if (NewInitSymbols.empty()) { + // Build the dep info map to return. + COFFJITDylibDepInfoMap DIM; + DIM.reserve(JDDepMap.size()); + for (auto &KV : JDDepMap) { + std::lock_guard Lock(PlatformMutex); + COFFJITDylibDepInfo DepInfo; + DepInfo.reserve(KV.second.size()); + for (auto &Dep : KV.second) { + DepInfo.push_back(JITDylibToHeaderAddr[Dep]); + } + auto H = JITDylibToHeaderAddr[KV.first]; + DIM.push_back(std::make_pair(H, std::move(DepInfo))); + } + SendResult(DIM); + return; + } + + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD, + JDDepMap = std::move(JDDepMap)](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + pushInitializersLoop(std::move(SendResult), JD, JDDepMap); + }, + ES, std::move(NewInitSymbols)); +} + +void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr) { + JITDylibSP JD; + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(JDHeaderAddr); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + LLVM_DEBUG({ + dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") "; + if (JD) + dbgs() << "pushing initializers for " << JD->getName() << "\n"; + else + dbgs() << "No JITDylib for header address.\n"; + }); + + if (!JD) { + SendResult( + make_error("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr.getValue()), + inconvertibleErrorCode())); + return; + } + + auto JDDepMap = buildJDDepMap(*JD); + if (!JDDepMap) { + SendResult(JDDepMap.takeError()); + return; + } + + pushInitializersLoop(std::move(SendResult), JD, *JDDepMap); +} + +void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddr Handle, StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "COFFPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // Use functor class to work around XL build compiler issue on AIX. + class RtLookupNotifyComplete { + public: + RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) + : SendResult(std::move(SendResult)) {} + void operator()(Expected Result) { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + } else { + SendResult(Result.takeError()); + } + } + + private: + SendSymbolAddressFn SendResult; + }; + + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready, + RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); +} + +Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutionSession::JITDispatchHandlerAssociationMap WFs; + + using LookupSymbolSPSSig = + SPSExpected(SPSExecutorAddr, SPSString); + WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] = + ES.wrapAsyncWithSPS(this, + &COFFPlatform::rt_lookupSymbol); + using PushInitializersSPSSig = + SPSExpected(SPSExecutorAddr); + WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] = + ES.wrapAsyncWithSPS( + this, &COFFPlatform::rt_pushInitializers); + + return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); +} + +Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) { + llvm::sort(BState.Initializers); + if (auto Err = + runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ")) + return Err; + + if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init")) + return Err; + + if (auto Err = + runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ")) + return Err; + return Error::success(); +} + +Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState, + StringRef Start, + StringRef End) { + for (auto &Initializer : BState.Initializers) + if (Initializer.first >= Start && Initializer.first <= End && + Initializer.second) { + auto Res = + ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second); + if (!Res) + return Res.takeError(); + } + return Error::success(); +} + +Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) { + // Lookup of runtime symbols causes the collection of initializers if + // it's static linking setting. + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + { + {ES.intern("__orc_rt_coff_platform_bootstrap"), + &orc_rt_coff_platform_bootstrap}, + {ES.intern("__orc_rt_coff_platform_shutdown"), + &orc_rt_coff_platform_shutdown}, + {ES.intern("__orc_rt_coff_register_jitdylib"), + &orc_rt_coff_register_jitdylib}, + {ES.intern("__orc_rt_coff_deregister_jitdylib"), + &orc_rt_coff_deregister_jitdylib}, + {ES.intern("__orc_rt_coff_register_object_sections"), + &orc_rt_coff_register_object_sections}, + {ES.intern("__orc_rt_coff_deregister_object_sections"), + &orc_rt_coff_deregister_object_sections}, + })) + return Err; + + // Run static initializers collected in bootstrap stage. + for (auto KV : JDBootstrapStates) { + auto &JDBState = KV.second; + if (auto Err = runBootstrapInitializers(JDBState)) + return Err; + } + + // Call bootstrap functions + if (auto Err = ES.callSPSWrapper(orc_rt_coff_platform_bootstrap)) + return Err; + + // Do the pending jitdylib registration actions that we couldn't do + // because orc runtime was not linked fully. + for (auto KV : JDBootstrapStates) { + auto &JDBState = KV.second; + if (auto Err = ES.callSPSWrapper( + orc_rt_coff_register_jitdylib, JDBState.JDName, + JDBState.HeaderAddr)) + return Err; + + for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps) + if (auto Err = ES.callSPSWrapper( + orc_rt_coff_register_object_sections, JDBState.HeaderAddr, + ObjSectionMap, false)) + return Err; + } + + return Error::success(); +} + +Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD, + StringRef SymbolName) { + ExecutorAddr jit_function; + auto AfterCLookupErr = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + {{ES.intern(SymbolName), &jit_function}}); + if (!AfterCLookupErr) { + auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function); + if (!Res) + return Res.takeError(); + return Error::success(); + } + if (!AfterCLookupErr.isA()) + return AfterCLookupErr; + consumeError(std::move(AfterCLookupErr)); + return Error::success(); +} + +void COFFPlatform::COFFPlatformPlugin::modifyPassConfig( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &Config) { + + bool IsBootstrapping = CP.Bootstrapping.load(); + + if (auto InitSymbol = MR.getInitializerSymbol()) { + if (InitSymbol == CP.COFFHeaderStartSymbol) { + Config.PostAllocationPasses.push_back( + [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) { + return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping); + }); + return; + } + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return preserveInitializerSections(G, MR); + }); + } + + if (!IsBootstrapping) + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerObjectPlatformSections(G, JD); + }); + else + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerObjectPlatformSectionsInBootstrap(G, JD); + }); +} + +ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap +COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies( + MaterializationResponsibility &MR) { + std::lock_guard Lock(PluginMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + SyntheticSymbolDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return SyntheticSymbolDependenciesMap(); +} + +Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + bool IsBootstraping) { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *CP.COFFHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && "Missing COFF header start symbol"); + + auto &JD = MR.getTargetJITDylib(); + std::lock_guard Lock(CP.PlatformMutex); + auto HeaderAddr = (*I)->getAddress(); + CP.JITDylibToHeaderAddr[&JD] = HeaderAddr; + CP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + if (!IsBootstraping) { + G.allocActions().push_back( + {cantFail(WrapperFunctionCall::Create< + SPSArgList>( + CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)), + cantFail(WrapperFunctionCall::Create>( + CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); + } else { + G.allocActions().push_back( + {{}, + cantFail(WrapperFunctionCall::Create>( + CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); + JDBootstrapState BState; + BState.JD = &JD; + BState.JDName = JD.getName(); + BState.HeaderAddr = HeaderAddr; + CP.JDBootstrapStates.emplace(&JD, BState); + } + + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections( + jitlink::LinkGraph &G, JITDylib &JD) { + COFFObjectSectionsMap ObjSecs; + auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; + assert(HeaderAddr && "Must be registered jitdylib"); + for (auto &S : G.sections()) { + jitlink::SectionRange Range(S); + if (Range.getSize()) + ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); + } + + G.allocActions().push_back( + {cantFail(WrapperFunctionCall::Create( + CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)), + cantFail( + WrapperFunctionCall::Create( + CP.orc_rt_coff_deregister_object_sections, HeaderAddr, + ObjSecs))}); + + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + JITLinkSymbolSet InitSectionSymbols; + for (auto &Sec : G.sections()) + if (COFFPlatform::isInitializerSection(Sec.getName())) + for (auto *B : Sec.blocks()) + if (!B->edges_empty()) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, 0, false, true)); + + std::lock_guard Lock(PluginMutex); + InitSymbolDeps[&MR] = InitSectionSymbols; + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin:: + registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G, + JITDylib &JD) { + std::lock_guard Lock(CP.PlatformMutex); + auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; + COFFObjectSectionsMap ObjSecs; + for (auto &S : G.sections()) { + jitlink::SectionRange Range(S); + if (Range.getSize()) + ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); + } + + G.allocActions().push_back( + {{}, + cantFail( + WrapperFunctionCall::Create( + CP.orc_rt_coff_deregister_object_sections, HeaderAddr, + ObjSecs))}); + + auto &BState = CP.JDBootstrapStates[&JD]; + BState.ObjectSectionsMaps.push_back(std::move(ObjSecs)); + + // Collect static initializers + for (auto &S : G.sections()) + if (COFFPlatform::isInitializerSection(S.getName())) + for (auto *B : S.blocks()) { + if (B->edges_empty()) + continue; + for (auto &E : B->edges()) + BState.Initializers.push_back(std::make_pair( + S.getName().str(), + ExecutorAddr(E.getTarget().getAddress() + E.getAddend()))); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp --- a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/Object/COFF.h" @@ -214,7 +215,16 @@ I.SymbolFlags[ES.intern(*Name)] = std::move(*SymFlags); } - // FIXME: handle init symbols + SymbolStringPtr InitSymbol; + for (auto &Sec : Obj.sections()) { + if (auto SecName = Sec.getName()) { + if (COFFPlatform::isInitializerSection(*SecName)) { + addInitSymbol(I, ES, Obj.getFileName()); + break; + } + } else + return SecName.takeError(); + } return I; } diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.h b/llvm/tools/llvm-jitlink/llvm-jitlink.h --- a/llvm/tools/llvm-jitlink/llvm-jitlink.h +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.h @@ -53,9 +53,13 @@ StringMap GOTEntryInfos; }; + using DynLibJDMap = std::map; using SymbolInfoMap = StringMap; using FileInfoMap = StringMap; + Expected getOrLoadDynamicLibrary(StringRef LibPath); + Error loadAndLinkDynamicLibrary(orc::JITDylib &JD, StringRef LibPath); + Expected findFileInfo(StringRef FileName); Expected findSectionInfo(StringRef FileName, StringRef SectionName); @@ -68,6 +72,8 @@ Expected findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem); + DynLibJDMap DynLibJDs; + SymbolInfoMap SymbolInfos; FileInfoMap FileInfos; uint64_t SizeBeforePruning = 0; diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -15,6 +15,8 @@ #include "llvm-jitlink.h" #include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" +#include "llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h" #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" #include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" @@ -810,10 +812,8 @@ LLVM_DEBUG(dbgs() << "Loading dylibs...\n"); for (const auto &Dylib : Dylibs) { LLVM_DEBUG(dbgs() << " " << Dylib << "\n"); - auto G = orc::EPCDynamicLibrarySearchGenerator::Load(S.ES, Dylib.c_str()); - if (!G) - return G.takeError(); - S.MainJD->addGenerator(std::move(*G)); + if (auto Err = S.loadAndLinkDynamicLibrary(*S.MainJD, Dylib)) + return Err; } return Error::success(); @@ -1067,6 +1067,14 @@ if (!NoProcessSymbols) ExitOnErr(loadProcessSymbols(*this)); + else { + // This symbol is used in testcases. + ExitOnErr(MainJD->define(absoluteSymbols( + {{ES.intern("llvm_jitlink_setTestResultOverride"), + {pointerToJITTargetAddress(llvm_jitlink_setTestResultOverride), + JITSymbolFlags::Exported}}}))); + } + ExitOnErr(loadDylibs(*this)); auto &TT = ES.getExecutorProcessControl().getTargetTriple(); @@ -1092,6 +1100,28 @@ Err = P.takeError(); return; } + } else if (TT.isOSBinFormatCOFF() && !OrcRuntime.empty()) { + auto LoadDynLibrary = [&, this](JITDylib &JD, StringRef DLLName) -> Error { + if (!DLLName.endswith_insensitive(".dll")) + return make_error("DLLName not ending with .dll", + inconvertibleErrorCode()); + return loadAndLinkDynamicLibrary(JD, DLLName); + }; + + if (auto P = COFFPlatform::Create(ES, ObjLayer, *MainJD, OrcRuntime.c_str(), + std::move(LoadDynLibrary))) { + // Set platform early to register jitdylib of dynamic libraries. + auto &CP = **P; + ES.setPlatform(std::move(*P)); + + if (auto E2 = CP.bootstrap(*MainJD)) { + Err = std::move(E2); + return; + } + } else { + Err = P.takeError(); + return; + } } else if (TT.isOSBinFormatELF()) { if (!NoExec) ObjLayer.addPlugin(std::make_unique( @@ -1191,6 +1221,37 @@ PassConfig.PostPrunePasses.push_back(addSelfRelocations); } +Expected Session::getOrLoadDynamicLibrary(StringRef LibPath) { + auto It = DynLibJDs.find(LibPath.str()); + if (It != DynLibJDs.end()) { + return It->second; + } + auto G = EPCDynamicLibrarySearchGenerator::Load(ES, LibPath.data()); + if (!G) + return G.takeError(); + auto JD = &ES.createBareJITDylib(LibPath.str()); + + JD->addGenerator(std::move(*G)); + DynLibJDs.emplace(LibPath.str(), JD); + LLVM_DEBUG({ + dbgs() << "Loaded dynamic library " << LibPath.data() << " for " << LibPath + << "\n"; + }); + return JD; +} + +Error Session::loadAndLinkDynamicLibrary(JITDylib &JD, StringRef LibPath) { + auto DL = getOrLoadDynamicLibrary(LibPath); + if (!DL) + return DL.takeError(); + JD.addToLinkOrder(**DL); + LLVM_DEBUG({ + dbgs() << "Linking dynamic library " << LibPath << " to " << JD.getName() + << "\n"; + }); + return Error::success(); +} + Expected Session::findFileInfo(StringRef FileName) { auto FileInfoItr = FileInfos.find(FileName); if (FileInfoItr == FileInfos.end()) @@ -1762,18 +1823,8 @@ case file_magic::pecoff_executable: case file_magic::elf_shared_object: case file_magic::macho_dynamically_linked_shared_lib: { - // TODO: On first reference to LibPath this should create a JITDylib - // with a generator and add it to JD's links-against list. Subsquent - // references should use the JITDylib created on the first - // reference. - auto G = EPCDynamicLibrarySearchGenerator::Load(S.ES, LibPath.data()); - if (!G) - return G.takeError(); - LLVM_DEBUG({ - dbgs() << "Adding generator for dynamic library " << LibPath.data() - << " to " << JD.getName() << "\n"; - }); - JD.addGenerator(std::move(*G)); + if (auto Err = S.loadAndLinkDynamicLibrary(JD, LibPath.data())) + return Err; break; } case file_magic::archive: