[ORC][ORC_RT][COFF] Support dynamic VC runtime.

[sunho]

Summary

Supports dynamic VC runtime. It implements atexits handling which is required to load msvcrt.lib successfully. (the object file containing atexit symbol somehow resolves to static vc runtim symbols) It also default to dynamic vc runtime which tends to be more robust.

[lhames]

Very cool!

Would it be possible to test this in the ORC runtime? (e.g. with an atexit registration?

[sunho]

Add & fix tests.

[lhames]

Thanks for adding the tests. This is great stuff!

LGTM.

compiler-rt/lib/orc/CMakeLists.txt

 # Build for all components of the ORC runtime support library.
 
 # ORC runtime library common implementation files.
 set(ORC_COMMON_SOURCES
   debug.cpp
   extensible_rtti.cpp
   log_error_to_stderr.cpp
   run_program_wrapper.cpp
   dlfcn_wrapper.cpp
   )
 
 # ORC runtime library implementation files for all ORC architectures.s
 set(ALL_ORC_SOURCES
   ${ORC_COMMON_SOURCES}
   coff_platform.cpp
+  coff_platform.per_jd.cpp
   elfnix_platform.cpp
   macho_ehframe_registration.cpp
   macho_platform.cpp
   )
 
 # Implementation files for all ORC architectures.
 set(ALL_ORC_ASM_SOURCES
   macho_tlv.x86-64.S
if (WIN32)
     set(ORC_IMPL_HEADERS
       ${ORC_COMMON_IMPL_HEADERS}
       coff_platform.h
       )
 
     set(ORC_SOURCES
       ${ORC_COMMON_SOURCES}
       coff_platform.cpp
+      coff_platform.per_jd.cpp
       )
+
+    if (MSVC)
+      set(ORC_CFLAGS "${ORC_CFLAGS} /MD")
+    endif()
   else()
     set(ORC_BUILD_TYPE STATIC)
 
     set(ORC_IMPL_HEADERS
       ${ORC_COMMON_IMPL_HEADERS}
       elfnix_platform.h
       )
     

compiler-rt/lib/orc/coff_platform.cpp

private:
   };
 
   struct JITDylibState {
     std::string Name;
     void *Header = nullptr;
     size_t LinkedAgainstRefCount = 0;
     size_t DlRefCount = 0;
     std::vector<JITDylibState *> Deps;
+    std::vector<void (*)(void)> AtExits;
     XtorSection CInitSection;    // XIA~XIZ
     XtorSection CXXInitSection;  // XCA~XCZ
     XtorSection CPreTermSection; // XPA~XPZ
     XtorSection CTermSection;    // XTA~XTZ
 
     bool referenced() const {
       return LinkedAgainstRefCount != 0 || DlRefCount != 0;
     }
   };
 
 public:
   static void initialize();
   static COFFPlatformRuntimeState &get();
+  static bool isInitialized() { return CPS; }
   static void destroy();
 
   COFFPlatformRuntimeState() = default;
 
   // Delete copy and move constructors.
   COFFPlatformRuntimeState(const COFFPlatformRuntimeState &) = delete;
   COFFPlatformRuntimeState &
   operator=(const COFFPlatformRuntimeState &) = delete;
   COFFPlatformRuntimeState(COFFPlatformRuntimeState &&) = delete;
   COFFPlatformRuntimeState &operator=(COFFPlatformRuntimeState &&) = delete;
 
   const char *dlerror();
   void *dlopen(std::string_view Name, int Mode);
   int dlclose(void *Header);
   void *dlsym(void *Header, std::string_view Symbol);
 
   Error registerJITDylib(std::string Name, void *Header);
   Error deregisterJITDylib(void *Header);
 
+  Error registerAtExit(ExecutorAddr HeaderAddr, void (*AtExit)(void));
+
   Error registerObjectSections(
       ExecutorAddr HeaderAddr,
       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs,
       bool RunInitializers);
   Error deregisterObjectSections(
       ExecutorAddr HeaderAddr,
       std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
 

 }
 
 Error COFFPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) {
   ORC_RT_DEBUG({
     printdbg("COFFPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
              JDS.Name.c_str());
   });
 
-  // FIXME: call atexits.
+  // Run atexits
+  for (auto AtExit : JDS.AtExits)
+    AtExit();
+  JDS.AtExits.clear();
 
   // Run static terminators.
   JDS.CPreTermSection.RunAllNewAndFlush();
   JDS.CTermSection.RunAllNewAndFlush();
 
   // Queue all xtors as new again.
   JDS.CInitSection.Reset();
   JDS.CXXInitSection.Reset();

 Error COFFPlatformRuntimeState::registerObjectSections(
     ExecutorAddr HeaderAddr,
     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs,
     bool RunInitializers) {
   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
   auto I = JDStates.find(HeaderAddr.toPtr<void *>());
   if (I == JDStates.end()) {
     std::ostringstream ErrStream;
-    ErrStream << "Attempted to register unrecognized header "
-              << HeaderAddr.getValue();
+    ErrStream << "Unrecognized header " << HeaderAddr.getValue();
     return make_error<StringError>(ErrStream.str());
   }
   auto &JDState = I->second;
   for (auto &KV : Secs) {
     if (auto Err = registerBlockRange(HeaderAddr, KV.second))
       return Err;
     if (KV.first.empty())
       continue;

 
 Error COFFPlatformRuntimeState::deregisterObjectSections(
     ExecutorAddr HeaderAddr,
     std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
   std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
   auto I = JDStates.find(HeaderAddr.toPtr<void *>());
   if (I == JDStates.end()) {
     std::ostringstream ErrStream;
-    ErrStream << "Attempted to register unrecognized header "
+    ErrStream << "Attempted to deregister unrecognized header "
               << HeaderAddr.getValue();
     return make_error<StringError>(ErrStream.str());
   }
   auto &JDState = I->second;
   for (auto &KV : Secs) {
     if (auto Err = deregisterBlockRange(HeaderAddr, KV.second))
       return Err;
     if (KV.first == ".pdata")

 Error COFFPlatformRuntimeState::deregisterBlockRange(ExecutorAddr HeaderAddr,
                                                      ExecutorAddrRange Range) {
   assert(BlockRanges.count(Range.Start.toPtr<void *>()) &&
          "Block range address not registered");
   BlockRanges.erase(Range.Start.toPtr<void *>());
   return Error::success();
 }
 
+Error COFFPlatformRuntimeState::registerAtExit(ExecutorAddr HeaderAddr,
+                                               void (*AtExit)(void)) {
+  std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
+  auto I = JDStates.find(HeaderAddr.toPtr<void *>());
+  if (I == JDStates.end()) {
+    std::ostringstream ErrStream;
+    ErrStream << "Unrecognized header " << HeaderAddr.getValue();
+    return make_error<StringError>(ErrStream.str());
+  }
+  I->second.AtExits.push_back(AtExit);
+  return Error::success();
+}
+
 void COFFPlatformRuntimeState::initialize() {
   assert(!CPS && "COFFPlatformRuntimeState should be null");
   CPS = new COFFPlatformRuntimeState();
 }
 
 COFFPlatformRuntimeState &COFFPlatformRuntimeState::get() {
   assert(CPS && "COFFPlatformRuntimeState not initialized");
   return *CPS;
const ULONG_PTR parameters[] = {
       reinterpret_cast<ULONG_PTR>(pThrowInfo),
       reinterpret_cast<ULONG_PTR>(BaseAddr),
   };
   RaiseException(EH_EXCEPTION_NUMBER, EXCEPTION_NONCONTINUABLE,
                  _countof(parameters), parameters);
 }
 
 //------------------------------------------------------------------------------
+//                             COFF atexits
+//------------------------------------------------------------------------------
+
+typedef int (*OnExitFunction)(void);
+typedef void (*AtExitFunction)(void);
+
+ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header,
+                                                     OnExitFunction Func) {
+  if (auto Err = COFFPlatformRuntimeState::get().registerAtExit(
+          ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) {
+    consumeError(std::move(Err));
+    return nullptr;
+  }
+  return Func;
+}
+
+ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func) {
+  if (auto Err = COFFPlatformRuntimeState::get().registerAtExit(
+          ExecutorAddr::fromPtr(Header), (void (*)(void))Func)) {
+    consumeError(std::move(Err));
+    return -1;
+  }
+  return 0;
+}
+
+//------------------------------------------------------------------------------
 //                             COFF Run Program
 //------------------------------------------------------------------------------
 
 ORC_RT_INTERFACE int64_t __orc_rt_coff_run_program(const char *JITDylibName,
                                                    const char *EntrySymbolName,
                                                    int argc, char *argv[]) {
   using MainTy = int (*)(int, char *[]);
 

compiler-rt/lib/orc/coff_platform.per_jd.cpp

+//===- coff_platform.per_jd.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code that will be loaded per each JITDylib.
+//
+//===----------------------------------------------------------------------===//
+#include "compiler.h"
+
+ORC_RT_INTERFACE void __orc_rt_coff_per_jd_marker() {}
+
+typedef int (*OnExitFunction)(void);
+typedef void (*AtExitFunction)(void);
+
+extern "C" void *__ImageBase;
+ORC_RT_INTERFACE OnExitFunction __orc_rt_coff_onexit(void *Header,
+                                                     OnExitFunction Func);
+ORC_RT_INTERFACE int __orc_rt_coff_atexit(void *Header, AtExitFunction Func);
+
+ORC_RT_INTERFACE OnExitFunction
+__orc_rt_coff_onexit_per_jd(OnExitFunction Func) {
+  return __orc_rt_coff_onexit(&__ImageBase, Func);
+}
+
+ORC_RT_INTERFACE int __orc_rt_coff_atexit_per_jd(AtExitFunction Func) {
+  return __orc_rt_coff_atexit(&__ImageBase, Func);
+}
 No newline at end of file

compiler-rt/test/orc/TestCases/Windows/x86-64/Inputs/standalone-dylib.c

 #include <stdio.h>
+#include <stdlib.h>
+
+void Dtor() { printf("destructor\n"); }
 
 int Ctor() {
   printf("constructor\n");
+  atexit(Dtor);
   return 0;
 }
 
 #pragma section(".CRT$XIV", long, read)
-__declspec(allocate(".CRT$XIV")) int (*i1)(void) = Ctor;
+__declspec(allocate(".CRT$XIV")) int (*i1)(void) = Ctor;
 No newline at end of file

compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.c

-// RUN: %clang -c -o %t %s
+// RUN: %clang_cl -MD -c -o %t %s
 // RUN: %llvm_jitlink %t 2>&1 | FileCheck %s
 // CHECK: Hello, world!
 
 #include <stdio.h>
 int main(int argc, char *argv[]) {
   printf("Hello, world!\n");
   fflush(stdout);
   return 0;
 }

compiler-rt/test/orc/TestCases/Windows/x86-64/hello-world.cpp

-// RUN: %clangxx -c -o %t %s
+// RUN: %clang_cl -MD -c -o %t %s
 // RUN: %llvm_jitlink %t 2>&1 | FileCheck %s
 // CHECK: Hello, world!
 
 #include <iostream>
 int main(int argc, char *argv[]) {
   std::cout << "Hello, world!" << std::endl;
   return 0;
 }

compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer-three.c

-// RUN: %clang -c -o %t %s
+// RUN: %clang_cl -MD -c -o %t %s
 // RUN: %llvm_jitlink %t 2>&1 | FileCheck %s
 // CHECK: init1
 // CHECK-NEXT: init2
 // CHECK-NEXT: init3
 
 #include <stdio.h>
 
 int init1() {

compiler-rt/test/orc/TestCases/Windows/x86-64/priority-static-initializer.c

-// RUN: %clang -c -o %t %s
+// RUN: %clang_cl -MD -c -o %t %s
 // RUN: %llvm_jitlink %t 2>&1 | FileCheck %s
 // CHECK: init1
 // CHECK-NEXT: init2
 
 #include <stdio.h>
 
 int init1() {
   printf("init1\n");

compiler-rt/test/orc/TestCases/Windows/x86-64/sehframe-handling.cpp

-// RUN: %clangxx -c -o %t %s
+// RUN: %clang_cl -EHsc -MD -c -o %t %s
 // RUN: %llvm_jitlink %t
 
 extern "C" __declspec(dllimport) void llvm_jitlink_setTestResultOverride(
     long Value);
 
 int main(int argc, char *argv[]) {
   llvm_jitlink_setTestResultOverride(1);
   try {
     throw 0;
   } catch (int X) {
     llvm_jitlink_setTestResultOverride(X);
   }
   return 0;
 }

compiler-rt/test/orc/TestCases/Windows/x86-64/static-initializer.cpp

-// RUN: %clangxx -c -o %t %s
+// RUN: %clang_cl -MD -c -o %t %s
 // RUN: %llvm_jitlink %t
 
 extern "C" __declspec(dllimport) void llvm_jitlink_setTestResultOverride(
     long Value);
 
 extern "C" int init() { return -1; }
 extern "C" int init2() { return -2; }
 
 extern "C" int a = init();
 extern "C" int b = 2;
 extern "C" int c = init2();
 
 int main(int argc, char *argv[]) {
   llvm_jitlink_setTestResultOverride(a + b + c + 1);
   return 0;
 }

compiler-rt/test/orc/TestCases/Windows/x86-64/trivial-atexit.c

+// RUN: %clang_cl -MD -c -o %t %s
+// RUN: %llvm_jitlink %t 2>&1 | FileCheck %s
+// CHECK: Entering main
+// CHECK-NEXT: Meow
+
+#include <stdio.h>
+#include <stdlib.h>
+
+void meow() {
+  printf("Meow\n");
+  fflush(stdout);
+}
+
+int main(int argc, char *argv[]) {
+  atexit(meow);
+  printf("Entering main\n");
+  return 0;
+}

compiler-rt/test/orc/TestCases/Windows/x86-64/trivial-jit-dlopen.c

-// XFAIL: *
 // Test that __orc_rt_coff_jit_dlopen and __orc_rt_coff_jit_dlclose work as
 // expected for a straightforward dlopen; dlclose sequence: first the
 // constructors should be run.
 //
-// RUN: %clang -c -o %t.inits.o %p/Inputs/standalone-dylib.c
-// RUN: %clang -c -o %t.test.o %s
+// RUN: %clang_cl -MD -c -o %t.inits.o %p/Inputs/standalone-dylib.c
+// RUN: %clang_cl -MD -c -o %t.test.o %s
 // RUN: %llvm_jitlink \
 // RUN:   -alias dlopen=__orc_rt_coff_jit_dlopen \
 // RUN:   -alias dlclose=__orc_rt_coff_jit_dlclose \
 // RUN:   %t.test.o -jd inits %t.inits.o -lmain | FileCheck %s
 
 // CHECK: entering main
 // CHECK-NEXT: constructor
+// CHECK-NEXT: destructor
 // CHECK-NEXT: leaving main
 
 #include <stdio.h>
-__declspec(dllimport) void *dlopen(const char *path, int mode);
-__declspec(dllimport) int dlclose(void *handle);
+void *dlopen(const char *path, int mode);
+int dlclose(void *handle);
 
 int main(int argc, char *argv[]) {
   printf("entering main\n");
   void *H = dlopen("inits", 0);
   if (!H) {
     printf("failed\n");
     return -1;
   }
   if (dlclose(H) == -1) {
     printf("failed\n");
     return -1;
   }
   printf("leaving main\n");
   return 0;
 }

compiler-rt/test/orc/lit.cfg.py

if config.libunwind_shared:
   shared_libunwind_path = os.path.join(config.libunwind_install_dir, 'libunwind.so')
   config.substitutions.append( ("%shared_libunwind", shared_libunwind_path) )
 
 config.substitutions.append(
     ('%clang ', build_invocation([config.target_cflags])))
 config.substitutions.append(
     ('%clangxx ',
      build_invocation(config.cxx_mode_flags + [config.target_cflags])))
+config.substitutions.append(
+    ('%clang_cl ',
+     build_invocation(['--driver-mode=cl'] + [config.target_cflags])))
 if config.host_os == 'Windows':
   config.substitutions.append(
       ('%llvm_jitlink', (llvm_jitlink + ' -orc-runtime=' +
        orc_rt_path + ' -no-process-syms=true -slab-allocate=64MB')))
 else:
   config.substitutions.append(
       ('%llvm_jitlink', (llvm_jitlink + ' -orc-runtime=' + orc_rt_path)))
 config.substitutions.append(

llvm/include/llvm/ExecutionEngine/Orc/COFFPlatform.h

public:
   using LoadDynamicLibrary =
       unique_function<Error(JITDylib &JD, StringRef DLLFileName)>;
 
   /// Try to create a COFFPlatform instance, adding the ORC runtime to the
   /// given JITDylib.
   static Expected<std::unique_ptr<COFFPlatform>>
   Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
          JITDylib &PlatformJD, const char *OrcRuntimePath,
-         LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath = nullptr,
+         LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime = false,
+         const char *VCRuntimePath = nullptr,
          Optional<SymbolAliasMap> RuntimeAliases = None);
 
   Error bootstrap(JITDylib &PlatformJD);
 
   ExecutionSession &getExecutionSession() const { return ES; }
   ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; }
 
   Error setupJITDylib(JITDylib &JD) override;
struct JDBootstrapState {
     std::string JDName;
     ExecutorAddr HeaderAddr;
     std::list<COFFObjectSectionsMap> ObjectSectionsMaps;
     SmallVector<std::pair<std::string, ExecutorAddr>> Initializers;
   };
 
   static bool supportedTarget(const Triple &TT);
 
-  COFFPlatform(
-      ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
-      JITDylib &PlatformJD,
-      std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
-      LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath, Error &Err);
+  COFFPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
+               JITDylib &PlatformJD, const char *OrcRuntimePath,
+               LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
+               const char *VCRuntimePath, Error &Err);
 
   // Associate COFFPlatform JIT-side runtime support functions with handlers.
   Error associateRuntimeSupportFunctions(JITDylib &PlatformJD);
 
   // Records the addresses of runtime symbols used by the platform.
   Error bootstrapCOFFRuntime(JITDylib &PlatformJD);
 
   // Run a specific void function if it exists.
   Error runSymbolIfExists(JITDylib &PlatformJD, StringRef SymbolName);
 
   // Run collected initializers in boostrap stage.
   Error runBootstrapInitializers(JDBootstrapState &BState);
   Error runBootstrapSubsectionInitializers(JDBootstrapState &BState,
                                            StringRef Start, StringRef End);
 
   // Build dependency graph of a JITDylib
   Expected<JITDylibDepMap> buildJDDepMap(JITDylib &JD);
 
+  Expected<MemoryBufferRef> getPerJDObjectFile();
+
   // Implements rt_pushInitializers by making repeat async lookups for
   // initializer symbols (each lookup may spawn more initializer symbols if
   // it pulls in new materializers, e.g. from objects in a static library).
   void pushInitializersLoop(PushInitializersSendResultFn SendResult,
                             JITDylibSP JD, JITDylibDepMap &JDDepMap);
 
   void rt_pushInitializers(PushInitializersSendResultFn SendResult,
                            ExecutorAddr JDHeaderAddr);
 
   void rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle,
                        StringRef SymbolName);
 
   ExecutionSession &ES;
   ObjectLinkingLayer &ObjLinkingLayer;
 
   LoadDynamicLibrary LoadDynLibrary;
   std::unique_ptr<COFFVCRuntimeBootstrapper> VCRuntimeBootstrap;
+  std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer;
+  std::unique_ptr<object::Archive> OrcRuntimeArchive;
+  bool StaticVCRuntime;
 
   SymbolStringPtr COFFHeaderStartSymbol;
 
   // State of bootstrap in progress
   std::map<JITDylib *, JDBootstrapState> JDBootstrapStates;
   std::atomic<bool> Bootstrapping;
 
   ExecutorAddr orc_rt_coff_platform_bootstrap;

llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp

 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
+#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 
 #include "llvm/Object/COFF.h"
 
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
 
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 
 #define DEBUG_TYPE "orc"

 } // end anonymous namespace
 
 namespace llvm {
 namespace orc {
 
 Expected<std::unique_ptr<COFFPlatform>>
 COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
                      JITDylib &PlatformJD, const char *OrcRuntimePath,
-                     LoadDynamicLibrary LoadDynLibrary,
+                     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
                      const char *VCRuntimePath,
                      Optional<SymbolAliasMap> RuntimeAliases) {
   auto &EPC = ES.getExecutorProcessControl();
 
   // If the target is not supported then bail out immediately.
   if (!supportedTarget(EPC.getTargetTriple()))
     return make_error<StringError>("Unsupported COFFPlatform triple: " +
                                        EPC.getTargetTriple().str(),
if (auto Err = HostFuncJD.define(absoluteSymbols(
           {{ES.intern("__orc_rt_jit_dispatch"),
             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
              JITSymbolFlags::Exported}},
            {ES.intern("__orc_rt_jit_dispatch_ctx"),
             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
              JITSymbolFlags::Exported}}})))
     return std::move(Err);
 
-  // Create a generator for the ORC runtime archive.
-  auto OrcRuntimeArchiveGenerator =
-      StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
-  if (!OrcRuntimeArchiveGenerator)
-    return OrcRuntimeArchiveGenerator.takeError();
   PlatformJD.addToLinkOrder(HostFuncJD);
 
   // Create the instance.
   Error Err = Error::success();
   auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
-      ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
-      std::move(LoadDynLibrary), VCRuntimePath, Err));
+      ES, ObjLinkingLayer, PlatformJD, OrcRuntimePath,
+      std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err));
   if (Err)
     return std::move(Err);
   return std::move(P);
 }
 
+Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
+  auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
+  if (!PerJDObj)
+    return PerJDObj.takeError();
+
+  if (!*PerJDObj)
+    return make_error<StringError>("Could not find per jd object file",
+                                   inconvertibleErrorCode());
+
+  auto Buffer = (*PerJDObj)->getAsBinary();
+  if (!Buffer)
+    return Buffer.takeError();
+
+  return (*Buffer)->getMemoryBufferRef();
+}
+
+static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
+                       ArrayRef<std::pair<const char *, const char *>> AL) {
+  for (auto &KV : AL) {
+    auto AliasName = ES.intern(KV.first);
+    assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
+    Aliases[std::move(AliasName)] = {ES.intern(KV.second),
+                                     JITSymbolFlags::Exported};
+  }
+}
+
 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
   if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
           *this, COFFHeaderStartSymbol)))
     return Err;
 
   if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
     return Err;
 
+  // Define the CXX aliases.
+  SymbolAliasMap CXXAliases;
+  addAliases(ES, CXXAliases, requiredCXXAliases());
+  if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
+    return std::move(Err);
+
+  auto PerJDObj = getPerJDObjectFile();
+  if (!PerJDObj)
+    return PerJDObj.takeError();
+
+  auto I = getObjectFileInterface(ES, *PerJDObj);
+  if (!I)
+    return I.takeError();
+
+  if (auto Err = ObjLinkingLayer.add(
+          JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
+    return Err;
+
   if (!Bootstrapping) {
-    auto ImportedLibs = VCRuntimeBootstrap->loadStaticVCRuntime(JD);
+    auto ImportedLibs = StaticVCRuntime
+                            ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
+                            : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
     if (!ImportedLibs)
       return ImportedLibs.takeError();
     for (auto &Lib : *ImportedLibs)
       if (auto Err = LoadDynLibrary(JD, Lib))
         return Err;
+    if (StaticVCRuntime)
       if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
         return Err;
   }
 
   JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
   return Error::success();
 }
 
 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
   std::lock_guard<std::mutex> Lock(PlatformMutex);
Error COFFPlatform::notifyAdding(ResourceTracker &RT,
   });
   return Error::success();
 }
 
 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
   llvm_unreachable("Not supported yet");
 }
 
-static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
-                       ArrayRef<std::pair<const char *, const char *>> AL) {
-  for (auto &KV : AL) {
-    auto AliasName = ES.intern(KV.first);
-    assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
-    Aliases[std::move(AliasName)] = {ES.intern(KV.second),
-                                     JITSymbolFlags::Exported};
-  }
-}
-
 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
   SymbolAliasMap Aliases;
-  addAliases(ES, Aliases, requiredCXXAliases());
   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
   return Aliases;
 }
 
 ArrayRef<std::pair<const char *, const char *>>
 COFFPlatform::requiredCXXAliases() {
   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
       {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
-  };
+      {"_onexit", "__orc_rt_coff_onexit_per_jd"},
+      {"atexit", "__orc_rt_coff_atexit_per_jd"}};
 
   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
 }
 
 ArrayRef<std::pair<const char *, const char *>>
 COFFPlatform::standardRuntimeUtilityAliases() {
   static const std::pair<const char *, const char *>
       StandardRuntimeUtilityAliases[] = {
bool COFFPlatform::supportedTarget(const Triple &TT) {
   switch (TT.getArch()) {
   case Triple::x86_64:
     return true;
   default:
     return false;
   }
 }
 
-COFFPlatform::COFFPlatform(
-    ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
-    JITDylib &PlatformJD,
-    std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
-    LoadDynamicLibrary LoadDynLibrary, const char *VCRuntimePath, Error &Err)
+COFFPlatform::COFFPlatform(ExecutionSession &ES,
+                           ObjectLinkingLayer &ObjLinkingLayer,
+                           JITDylib &PlatformJD, const char *OrcRuntimePath,
+                           LoadDynamicLibrary LoadDynLibrary,
+                           bool StaticVCRuntime, const char *VCRuntimePath,
+                           Error &Err)
     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
       LoadDynLibrary(std::move(LoadDynLibrary)),
+      StaticVCRuntime(StaticVCRuntime),
       COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
   ErrorAsOutParameter _(&Err);
 
+  // Create a generator for the ORC runtime archive.
+  auto OrcRuntimeArchiveGenerator =
+      StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
+  if (!OrcRuntimeArchiveGenerator) {
+    Err = std::move(OrcRuntimeArchiveGenerator.takeError());
+    return;
+  }
+
+  auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
+  if (!ArchiveBuffer) {
+    Err = createFileError(OrcRuntimePath, ArchiveBuffer.getError());
+    return;
+  }
+  OrcRuntimeArchiveBuffer = std::move(*ArchiveBuffer);
+  OrcRuntimeArchive =
+      std::make_unique<object::Archive>(*OrcRuntimeArchiveBuffer, Err);
+  if (Err)
+    return;
+
   Bootstrapping.store(true);
   ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
 
   // Load vc runtime
   auto VCRT =
       COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
   if (!VCRT) {
     Err = std::move(VCRT.takeError());
     return;
   }
   VCRuntimeBootstrap = std::move(*VCRT);
 
-  for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries())
+  for (auto &Lib : (*OrcRuntimeArchiveGenerator)->getImportedDynamicLibraries())
     DylibsToPreload.insert(Lib);
 
-  auto ImportedLibs = VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD);
+  auto ImportedLibs =
+      StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
+                      : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
   if (!ImportedLibs) {
     Err = ImportedLibs.takeError();
     return;
   }
 
   for (auto &Lib : *ImportedLibs)
     DylibsToPreload.insert(Lib);
 
+  PlatformJD.addGenerator(std::move(*OrcRuntimeArchiveGenerator));
+
   // PlatformJD hasn't been set up by the platform yet (since we're creating
   // the platform now), so set it up.
   if (auto E2 = setupJITDylib(PlatformJD)) {
     Err = std::move(E2);
     return;
   }
-
-  PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
 }
 
 Error COFFPlatform::bootstrap(JITDylib &PlatformJD) {
   for (auto &Lib : DylibsToPreload)
     if (auto Err = LoadDynLibrary(PlatformJD, Lib))
       return Err;
 
+  if (StaticVCRuntime)
     if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD))
       return Err;
 
   // Associate wrapper function tags with JIT-side function implementations.
   if (auto Err = associateRuntimeSupportFunctions(PlatformJD)) {
     return Err;
   }
 
   // Lookup addresses of runtime functions callable by the platform,
   // call the platform bootstrap function to initialize the platform-state
if (auto Err = lookupAndRecordAddrs(
                &orc_rt_coff_deregister_jitdylib},
               {ES.intern("__orc_rt_coff_register_object_sections"),
                &orc_rt_coff_register_object_sections},
               {ES.intern("__orc_rt_coff_deregister_object_sections"),
                &orc_rt_coff_deregister_object_sections},
           }))
     return Err;
 
-  // Run static initializers collected in bootstrap stage.
-  for (auto KV : JDBootstrapStates) {
-    auto &JDBState = KV.second;
-    if (auto Err = runBootstrapInitializers(JDBState))
-      return Err;
-  }
-
   // Call bootstrap functions
   if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
     return Err;
 
   // Do the pending jitdylib registration actions that we couldn't do
   // because orc runtime was not linked fully.
   for (auto KV : JDBootstrapStates) {
     auto &JDBState = KV.second;
     if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
             orc_rt_coff_register_jitdylib, JDBState.JDName,
             JDBState.HeaderAddr))
       return Err;
 
     for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
       if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
                                             SPSCOFFObjectSectionsMap, bool)>(
               orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
               ObjSectionMap, false))
         return Err;
   }
 
+  // Run static initializers collected in bootstrap stage.
+  for (auto KV : JDBootstrapStates) {
+    auto &JDBState = KV.second;
+    if (auto Err = runBootstrapInitializers(JDBState))
+      return Err;
+  }
+
   return Error::success();
 }
 
 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
                                       StringRef SymbolName) {
   ExecutorAddr jit_function;
   auto AfterCLookupErr = lookupAndRecordAddrs(
       ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),

llvm/tools/llvm-jitlink/llvm-jitlink.cpp

else {
     Err = MainJDOrErr.takeError();
     return;
   }
 
   if (!NoProcessSymbols)
     ExitOnErr(loadProcessSymbols(*this));
   else {
     // This symbol is used in testcases.
-    ExitOnErr(MainJD->define(absoluteSymbols(
+    auto &TestResultJD = ES.createBareJITDylib("<TestResultJD>");
+    ExitOnErr(TestResultJD.define(absoluteSymbols(
         {{ES.intern("llvm_jitlink_setTestResultOverride"),
           {pointerToJITTargetAddress(llvm_jitlink_setTestResultOverride),
            JITSymbolFlags::Exported}}})));
+    MainJD->addToLinkOrder(TestResultJD);
   }
 
   ExitOnErr(loadDylibs(*this));
 
   auto &TT = ES.getExecutorProcessControl().getTargetTriple();
 
   if (DebuggerSupport && TT.isOSBinFormatMachO())
     ObjLayer.addPlugin(ExitOnErr(