diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -68,6 +68,17 @@
   lld::stdoutOS = &stdoutOS;
   lld::stderrOS = &stderrOS;
 
+  errorHandler().cleanupCallback = []() {
+    freeArena();
+    ObjFile::instances.clear();
+    PDBInputFile::instances.clear();
+    ImportFile::instances.clear();
+    BitcodeFile::instances.clear();
+    memset(MergeChunk::instances, 0, sizeof(MergeChunk::instances));
+    TpiSource::clear();
+    OutputSection::clear();
+  };
+
   errorHandler().logName = args::getFilenameWithoutExe(args[0]);
   errorHandler().errorLimitExceededMsg =
       "too many errors emitted, stopping now"
@@ -85,13 +96,6 @@
   if (canExitEarly)
     exitLld(errorCount() ? 1 : 0);
 
-  freeArena();
-  ObjFile::instances.clear();
-  ImportFile::instances.clear();
-  BitcodeFile::instances.clear();
-  memset(MergeChunk::instances, 0, sizeof(MergeChunk::instances));
-  TpiSource::clear();
-
   return !errorCount();
 }
 
@@ -1204,6 +1208,7 @@
   v.push_back("lld-link (LLVM option parsing)");
   for (auto *arg : args.filtered(OPT_mllvm))
     v.push_back(arg->getValue());
+  cl::ResetAllOptionOccurrences();
   cl::ParseCommandLineOptions(v.size(), v.data());
 
   // Handle /errorlimit early, because error() depends on it.
diff --git a/lld/COFF/Writer.h b/lld/COFF/Writer.h
--- a/lld/COFF/Writer.h
+++ b/lld/COFF/Writer.h
@@ -50,6 +50,9 @@
   void writeHeaderTo(uint8_t *buf);
   void addContributingPartialSection(PartialSection *sec);
 
+  // Clear the output sections static container.
+  static void clear();
+
   // Returns the size of this section in an executable memory image.
   // This may be smaller than the raw size (the raw size is multiple
   // of disk sector size, so there may be padding at end), or may be
diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -88,6 +88,8 @@
   return osidx == 0 ? nullptr : outputSections[osidx - 1];
 }
 
+void OutputSection::clear() { outputSections.clear(); }
+
 namespace {
 
 class DebugDirectoryChunk : public NonSectionChunk {
diff --git a/lld/Common/ErrorHandler.cpp b/lld/Common/ErrorHandler.cpp
--- a/lld/Common/ErrorHandler.cpp
+++ b/lld/Common/ErrorHandler.cpp
@@ -13,15 +13,13 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/raw_ostream.h"
 #include <mutex>
 #include <regex>
 
-#if !defined(_MSC_VER) && !defined(__MINGW32__)
-#include <unistd.h>
-#endif
-
 using namespace llvm;
 using namespace lld;
 
@@ -43,14 +41,26 @@
 raw_ostream *lld::stdoutOS;
 raw_ostream *lld::stderrOS;
 
-raw_ostream &lld::outs() { return stdoutOS ? *stdoutOS : llvm::outs(); }
-raw_ostream &lld::errs() { return stderrOS ? *stderrOS : llvm::errs(); }
-
 ErrorHandler &lld::errorHandler() {
   static ErrorHandler handler;
   return handler;
 }
 
+raw_ostream &lld::outs() {
+  if (errorHandler().disableOutput) {
+    static raw_null_ostream discard;
+    return discard;
+  }
+  return stdoutOS ? *stdoutOS : llvm::outs();
+}
+raw_ostream &lld::errs() {
+  if (errorHandler().disableOutput) {
+    static raw_null_ostream discard;
+    return discard;
+  }
+  return stderrOS ? *stderrOS : llvm::errs();
+}
+
 void lld::exitLld(int val) {
   // Delete any temporary file, while keeping the memory mapping open.
   if (errorHandler().outputBuffer)
@@ -60,14 +70,15 @@
   // In an LTO build, allows us to get the output of -time-passes.
   // Ensures that the thread pool for the parallel algorithms is stopped to
   // avoid intermittent crashes on Windows when exiting.
-  llvm_shutdown();
+  if (!CrashRecoveryContext::GetCurrent())
+    llvm_shutdown();
 
   {
     std::lock_guard<std::mutex> lock(mu);
     lld::outs().flush();
     lld::errs().flush();
   }
-  _exit(val);
+  llvm::sys::Process::Exit(val);
 }
 
 void lld::diagnosticHandler(const DiagnosticInfo &di) {
@@ -155,11 +166,15 @@
 void ErrorHandler::log(const Twine &msg) {
   if (!verbose)
     return;
+  if (disableOutput)
+    return;
   std::lock_guard<std::mutex> lock(mu);
   lld::errs() << logName << ": " << msg << "\n";
 }
 
 void ErrorHandler::message(const Twine &msg) {
+  if (disableOutput)
+    return;
   std::lock_guard<std::mutex> lock(mu);
   lld::outs() << msg << "\n";
   lld::outs().flush();
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -80,6 +80,8 @@
   lld::stdoutOS = &stdoutOS;
   lld::stderrOS = &stderrOS;
 
+  errorHandler().cleanupCallback = []() { freeArena(); };
+
   errorHandler().logName = args::getFilenameWithoutExe(args[0]);
   errorHandler().errorLimitExceededMsg =
       "too many errors emitted, stopping now (use "
@@ -119,7 +121,6 @@
   if (canExitEarly)
     exitLld(errorCount() ? 1 : 0);
 
-  freeArena();
   return !errorCount();
 }
 
@@ -887,6 +888,7 @@
   raw_string_ostream os(err);
 
   const char *argv[] = {config->progName.data(), opt.data()};
+  cl::ResetAllOptionOccurrences();
   if (cl::ParseCommandLineOptions(2, argv, "", &os))
     return;
   os.flush();
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -478,6 +478,8 @@
   stderrOS.enable_colors(stderrOS.has_colors());
   // TODO: Set up error handler properly, e.g. the errorLimitExceededMsg
 
+  errorHandler().cleanupCallback = []() { freeArena(); };
+
   MachOOptTable parser;
   opt::InputArgList args = parser.parse(argsArr.slice(1));
 
@@ -630,6 +632,5 @@
   if (canExitEarly)
     exitLld(errorCount() ? 1 : 0);
 
-  freeArena();
   return !errorCount();
 }
diff --git a/lld/include/lld/Common/ErrorHandler.h b/lld/include/lld/Common/ErrorHandler.h
--- a/lld/include/lld/Common/ErrorHandler.h
+++ b/lld/include/lld/Common/ErrorHandler.h
@@ -99,6 +99,8 @@
   bool fatalWarnings = false;
   bool verbose = false;
   bool vsDiagnostics = false;
+  bool disableOutput = false;
+  std::function<void()> cleanupCallback;
 
   void error(const Twine &msg);
   LLVM_ATTRIBUTE_NORETURN void fatal(const Twine &msg);
@@ -106,6 +108,12 @@
   void message(const Twine &msg);
   void warn(const Twine &msg);
 
+  void reset() {
+    if (cleanupCallback)
+      cleanupCallback();
+    *this = ErrorHandler();
+  }
+
   std::unique_ptr<llvm::FileOutputBuffer> outputBuffer;
 
 private:
diff --git a/lld/lib/Driver/DarwinLdDriver.cpp b/lld/lib/Driver/DarwinLdDriver.cpp
--- a/lld/lib/Driver/DarwinLdDriver.cpp
+++ b/lld/lib/Driver/DarwinLdDriver.cpp
@@ -307,6 +307,7 @@
     for (unsigned i = 0; i != numArgs; ++i)
       args[i + 1] = ctx.llvmOptions()[i];
     args[numArgs + 1] = nullptr;
+    llvm::cl::ResetAllOptionOccurrences();
     llvm::cl::ParseCommandLineOptions(numArgs + 1, args);
   }
 }
diff --git a/lld/test/COFF/dll.test b/lld/test/COFF/dll.test
--- a/lld/test/COFF/dll.test
+++ b/lld/test/COFF/dll.test
@@ -13,10 +13,10 @@
 EXPORT-NEXT:       3   0x1010  exportfn3
 EXPORT-NEXT:       4   0x1010  mangled
 
-# RUN: yaml2obj < %p/Inputs/export2.yaml > %t5.obj
-# RUN: rm -f %t5.lib
-# RUN: llvm-ar cru %t5.lib %t5.obj
-# RUN: lld-link /out:%t5.dll /dll %t.obj %t5.lib /export:mangled2
+# RUN: yaml2obj < %p/Inputs/export2.yaml > %t4.obj
+# RUN: rm -f %t4.lib
+# RUN: llvm-ar cru %t4.lib %t4.obj
+# RUN: lld-link /out:%t5.dll /dll %t.obj %t4.lib /export:mangled2
 # RUN: llvm-objdump -p %t5.dll | FileCheck --check-prefix=EXPORT2 %s
 
 EXPORT2:      Export Table:
diff --git a/lld/test/COFF/guardcf-lto.ll b/lld/test/COFF/guardcf-lto.ll
--- a/lld/test/COFF/guardcf-lto.ll
+++ b/lld/test/COFF/guardcf-lto.ll
@@ -8,8 +8,8 @@
 ; RUN: llvm-mc -triple x86_64-windows-msvc -filetype=obj %S/Inputs/loadconfig-cfg-x64.s -o %t.ldcfg.obj
 
 ; RUN: llvm-as %s -o %t.bc
-; RUN: lld-link -entry:main -guard:cf -dll %t.bc %t.lib %t.ldcfg.obj -out:%t.dll
-; RUN: llvm-readobj --coff-load-config %t.dll | FileCheck %s
+; RUN: lld-link -entry:main -guard:cf -dll %t.bc %t.lib %t.ldcfg.obj -out:%t2.dll
+; RUN: llvm-readobj --coff-load-config %t2.dll | FileCheck %s
 
 ; There must be *two* entries in the table: DLL entry point, and my_handler.
 
diff --git a/lld/test/COFF/lit.local.cfg b/lld/test/COFF/lit.local.cfg
new file mode 100644
--- /dev/null
+++ b/lld/test/COFF/lit.local.cfg
@@ -0,0 +1 @@
+config.environment['LLD_IN_TEST'] = '2'
diff --git a/lld/test/ELF/invalid-cie-reference.s b/lld/test/ELF/invalid-cie-reference.s
--- a/lld/test/ELF/invalid-cie-reference.s
+++ b/lld/test/ELF/invalid-cie-reference.s
@@ -1,7 +1,9 @@
 // REQUIRES: x86
 
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t
-// RUN: not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
+// FIXME: Disabling LLD_IN_TEST here because otherwise the following RUN line
+// fails on Windows CRT with _ITERATOR_DEBUG_LEVEL=2, that is Debug builds.
+// RUN: env LLD_IN_TEST=0 not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
 
         .section .eh_frame,"a",@unwind
         .long 0x14
diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp
--- a/lld/tools/lld/lld.cpp
+++ b/lld/tools/lld/lld.cpp
@@ -26,6 +26,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lld/Common/Driver.h"
+#include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -33,12 +34,17 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PluginLoader.h"
 #include <cstdlib>
 
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h> // for _exit
+#endif
+
 using namespace lld;
 using namespace llvm;
 using namespace llvm::sys;
@@ -128,36 +134,99 @@
   return parseProgname(arg0);
 }
 
-// If this function returns true, lld calls _exit() so that it quickly
-// exits without invoking destructors of globally allocated objects.
-//
-// We don't want to do that if we are running tests though, because
-// doing that breaks leak sanitizer. So, lit sets this environment variable,
-// and we use it to detect whether we are running tests or not.
-static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; }
-
 /// Universal linker main(). This linker emulates the gnu, darwin, or
 /// windows linker based on the argv[0] or -flavor option.
-int main(int argc, const char **argv) {
-  InitLLVM x(argc, argv);
-
+int lldMain(int argc, const char **argv, bool exitEarly = true) {
   std::vector<const char *> args(argv, argv + argc);
   switch (parseFlavor(args)) {
   case Gnu:
     if (isPETarget(args))
-      return !mingw::link(args, canExitEarly(), llvm::outs(), llvm::errs());
-    return !elf::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+      return !mingw::link(args, exitEarly, llvm::outs(), llvm::errs());
+    return !elf::link(args, exitEarly, llvm::outs(), llvm::errs());
   case WinLink:
-    return !coff::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+    return !coff::link(args, exitEarly, llvm::outs(), llvm::errs());
   case Darwin:
-    return !mach_o::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+    return !mach_o::link(args, exitEarly, llvm::outs(), llvm::errs());
   case DarwinNew:
-    return !macho::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+    return !macho::link(args, exitEarly, llvm::outs(), llvm::errs());
   case Wasm:
-    return !wasm::link(args, canExitEarly(), llvm::outs(), llvm::errs());
+    return !lld::wasm::link(args, exitEarly, llvm::outs(), llvm::errs());
   default:
     die("lld is a generic driver.\n"
         "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld"
         " (WebAssembly) instead");
   }
 }
+
+struct SafeReturn {
+  int ret;
+  bool canRunAgain;
+};
+
+// Similar to lldMain except that exceptions are caught.
+SafeReturn safeLldMain(int argc, const char **argv) {
+  int r = 0;
+  {
+    // The crash recovery is here only to be able to recover from arbitrary
+    // control flow when fatal() is called (through setjmp/longjmp or
+    // __try/__except).
+    llvm::CrashRecoveryContext crc;
+    if (!crc.RunSafely([&]() { r = lldMain(argc, argv, /*exitEarly=*/false); }))
+      r = crc.RetCode;
+  }
+
+  // Cleanup memory and reset everything back in pristine condition. This path
+  // is only taken when LLD is in test, or when it is used as a library.
+  llvm::CrashRecoveryContext crc;
+  if (!crc.RunSafely([&]() { errorHandler().reset(); })) {
+    // The memory is corrupted beyond any possible recovery.
+    return {r, /*canRunAgain=*/false};
+  }
+  return {r, /*canRunAgain=*/true};
+}
+
+// When in lit tests, tells how many times the LLD tool should re-execute the
+// main loop with the same inputs. When not in test, returns a value of 0 which
+// signifies that LLD shall not release any memory after execution, to speed up
+// process destruction.
+static unsigned inTestVerbosity() {
+  unsigned v = 0;
+  StringRef(getenv("LLD_IN_TEST")).getAsInteger(10, v);
+  return v;
+}
+
+int main(int argc, const char **argv) {
+  InitLLVM x(argc, argv);
+
+  if (!inTestVerbosity())
+    return lldMain(argc, argv);
+
+  Optional<int> mainRet;
+  CrashRecoveryContext::Enable();
+
+  for (unsigned i = inTestVerbosity(); i > 0; --i) {
+    // Disable stdout/stderr for all iterations but the last one.
+    if (i != 1)
+      errorHandler().disableOutput = true;
+
+    // Execute one iteration.
+    auto r = safeLldMain(argc, argv);
+    if (!r.canRunAgain)
+      _exit(r.ret); // Exit now, can't re-execute again.
+
+    if (!mainRet) {
+      mainRet = r.ret;
+    } else if (r.ret != *mainRet) {
+      // Exit now, to fail the tests if the result is different between runs.
+      return r.ret;
+    }
+  }
+#if LLVM_ON_UNIX
+  // Re-throw the signal so it can be caught by WIFSIGNALED in
+  // llvm/lib/Support/Unix/Program.inc. This is required to correctly handle
+  // usages of `not --crash`.
+  if (*mainRet < 0)
+    LLVM_BUILTIN_TRAP;
+#endif
+  return *mainRet;
+}
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -85,6 +85,8 @@
   lld::stdoutOS = &stdoutOS;
   lld::stderrOS = &stderrOS;
 
+  errorHandler().cleanupCallback = []() { freeArena(); };
+
   errorHandler().logName = args::getFilenameWithoutExe(args[0]);
   errorHandler().errorLimitExceededMsg =
       "too many errors emitted, stopping now (use "
@@ -103,7 +105,6 @@
   if (canExitEarly)
     exitLld(errorCount() ? 1 : 0);
 
-  freeArena();
   return !errorCount();
 }
 
@@ -772,6 +773,7 @@
   v.push_back("wasm-ld (LLVM option parsing)");
   for (auto *arg : args.filtered(OPT_mllvm))
     v.push_back(arg->getValue());
+  cl::ResetAllOptionOccurrences();
   cl::ParseCommandLineOptions(v.size(), v.data());
 
   errorHandler().errorLimit = args::getInteger(args, OPT_error_limit, 20);
diff --git a/llvm/include/llvm/Support/CrashRecoveryContext.h b/llvm/include/llvm/Support/CrashRecoveryContext.h
--- a/llvm/include/llvm/Support/CrashRecoveryContext.h
+++ b/llvm/include/llvm/Support/CrashRecoveryContext.h
@@ -44,11 +44,11 @@
 /// executed in any case, whether crash occurs or not. These actions may be used
 /// to reclaim resources in the case of crash.
 class CrashRecoveryContext {
-  void *Impl;
-  CrashRecoveryContextCleanup *head;
+  void *Impl = nullptr;
+  CrashRecoveryContextCleanup *head = nullptr;
 
 public:
-  CrashRecoveryContext() : Impl(nullptr), head(nullptr) {}
+  CrashRecoveryContext();
   ~CrashRecoveryContext();
 
   /// Register cleanup handler, which is used when the recovery context is
diff --git a/llvm/lib/Support/CrashRecoveryContext.cpp b/llvm/lib/Support/CrashRecoveryContext.cpp
--- a/llvm/lib/Support/CrashRecoveryContext.cpp
+++ b/llvm/lib/Support/CrashRecoveryContext.cpp
@@ -97,6 +97,13 @@
 
 CrashRecoveryContextCleanup::~CrashRecoveryContextCleanup() {}
 
+CrashRecoveryContext::CrashRecoveryContext() {
+  // On Windows, if abort() was previously triggered (and caught by a previous
+  // CrashRecoveryContext) the Windows CRT removes our installed signal handler,
+  // so we need to install it again.
+  sys::DisableSystemDialogsOnCrash();
+}
+
 CrashRecoveryContext::~CrashRecoveryContext() {
   // Reclaim registered resources.
   CrashRecoveryContextCleanup *i = head;