Index: include/llvm/Support/thread.h
===================================================================
--- include/llvm/Support/thread.h
+++ include/llvm/Support/thread.h
@@ -57,6 +57,7 @@
   thread(const thread &) = delete;
 
   void join() {}
+  static unsigned hardware_concurrency() { return 1; };
 };
 
 }
Index: test/tools/gold/X86/Inputs/thinlto_linkonceresolution.ll
===================================================================
--- /dev/null
+++ test/tools/gold/X86/Inputs/thinlto_linkonceresolution.ll
@@ -0,0 +1,4 @@
+target triple = "x86_64-unknown-linux-gnu"
+define linkonce_odr hidden void @f() {
+  ret void
+}
Index: test/tools/gold/X86/pr19901_thinlto.ll
===================================================================
--- /dev/null
+++ test/tools/gold/X86/pr19901_thinlto.ll
@@ -0,0 +1,25 @@
+; RUN: llc %s -o %t.o -filetype=obj -relocation-model=pic
+; RUN: llvm-as -function-summary %p/Inputs/pr19901-1.ll -o %t2.o
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     -shared -m elf_x86_64 -o %t.so %t2.o %t.o
+; RUN: llvm-readobj -t %t.so | FileCheck %s
+
+; CHECK:       Symbol {
+; CHECK:         Name: f
+; CHECK-NEXT:    Value:
+; CHECK-NEXT:    Size:
+; CHECK-NEXT:    Binding: Local
+; CHECK-NEXT:    Type: Function
+; CHECK-NEXT:    Other: {{2|0}}
+; CHECK-NEXT:    Section: .text
+; CHECK-NEXT:  }
+
+target triple = "x86_64-unknown-linux-gnu"
+define i32 @g() {
+  call void @f()
+  ret i32 0
+}
+define linkonce_odr hidden void @f() {
+  ret void
+}
Index: test/tools/gold/X86/thinlto.ll
===================================================================
--- test/tools/gold/X86/thinlto.ll
+++ test/tools/gold/X86/thinlto.ll
@@ -4,17 +4,55 @@
 ; RUN: llvm-as %p/Inputs/thinlto.ll -o %t2.o
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=thinlto-index-only \
 ; RUN:    -shared %t.o %t2.o -o %t3
+; RUN: not test -e %t3
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    -shared %t.o %t2.o -o %t4
+; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
 
+; Next generate function summary sections and test gold handling.
 ; RUN: llvm-as -function-summary %s -o %t.o
 ; RUN: llvm-as -function-summary %p/Inputs/thinlto.ll -o %t2.o
 
+; Ensure gold generates an index and not a binary if requested.
 ; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
 ; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=thinlto-index-only \
 ; RUN:    -shared %t.o %t2.o -o %t3
 ; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | FileCheck %s --check-prefix=COMBINED
 ; RUN: not test -e %t3
 
+; Ensure gold generates an index as well as a binary by default in ThinLTO mode.
+; First force single-threaded mode
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=jobs=1 \
+; RUN:    -shared %t.o %t2.o -o %t4
+; RUN: llvm-bcanalyzer -dump %t4.thinlto.bc | FileCheck %s --check-prefix=COMBINED
+; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
+
+; Next force multi-threaded mode
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=jobs=2 \
+; RUN:    -shared %t.o %t2.o -o %t4
+; RUN: llvm-bcanalyzer -dump %t4.thinlto.bc | FileCheck %s --check-prefix=COMBINED
+; RUN: llvm-nm %t4 | FileCheck %s --check-prefix=NM
+
+; Test --plugin-opt=obj-path to ensure unique object files generated.
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:    --plugin-opt=thinlto \
+; RUN:    --plugin-opt=jobs=2 \
+; RUN:    --plugin-opt=obj-path=%t5.o \
+; RUN:    -shared %t.o %t2.o -o %t4
+; RUN: llvm-nm %t5.o0 | FileCheck %s --check-prefix=NM2
+; RUN: llvm-nm %t5.o1 | FileCheck %s --check-prefix=NM2
+
+; NM: T f
+; NM2: T {{f|g}}
+
 ; COMBINED: <MODULE_STRTAB_BLOCK
 ; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
 ; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
@@ -28,7 +66,10 @@
 ; COMBINED-NEXT: <COMBINED_FNENTRY {{.*}} record string = '{{f|g}}'
 ; COMBINED-NEXT: </VALUE_SYMTAB
 
+declare void @g(...)
+
 define void @f() {
 entry:
+  call void (...) @g()
   ret void
 }
Index: test/tools/gold/X86/thinlto_linkonceresolution.ll
===================================================================
--- /dev/null
+++ test/tools/gold/X86/thinlto_linkonceresolution.ll
@@ -0,0 +1,25 @@
+; RUN: llvm-as -function-summary %s -o %t.o
+; RUN: llvm-as -function-summary %p/Inputs/thinlto_linkonceresolution.ll -o %t2.o
+
+; Ensure the plugin ensures that for ThinLTO the prevailing copy of a
+; linkonce symbol is changed to weak to ensure it is not eliminated.
+; Note that gold picks the first copy of f() as the prevailing one,
+; so listing %t2.o first is sufficient to ensure that this copy is
+; preempted. Also, set the import-instr-limit to 0 to prevent f() from
+; being imported from %t2.o which hides the problem.
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold.so \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=-import-instr-limit=0 \
+; RUN:     -o %t3.o %t2.o %t.o
+; RUN: llvm-nm %t3.o | FileCheck %s
+
+; CHECK-NOT: U f
+
+target triple = "x86_64-unknown-linux-gnu"
+define i32 @g() {
+  call void @f()
+  ret i32 0
+}
+define linkonce_odr hidden void @f() {
+  ret void
+}
Index: tools/gold/gold-plugin.cpp
===================================================================
--- tools/gold/gold-plugin.cpp
+++ tools/gold/gold-plugin.cpp
@@ -20,7 +20,6 @@
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/CommandFlags.h"
-#include "llvm/CodeGen/ParallelCG.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticInfo.h"
@@ -30,6 +29,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Linker/IRMover.h"
+#include "llvm/Linker/Linker.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Object/FunctionIndexObjectFile.h"
 #include "llvm/Object/IRObjectFile.h"
@@ -38,11 +38,14 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/thread.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/Utils/GlobalStatus.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
+#include "llvm/Transforms/Utils/SplitModule.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <list>
 #include <plugin-api.h>
@@ -76,17 +79,32 @@
 /// RAII wrapper to manage opening and releasing of a ld_plugin_input_file.
 struct PluginInputFile {
   void *Handle;
-  ld_plugin_input_file File;
+  std::unique_ptr<ld_plugin_input_file> File;
 
   PluginInputFile(void *Handle) : Handle(Handle) {
-    if (get_input_file(Handle, &File) != LDPS_OK)
+    File = llvm::make_unique<ld_plugin_input_file>();
+    if (get_input_file(Handle, File.get()) != LDPS_OK)
       message(LDPL_FATAL, "Failed to get file information");
   }
   ~PluginInputFile() {
-    if (release_input_file(Handle) != LDPS_OK)
-      message(LDPL_FATAL, "Failed to release file information");
+    // File would have been reset to nullptr if we moved this object
+    // to a new owner.
+    if (File)
+      if (release_input_file(Handle) != LDPS_OK)
+        message(LDPL_FATAL, "Failed to release file information");
+  }
+
+  ld_plugin_input_file &file() { return *File; }
+
+  PluginInputFile(PluginInputFile &&RHS) {
+    Handle = RHS.Handle;
+    File = std::move(RHS.File);
+  }
+  PluginInputFile &operator=(PluginInputFile &&RHS) {
+    Handle = RHS.Handle;
+    File = std::move(RHS.File);
+    return *this;
   }
-  ld_plugin_input_file &file() { return File; }
 };
 
 struct ResolutionInfo {
@@ -99,6 +117,40 @@
   unsigned CommonAlign = 0;
   claimed_file *CommonFile = nullptr;
 };
+
+/// Class to own information used by a task or during its cleanup.
+class TaskInfo {
+  /// The output stream the task will codegen into.
+  std::unique_ptr<raw_fd_ostream> OS;
+
+  /// The file name corresponding to the output stream, used during cleanup.
+  std::string Filename;
+
+  /// Flag indicating whether the output file is a temp file that must be
+  /// added to the cleanup list during cleanup.
+  bool TempOutFile;
+
+public:
+  TaskInfo(std::unique_ptr<raw_fd_ostream> OS, std::string Filename,
+           bool TempOutFile)
+      : OS(std::move(OS)), Filename(Filename), TempOutFile(TempOutFile) {}
+
+  /// Performs task related cleanup activities that must be done
+  /// single-threaded (i.e. call backs to gold).
+  void cleanup();
+};
+
+/// Class to own task information for a ThinLTO backend instantiation.
+class ThinLTOTaskInfo : public TaskInfo {
+  /// The input file holding the module bitcode read by the ThinLTO task.
+  PluginInputFile InputFile;
+
+public:
+  ThinLTOTaskInfo(PluginInputFile InputFile, std::unique_ptr<raw_fd_ostream> OS,
+                  std::string Filename, bool TempOutFile)
+      : TaskInfo(std::move(OS), Filename, TempOutFile),
+        InputFile(std::move(InputFile)) {}
+};
 }
 
 static ld_plugin_add_symbols add_symbols = nullptr;
@@ -123,7 +175,11 @@
   static bool generate_api_file = false;
   static OutputType TheOutputType = OT_NORMAL;
   static unsigned OptLevel = 2;
-  static unsigned Parallelism = 1;
+  // Default parallelism of 0 used to indicate that user did not specify.
+  // Actual parallelism default value depends on implementation.
+  // Currently, code generation defaults to no parallelism, whereas
+  // ThinLTO uses the hardware_concurrency as the default.
+  static unsigned Parallelism = 0;
 #ifdef NDEBUG
   static bool DisableVerify = true;
 #else
@@ -137,6 +193,11 @@
   // the information from intermediate files and write a combined
   // global index for the ThinLTO backends.
   static bool thinlto = false;
+  // If false, all ThinLTO backend compilations through code gen are performed
+  // using multiple threads in the gold-plugin, before handing control back to
+  // gold. If true, exit after creating the combined index, the assuming is
+  // that the build system will launch the backend processes.
+  static bool thinlto_index_only = false;
   // Additional options to pass into the code generator.
   // Note: This array will contain all plugin options which are not claimed
   // as plugin exclusive to pass to the code generator.
@@ -168,6 +229,8 @@
       TheOutputType = OT_DISABLE;
     } else if (opt == "thinlto") {
       thinlto = true;
+    } else if (opt == "thinlto-index-only") {
+      thinlto_index_only = true;
     } else if (opt.size() == 2 && opt[0] == 'O') {
       if (opt[1] < '0' || opt[1] > '3')
         message(LDPL_FATAL, "Optimization level must be between 0 and 3");
@@ -431,7 +494,7 @@
 
   // If we are doing ThinLTO compilation, don't need to process the symbols.
   // Later we simply build a combined index file after all files are claimed.
-  if (options::thinlto)
+  if (options::thinlto && options::thinlto_index_only)
     return LDPS_OK;
 
   for (auto &Sym : Obj->symbols()) {
@@ -561,16 +624,22 @@
   Sym.comdat_key = nullptr;
 }
 
-static std::unique_ptr<FunctionInfoIndex>
-getFunctionIndexForFile(claimed_file &F, ld_plugin_input_file &Info) {
-
-  if (get_symbols(F.handle, F.syms.size(), &F.syms[0]) != LDPS_OK)
+/// Helper to get a file's symbols and a view into it via gold callbacks.
+static const void *getSymbolsAndView(claimed_file &F) {
+  if (get_symbols(F.handle, F.syms.size(), F.syms.data()) != LDPS_OK)
     message(LDPL_FATAL, "Failed to get symbol information");
 
   const void *View;
   if (get_view(F.handle, &View) != LDPS_OK)
     message(LDPL_FATAL, "Failed to get a view of file");
 
+  return View;
+}
+
+static std::unique_ptr<FunctionInfoIndex>
+getFunctionIndexForFile(claimed_file &F, ld_plugin_input_file &Info) {
+  const void *View = getSymbolsAndView(F);
+
   MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize),
                             Info.name);
 
@@ -592,19 +661,28 @@
   return Obj.takeIndex();
 }
 
+/// Force the prevailing copy of a linkonce to be kept by converting it to weak.
+/// This is necessary when there are either non-IR uses or we are doing
+/// ThinLTO compilation.
+static void convertLinkOnceToWeak(GlobalValue *GV) {
+  switch (GV->getLinkage()) {
+  default:
+    break;
+  case GlobalValue::LinkOnceAnyLinkage:
+    GV->setLinkage(GlobalValue::WeakAnyLinkage);
+    break;
+  case GlobalValue::LinkOnceODRLinkage:
+    GV->setLinkage(GlobalValue::WeakODRLinkage);
+    break;
+  }
+}
+
 static std::unique_ptr<Module>
-getModuleForFile(LLVMContext &Context, claimed_file &F,
+getModuleForFile(LLVMContext &Context, claimed_file &F, const void *View,
                  ld_plugin_input_file &Info, raw_fd_ostream *ApiFile,
                  StringSet<> &Internalize, StringSet<> &Maybe,
                  std::vector<GlobalValue *> &Keep) {
 
-  if (get_symbols(F.handle, F.syms.size(), F.syms.data()) != LDPS_OK)
-    message(LDPL_FATAL, "Failed to get symbol information");
-
-  const void *View;
-  if (get_view(F.handle, &View) != LDPS_OK)
-    message(LDPL_FATAL, "Failed to get a view of file");
-
   MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize),
                             Info.name);
   ErrorOr<std::unique_ptr<object::IRObjectFile>> ObjOrErr =
@@ -692,6 +770,15 @@
 
     case LDPR_PREVAILING_DEF_IRONLY: {
       Keep.push_back(GV);
+      if (options::thinlto) {
+        // For ThinLTO the IR files are compiled through the backend,
+        // independently, so we need to ensure that the prevailing copy will be
+        // emitted into the object file by making it weak.
+        convertLinkOnceToWeak(GV);
+        // Skip the below handling for internalization, which isn't
+        // performed in ThinLTO mode currently anyway.
+        break;
+      }
       // The IR linker has to be able to map this value to a declaration,
       // so we can only internalize after linking.
       if (!Used.count(GV))
@@ -702,25 +789,25 @@
     case LDPR_PREVAILING_DEF:
       Keep.push_back(GV);
       // There is a non IR use, so we have to force optimizations to keep this.
-      switch (GV->getLinkage()) {
-      default:
-        break;
-      case GlobalValue::LinkOnceAnyLinkage:
-        GV->setLinkage(GlobalValue::WeakAnyLinkage);
-        break;
-      case GlobalValue::LinkOnceODRLinkage:
-        GV->setLinkage(GlobalValue::WeakODRLinkage);
-        break;
-      }
+      convertLinkOnceToWeak(GV);
       break;
 
     case LDPR_PREVAILING_DEF_IRONLY_EXP: {
+      Keep.push_back(GV);
+      if (options::thinlto) {
+        // For ThinLTO the IR files are compiled through the backend,
+        // independently, so we need to ensure that the prevailing copy will be
+        // emitted into the object file by making it weak.
+        convertLinkOnceToWeak(GV);
+        // Skip the below handling for internalization, which isn't
+        // performed in ThinLTO mode currently anyway.
+        break;
+      }
       // We can only check for address uses after we merge the modules. The
       // reason is that this GV might have a copy in another module
       // and in that module the address might be significant, but that
       // copy will be LDPR_PREEMPTED_IR.
       Maybe.insert(GV->getName());
-      Keep.push_back(GV);
       break;
     }
     }
@@ -731,35 +818,82 @@
   return Obj.takeModule();
 }
 
-static void runLTOPasses(Module &M, TargetMachine &TM) {
-  M.setDataLayout(TM.createDataLayout());
-
-  legacy::PassManager passes;
-  passes.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
+void TaskInfo::cleanup() {
+  // Close the output file descriptor before we pass it to gold.
+  OS->close();
 
-  PassManagerBuilder PMB;
-  PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM.getTargetTriple()));
-  PMB.Inliner = createFunctionInliningPass();
-  // Unconditionally verify input since it is not verified before this
-  // point and has unknown origin.
-  PMB.VerifyInput = true;
-  PMB.VerifyOutput = !options::DisableVerify;
-  PMB.LoopVectorize = true;
-  PMB.SLPVectorize = true;
-  PMB.OptLevel = options::OptLevel;
-  PMB.populateLTOPassManager(passes);
-  passes.run(M);
+  if (add_input_file(Filename.c_str()) != LDPS_OK)
+    message(LDPL_FATAL,
+            "Unable to add .o file to the link. File left behind in: %s",
+            Filename.c_str());
+  if (TempOutFile)
+    Cleanup.push_back(Filename.c_str());
 }
 
-static void saveBCFile(StringRef Path, Module &M) {
-  std::error_code EC;
-  raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::F_None);
-  if (EC)
-    message(LDPL_FATAL, "Failed to write the output file.");
-  WriteBitcodeToFile(&M, OS, /* ShouldPreserveUseListOrder */ false);
+namespace {
+/// Class to manage optimization and code generation for a module, possibly
+/// in a thread (ThinLTO or split code generation). In the split code generation
+/// case, a new CodeGen object will be created for each split module part.
+class CodeGen {
+  /// The module (potentially split) for which this will generate code.
+  std::unique_ptr<llvm::Module> M;
+
+  /// The output stream to generate code into.
+  raw_fd_ostream *OS;
+
+  /// The task ID when this was invoked in a thread (ThinLTO or split code gen).
+  int TaskID;
+
+  /// The function index for ThinLTO tasks.
+  const FunctionInfoIndex *CombinedIndex;
+
+  /// The target machine for generating code for this module.
+  std::unique_ptr<TargetMachine> TM;
+
+  /// Filename to use as base when save-temps is enabled, used to get
+  /// a unique and identifiable save-temps output file for each ThinLTO backend.
+  std::string Filename;
+
+public:
+  CodeGen(std::unique_ptr<llvm::Module> M, raw_fd_ostream *OS = nullptr,
+          int TaskID = -1, const FunctionInfoIndex *CombinedIndex = nullptr,
+          std::string Filename = "")
+      : M(std::move(M)), OS(OS), TaskID(TaskID), CombinedIndex(CombinedIndex),
+        Filename(Filename) {
+    assert(options::thinlto == !!CombinedIndex &&
+           "Expected function index iff performing ThinLTO");
+    assert(!(CombinedIndex && TaskID == -1) &&
+           "Expected function index only in a ThinLTO backend thread");
+    initTargetMachine();
+  }
+
+  /// Set the output stream (used in the non-ThinLTO single-threaded case).
+  void setOutputStream(raw_fd_ostream *TheOS) {
+    assert(!OS && "Output stream already set");
+    OS = TheOS;
+  }
+
+  /// Invoke LTO passes and the code generator for the module.
+  void runAll();
+
+  /// Invoke the actual code generation to emit Module's object to file.
+  void runCodegenPasses();
+
+private:
+  /// Create a target machine for the module. Must be unique for each
+  /// module/task.
+  void initTargetMachine();
+
+  /// Run all LTO passes on the module.
+  void runLTOPasses();
+
+  /// Sets up output files necessary to perform optional multi-threaded
+  /// split code generation, and invokes the code generation implementation.
+  void splitCodeGen();
+};
 }
 
-static void codegen(std::unique_ptr<Module> M) {
+void CodeGen::initTargetMachine() {
   const std::string &TripleStr = M->getTargetTriple();
   Triple TheTriple(TripleStr);
 
@@ -768,9 +902,6 @@
   if (!TheTarget)
     message(LDPL_FATAL, "Target not found: %s", ErrMsg.c_str());
 
-  if (unsigned NumOpts = options::extra.size())
-    cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
-
   SubtargetFeatures Features;
   Features.getDefaultSubtargetFeatures(TheTriple);
   for (const std::string &A : MAttrs)
@@ -792,62 +923,270 @@
     CGOptLevel = CodeGenOpt::Aggressive;
     break;
   }
-  std::unique_ptr<TargetMachine> TM(TheTarget->createTargetMachine(
+  TM.reset(TheTarget->createTargetMachine(
       TripleStr, options::mcpu, Features.getString(), Options, RelocationModel,
       CodeModel::Default, CGOptLevel));
+}
+
+void CodeGen::runLTOPasses() {
+  M->setDataLayout(TM->createDataLayout());
+
+  legacy::PassManager passes;
+  passes.add(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
+
+  PassManagerBuilder PMB;
+  PMB.LibraryInfo = new TargetLibraryInfoImpl(Triple(TM->getTargetTriple()));
+  PMB.Inliner = createFunctionInliningPass();
+  // Unconditionally verify input since it is not verified before this
+  // point and has unknown origin.
+  PMB.VerifyInput = true;
+  PMB.VerifyOutput = !options::DisableVerify;
+  PMB.LoopVectorize = true;
+  PMB.SLPVectorize = true;
+  PMB.OptLevel = options::OptLevel;
+  PMB.FunctionIndex = CombinedIndex;
+  PMB.populateLTOPassManager(passes);
+  passes.run(*M);
+}
+
+static void saveBCFile(StringRef Path, Module &M) {
+  std::error_code EC;
+  raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::F_None);
+  if (EC)
+    message(LDPL_FATAL, "Failed to write the output file.");
+  WriteBitcodeToFile(&M, OS, /* ShouldPreserveUseListOrder */ false);
+}
 
-  runLTOPasses(*M, *TM);
+/// Open a file and return the new file descriptor given a base input
+/// file name, a flag indicating whether a temp file should be generated,
+/// and an optional task id. The new filename generated is
+/// returned in \p NewFilename.
+static int openOutputFile(SmallString<128> InFilename, bool TempOutFile,
+                          SmallString<128> &NewFilename, int TaskID = -1) {
+  int FD;
+  if (TempOutFile) {
+    std::error_code EC =
+        sys::fs::createTemporaryFile("lto-llvm", "o", FD, NewFilename);
+    if (EC)
+      message(LDPL_FATAL, "Could not create temporary file: %s",
+              EC.message().c_str());
+  } else {
+    NewFilename = InFilename;
+    if (TaskID >= 0)
+      NewFilename += utostr(TaskID);
+    std::error_code EC =
+        sys::fs::openFileForWrite(NewFilename, FD, sys::fs::F_None);
+    if (EC)
+      message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());
+  }
+  return FD;
+}
 
-  if (options::TheOutputType == options::OT_SAVE_TEMPS)
-    saveBCFile(output_name + ".opt.bc", *M);
+void CodeGen::runCodegenPasses() {
+  assert(OS && "Output stream must be set before emitting to file");
+  legacy::PassManager CodeGenPasses;
+  if (TM->addPassesToEmitFile(CodeGenPasses, *OS,
+                              TargetMachine::CGFT_ObjectFile))
+    report_fatal_error("Failed to setup codegen");
+  CodeGenPasses.run(*M);
+}
 
+/// Perform code generation for the split module part in \p BC.
+static void splitCodegenTask(const SmallVector<char, 0> &BC,
+                             llvm::raw_fd_ostream *OS, unsigned TaskID) {
+  LLVMContext Ctx;
+  ErrorOr<std::unique_ptr<Module>> MOrErr = parseBitcodeFile(
+      MemoryBufferRef(StringRef(BC.data(), BC.size()), "<split-module>"), Ctx);
+  if (!MOrErr)
+    report_fatal_error("Failed to read bitcode");
+
+  // We create a new CodeGen object for each thread, since each needs a
+  // separate output stream and TargetMachine.
+  CodeGen codeGen(std::move(MOrErr.get()), OS, TaskID);
+  codeGen.runCodegenPasses();
+}
+
+void CodeGen::splitCodeGen() {
   SmallString<128> Filename;
+  // Note that openOutputFile will append a unique ID for each task
   if (!options::obj_path.empty())
     Filename = options::obj_path;
   else if (options::TheOutputType == options::OT_SAVE_TEMPS)
     Filename = output_name + ".o";
-
-  std::vector<SmallString<128>> Filenames(options::Parallelism);
   bool TempOutFile = Filename.empty();
-  {
-    // Open a file descriptor for each backend thread. This is done in a block
-    // so that the output file descriptors are closed before gold opens them.
-    std::list<llvm::raw_fd_ostream> OSs;
-    std::vector<llvm::raw_pwrite_stream *> OSPtrs(options::Parallelism);
-    for (unsigned I = 0; I != options::Parallelism; ++I) {
-      int FD;
-      if (TempOutFile) {
-        std::error_code EC =
-            sys::fs::createTemporaryFile("lto-llvm", "o", FD, Filenames[I]);
-        if (EC)
-          message(LDPL_FATAL, "Could not create temporary file: %s",
-                  EC.message().c_str());
-      } else {
-        Filenames[I] = Filename;
-        if (options::Parallelism != 1)
-          Filenames[I] += utostr(I);
-        std::error_code EC =
-            sys::fs::openFileForWrite(Filenames[I], FD, sys::fs::F_None);
-        if (EC)
-          message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());
-      }
-      OSs.emplace_back(FD, true);
-      OSPtrs[I] = &OSs.back();
-    }
 
-    // Run backend threads.
-    splitCodeGen(std::move(M), OSPtrs, options::mcpu, Features.getString(),
-                 Options, RelocationModel, CodeModel::Default, CGOptLevel);
-  }
+  // TODO: Should this use thread::hardware_concurrency() if
+  // -jobs option was not specified? Currently preserve behavior of
+  // default parallelism being 1.
+  unsigned int MaxThreads = options::Parallelism ? options::Parallelism : 1;
 
-  for (auto &Filename : Filenames) {
-    if (add_input_file(Filename.c_str()) != LDPS_OK)
+  // In the single thread case, simply invoke runCodegenPasses and manage
+  // the output stream setup/cleanup here.
+  if (MaxThreads == 1) {
+    SmallString<128> NewFilename;
+    int FD = openOutputFile(Filename, TempOutFile, NewFilename);
+    {
+      raw_fd_ostream OS(FD, true);
+      setOutputStream(&OS);
+      runCodegenPasses();
+    }
+    if (add_input_file(NewFilename.c_str()) != LDPS_OK)
       message(LDPL_FATAL,
               "Unable to add .o file to the link. File left behind in: %s",
               Filename.c_str());
     if (TempOutFile)
-      Cleanup.push_back(Filename.c_str());
+      Cleanup.push_back(NewFilename.c_str());
+    return;
   }
+
+  std::list<SmallVector<char, 0>> BCs;
+  SplitModule(std::move(M), MaxThreads, [&](std::unique_ptr<Module> MPart) {
+    // We want to clone the module in a new context to multi-thread the codegen.
+    // We do it by serializing partition modules to bitcode (while still on the
+    // main thread, in order to avoid data races) and later spinning up new
+    // threads which deserialize the partitions into separate contexts.
+    BCs.emplace_back();
+    raw_svector_ostream BCOS(BCs.back());
+    WriteBitcodeToFile(MPart.get(), BCOS);
+  });
+
+  unsigned TaskCount = 0;
+  std::vector<std::unique_ptr<TaskInfo>> Tasks;
+
+  ThreadPool CodegenThreadPool(MaxThreads);
+  for (auto &BC : BCs) {
+    SmallString<128> NewFilename;
+    int FD = openOutputFile(Filename, TempOutFile, NewFilename, TaskCount++);
+    std::unique_ptr<raw_fd_ostream> OS =
+        llvm::make_unique<raw_fd_ostream>(FD, true);
+
+    // Enqueue the task
+    CodegenThreadPool.async(splitCodegenTask, std::ref(BC), OS.get(),
+                            TaskCount);
+
+    // Record the information needed by the task or during its cleanup
+    // to a TaskInfo instance. For information needed by the task
+    // the unique_ptr ownership is transferred to the TaskInfo.
+    Tasks.emplace_back(
+        new TaskInfo(std::move(OS), NewFilename.c_str(), TempOutFile));
+  }
+
+  CodegenThreadPool.wait();
+
+  for (auto &Task : Tasks)
+    Task->cleanup();
+}
+
+void CodeGen::runAll() {
+  runLTOPasses();
+
+  if (options::TheOutputType == options::OT_SAVE_TEMPS) {
+    std::string OptFilename = output_name;
+    // If the CodeGen client provided a filename, use it.
+    if (!Filename.empty())
+      OptFilename = Filename;
+    // Otherwise use the output_name from gold, and optionally append the
+    // TaskID to distinguish multiple output files in the splitCodeGen case.
+    else if (TaskID >= 0)
+      OptFilename += "." + utostr(TaskID);
+    saveBCFile(OptFilename + ".opt.bc", *M);
+  }
+
+  // If we are already in a thread (i.e. ThinLTO), just perform
+  // codegen passes directly.
+  if (TaskID >= 0)
+    runCodegenPasses();
+  // Otherwise attempt split code gen.
+  else
+    splitCodeGen();
+}
+
+/// Perform the ThinLTO backend on a single module, invoking the LTO and codegen
+/// pipelines.
+static void thinLTOBackendTask(claimed_file &F, const void *View,
+                               ld_plugin_input_file &File,
+                               raw_fd_ostream *ApiFile,
+                               const FunctionInfoIndex &CombinedIndex,
+                               raw_fd_ostream *OS, unsigned TaskID) {
+  // Need to use a separate context for each thread
+  LLVMContext Context;
+  Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true);
+
+  StringSet<> Dummy;
+  std::vector<GlobalValue *> Keep;
+  std::unique_ptr<Module> M =
+      getModuleForFile(Context, F, View, File, ApiFile, Dummy, Dummy, Keep);
+  if (!options::triple.empty())
+    M->setTargetTriple(options::triple.c_str());
+  else if (M->getTargetTriple().empty()) {
+    M->setTargetTriple(sys::getDefaultTargetTriple());
+  }
+
+  std::unique_ptr<llvm::Module> NewModule(
+      new llvm::Module(M->getModuleIdentifier(), M->getContext()));
+  IRMover L(*NewModule.get());
+  if (L.move(*M, Keep, [](GlobalValue &, IRMover::ValueAdder) {}))
+    message(LDPL_FATAL, "Failed to rename module for ThinLTO");
+  if (renameModuleForThinLTO(*NewModule, &CombinedIndex))
+    message(LDPL_FATAL, "Failed to rename module for ThinLTO");
+
+  CodeGen codeGen(std::move(NewModule), OS, TaskID, &CombinedIndex);
+  codeGen.runAll();
+}
+
+/// Launch each module's backend pipeline in a separate task in a thread pool.
+static void thinLTOBackends(raw_fd_ostream *ApiFile,
+                            const FunctionInfoIndex &CombinedIndex) {
+  unsigned TaskCount = 0;
+  std::vector<std::unique_ptr<ThinLTOTaskInfo>> Tasks;
+  unsigned int MaxThreads = options::Parallelism
+                                ? options::Parallelism
+                                : thread::hardware_concurrency();
+
+  ThreadPool ThinLTOThreadPool(MaxThreads);
+  for (claimed_file &F : Modules) {
+    // Do all the gold callbacks in the main thread, since gold is not thread
+    // safe by default.
+    PluginInputFile InputFile(F.handle);
+    const void *View = getSymbolsAndView(F);
+
+    SmallString<128> Filename;
+    if (!options::obj_path.empty())
+      // Note that openOutputFile will append a unique ID for each task
+      Filename = options::obj_path;
+    else if (options::TheOutputType == options::OT_SAVE_TEMPS) {
+      // Use the input file name so that we get a unique and identifiable
+      // output file for each ThinLTO backend thread.
+      Filename = InputFile.file().name;
+      Filename += ".thinlto.o";
+    }
+    bool TempOutFile = Filename.empty();
+
+    SmallString<128> NewFilename;
+    int FD = openOutputFile(Filename, TempOutFile, NewFilename,
+                            // Only append the TaskID if we will use the
+                            // non-unique obj_path.
+                            !options::obj_path.empty() ? TaskCount : -1);
+    TaskCount++;
+    std::unique_ptr<raw_fd_ostream> OS =
+        llvm::make_unique<raw_fd_ostream>(FD, true);
+
+    // Enqueue the task
+    ThinLTOThreadPool.async(thinLTOBackendTask, std::ref(F), View,
+                            std::ref(InputFile.file()), ApiFile,
+                            std::ref(CombinedIndex), OS.get(), TaskCount);
+
+    // Record the information needed by the task or during its cleanup
+    // to a ThinLTOTaskInfo instance. For information needed by the task
+    // the unique_ptr ownership is transferred to the ThinLTOTaskInfo.
+    Tasks.emplace_back(new ThinLTOTaskInfo(std::move(InputFile), std::move(OS),
+                                           NewFilename.c_str(), TempOutFile));
+  }
+
+  ThinLTOThreadPool.wait();
+
+  for (auto &Task : Tasks)
+    Task->cleanup();
 }
 
 /// gold informs us that all symbols have been read. At this point, we use
@@ -857,6 +1196,9 @@
   if (Modules.empty())
     return LDPS_OK;
 
+  if (unsigned NumOpts = options::extra.size())
+    cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
+
   // If we are doing ThinLTO compilation, simply build the combined
   // function index/summary and emit it. We don't need to parse the modules
   // and link them in this case.
@@ -883,8 +1225,13 @@
     WriteFunctionSummaryToFile(CombinedIndex, OS);
     OS.close();
 
-    cleanup_hook();
-    exit(0);
+    if (options::thinlto_index_only) {
+      cleanup_hook();
+      exit(0);
+    }
+
+    thinLTOBackends(ApiFile, CombinedIndex);
+    return LDPS_OK;
   }
 
   LLVMContext Context;
@@ -899,9 +1246,10 @@
   StringSet<> Maybe;
   for (claimed_file &F : Modules) {
     PluginInputFile InputFile(F.handle);
+    const void *View = getSymbolsAndView(F);
     std::vector<GlobalValue *> Keep;
     std::unique_ptr<Module> M = getModuleForFile(
-        Context, F, InputFile.file(), ApiFile, Internalize, Maybe, Keep);
+        Context, F, View, InputFile.file(), ApiFile, Internalize, Maybe, Keep);
     if (!options::triple.empty())
       M->setTargetTriple(options::triple.c_str());
     else if (M->getTargetTriple().empty())
@@ -940,7 +1288,8 @@
       return LDPS_OK;
   }
 
-  codegen(std::move(Combined));
+  CodeGen codeGen(std::move(Combined));
+  codeGen.runAll();
 
   if (!options::extra_library_path.empty() &&
       set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK)