Use a single context for lto

[• rafael]

Summary

Using multiple context used to be a really big memory saving because we could free memory from each file while the linker proceeded with the the symbol resolution. We are getting lazier about reading data from the bitcode, so I was curious if this was still a good tradeoff.

One thing that is a bit annoying is that we still have to copy the symbol names. The problem is that the names are stored in the Module and get freed when we move the module bits during linking.

Long term I think the solution is to add a symbol table to the bitcode. That way IRObject file will not need to use a Module or a Context and we can drop it while still keeping a StringRef to the names.

This patch might still be an interesting medium term improvement.

When linking llvm-as without debug info this patch is a small speedup:

master: 29.861877513 seconds
patch: 29.814533787 seconds

With debug info the numbers are

master: 34.765181469 seconds
patch: 34.563351584 seconds

The peak memory usage when linking llvm-as with debug info was

master: 599.10MB
patch: 600.13MB

[mehdi_amini]

Using multiple context used to be a really big memory saving because we could free memory from each file while the linker proceeded with the the symbol resolution.

Using multiple LLVMContext is an enabled to parse the bitcode in parallel, don't you care about that?

[pcc]

LGTM with nit.

ELF/InputFiles.h

25

Duplicate #include.

[ruiu]

LGTM with this change.

ELF/Driver.cpp

265–269

I'd move this to initLLVM and swap the order of initLLVM and readConfigs.

[Eugene.Zelenko]

Committed in r267921.

ELF/Driver.h

 
 extern class LinkerDriver *Driver;
 
 class LinkerDriver {
 public:
   void main(ArrayRef<const char *> Args);
   void addFile(StringRef Path);
   void addLibrary(StringRef Name);
+  llvm::LLVMContext Context;
 
 private:
   std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef MB);
   llvm::Optional<MemoryBufferRef> readFile(StringRef Path);
   void readConfigs(llvm::opt::InputArgList &Args);
   void createFiles(llvm::opt::InputArgList &Args);
   template <class ELFT> void link(llvm::opt::InputArgList &Args);
 

ELF/Driver.cpp

void LinkerDriver::main(ArrayRef<const char *> ArgsArr) {
   if (Args.hasArg(OPT_version)) {
     printVersion();
     return;
   }
 
   initLLVM(Args);
   readConfigs(Args);
 
+  // This is a flag to discard all but GlobalValue names.
+  // We want to enable it by default because it saves memory.
+  // Disable it only when a developer option (-save-temps) is given.
+  Context.setDiscardValueNames(!Config->SaveTemps);
+  Context.enableDebugTypeODRUniquing();
+
   if (!Config->Reproduce.empty())
     logCommandline(ArgsArr);
 
   createFiles(Args);
   checkOptions(Args);
   if (HasError)
     return;
 

ELF/InputFiles.h

 
 #include "lld/Core/LLVM.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/IRObjectFile.h"
+#include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/StringSaver.h"
 
 #include <map>
 
 namespace lld {
 namespace elf {
 
 using llvm::object::Archive;

 
 class BitcodeFile : public InputFile {
 public:
   explicit BitcodeFile(MemoryBufferRef M);
   static bool classof(const InputFile *F);
   void parse(llvm::DenseSet<StringRef> &ComdatGroups);
   ArrayRef<SymbolBody *> getSymbols() { return SymbolBodies; }
   static bool shouldSkip(const llvm::object::BasicSymbolRef &Sym);
+  std::unique_ptr<llvm::object::IRObjectFile> Obj;
 
 private:
   std::vector<SymbolBody *> SymbolBodies;
   llvm::BumpPtrAllocator Alloc;
   llvm::StringSaver Saver{Alloc};
   SymbolBody *
   createSymbolBody(const llvm::DenseSet<const llvm::Comdat *> &KeptComdats,
                    const llvm::object::IRObjectFile &Obj,

ELF/InputFiles.cpp

 //===- InputFiles.cpp -----------------------------------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "InputFiles.h"
+#include "Driver.h"
 #include "Error.h"
 #include "InputSection.h"
 #include "Symbols.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/Object/IRObjectFile.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 using namespace llvm::ELF;
 using namespace llvm::object;
 using namespace llvm::sys::fs;
 
 using namespace lld;
bool BitcodeFile::shouldSkip(const BasicSymbolRef &Sym) {
   if (!(Flags & BasicSymbolRef::SF_Global))
     return true;
   if (Flags & BasicSymbolRef::SF_FormatSpecific)
     return true;
   return false;
 }
 
 void BitcodeFile::parse(DenseSet<StringRef> &ComdatGroups) {
-  LLVMContext Context;
-  std::unique_ptr<IRObjectFile> Obj = check(IRObjectFile::create(MB, Context));
+  Obj = check(IRObjectFile::create(MB, Driver->Context));
   const Module &M = Obj->getModule();
 
   DenseSet<const Comdat *> KeptComdats;
   for (const auto &P : M.getComdatSymbolTable()) {
     StringRef N = Saver.save(P.first());
     if (ComdatGroups.insert(N).second)
       KeptComdats.insert(&P.second);
   }

ELF/LTO.h

 //===----------------------------------------------------------------------===//
 
 #ifndef LLD_ELF_LTO_H
 #define LLD_ELF_LTO_H
 
 #include "lld/Core/LLVM.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Linker/IRMover.h"
 
 namespace lld {
 namespace elf {
 
 class BitcodeFile;
 class InputFile;
 
 class BitcodeCompiler {
 public:
   BitcodeCompiler();
   void add(BitcodeFile &F);
   std::vector<std::unique_ptr<InputFile>> compile();
 
 private:
   std::vector<std::unique_ptr<InputFile>> runSplitCodegen(
       const std::function<std::unique_ptr<llvm::TargetMachine>()> &TMFactory);
 
-  llvm::LLVMContext Context;
   std::unique_ptr<llvm::Module> Combined;
   llvm::IRMover Mover;
   std::vector<SmallString<0>> OwningData;
   std::unique_ptr<MemoryBuffer> MB;
   llvm::StringSet<> InternalizedSyms;
   std::string TheTriple;
 };
 }
 }
 
 #endif

ELF/LTO.cpp

 //===- LTO.cpp ------------------------------------------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "LTO.h"
 #include "Config.h"
+#include "Driver.h"
 #include "Error.h"
 #include "InputFiles.h"
 #include "Symbols.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/CodeGen/ParallelCG.h"
static bool shouldInternalize(const SmallPtrSet<GlobalValue *, 8> &Used,
 
   if (Used.count(GV))
     return false;
 
   return !B.Backref->includeInDynsym();
 }
 
 BitcodeCompiler::BitcodeCompiler()
-    : Combined(new llvm::Module("ld-temp.o", Context)), Mover(*Combined) {
-  // This is a flag to discard all but GlobalValue names.
-  // We want to enable it by default because it saves memory.
-  // Disable it only when a developer option (-save-temps) is given.
-  Context.setDiscardValueNames(!Config->SaveTemps);
-
-  Context.enableDebugTypeODRUniquing();
-}
+    : Combined(new llvm::Module("ld-temp.o", Driver->Context)),
+      Mover(*Combined) {}
 
 void BitcodeCompiler::add(BitcodeFile &F) {
-  std::unique_ptr<IRObjectFile> Obj =
-      check(IRObjectFile::create(F.MB, Context));
+  std::unique_ptr<IRObjectFile> Obj = std::move(F.Obj);
   std::vector<GlobalValue *> Keep;
   unsigned BodyIndex = 0;
   ArrayRef<SymbolBody *> Bodies = F.getSymbols();
 
   Module &M = Obj->getModule();
   if (M.getDataLayoutStr().empty())
     fatal("invalid bitcode file: " + F.getName() + " has no datalayout");