[lld] Move Threads to Common

[inglorion]

Summary

This will allow using the functionality from other linkers. It is also
a prerequisite for sharing the error logging code.

[ruiu]

LGTM

lld/trunk/Common/CMakeLists.txt

 if(NOT LLD_BUILT_STANDALONE)
   set(tablegen_deps intrinsics_gen)
 endif()
 
 add_lld_library(lldCommon
   Reproduce.cpp
   TargetOptionsCommandFlags.cpp
+  Threads.cpp
   Version.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLD_INCLUDE_DIR}/lld/Common
 
   LINK_COMPONENTS
   Codegen
   MC
   Option
   Support
   Target
 
   DEPENDS
   ${tablegen_deps}
   )

lld/trunk/Common/Threads.cpp

+//===- Threads.cpp --------------------------------------------------------===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/Threads.h"
+#include <thread>
+
+static std::vector<std::thread> Threads;
+
+bool lld::ThreadsEnabled = true;
+
+// Runs a given function in a new thread.
+void lld::runBackground(std::function<void()> Fn) {
+  Threads.emplace_back(Fn);
+}
+
+// Wait for all threads spawned for runBackground() to finish.
+//
+// You need to call this function from the main thread before exiting
+// because it is not defined what will happen to non-main threads when
+// the main thread exits.
+void lld::waitForBackgroundThreads() {
+  for (std::thread &T : Threads)
+    if (T.joinable())
+      T.join();
+}

lld/trunk/ELF/CMakeLists.txt

add_lld_library(lldELF
   Relocations.cpp
   ScriptLexer.cpp
   ScriptParser.cpp
   Strings.cpp
   SymbolTable.cpp
   Symbols.cpp
   SyntheticSections.cpp
   Target.cpp
-  Threads.cpp
   Thunks.cpp
   Writer.cpp
 
   LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   BinaryFormat
   Core
   DebugInfoDWARF

lld/trunk/ELF/Config.h

struct Configuration {
   bool PrintGcSections;
   bool Relocatable;
   bool SaveTemps;
   bool SingleRoRx;
   bool Shared;
   bool Static = false;
   bool SysvHash = false;
   bool Target1Rel;
-  bool Threads;
   bool Trace;
   bool Verbose;
   bool WarnCommon;
   bool WarnMissingEntry;
   bool ZCombreloc;
   bool ZExecstack;
   bool ZNocopyreloc;
   bool ZNodelete;

lld/trunk/ELF/Driver.cpp

 #include "LinkerScript.h"
 #include "Memory.h"
 #include "OutputSections.h"
 #include "ScriptParser.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "SyntheticSections.h"
 #include "Target.h"
-#include "Threads.h"
 #include "Writer.h"
 #include "lld/Common/Driver.h"
+#include "lld/Common/Threads.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
void LinkerDriver::readConfigs(opt::InputArgList &Args) {
   Config->Sysroot = Args.getLastArgValue(OPT_sysroot);
   Config->Target1Rel = getArg(Args, OPT_target1_rel, OPT_target1_abs, false);
   Config->Target2 = getTarget2(Args);
   Config->ThinLTOCacheDir = Args.getLastArgValue(OPT_thinlto_cache_dir);
   Config->ThinLTOCachePolicy = check(
       parseCachePruningPolicy(Args.getLastArgValue(OPT_thinlto_cache_policy)),
       "--thinlto-cache-policy: invalid cache policy");
   Config->ThinLTOJobs = getInteger(Args, OPT_thinlto_jobs, -1u);
-  Config->Threads = getArg(Args, OPT_threads, OPT_no_threads, true);
+  ThreadsEnabled = getArg(Args, OPT_threads, OPT_no_threads, true);
   Config->Trace = Args.hasArg(OPT_trace);
   Config->Undefined = getArgs(Args, OPT_undefined);
   Config->UnresolvedSymbols = getUnresolvedSymbolPolicy(Args);
   Config->Verbose = Args.hasArg(OPT_verbose);
   Config->WarnCommon = Args.hasArg(OPT_warn_common);
   Config->ZCombreloc = !hasZOption(Args, "nocombreloc");
   Config->ZExecstack = hasZOption(Args, "execstack");
   Config->ZNocopyreloc = hasZOption(Args, "nocopyreloc");

lld/trunk/ELF/Error.cpp

 //===- Error.cpp ----------------------------------------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Error.h"
 #include "Config.h"
-#include "Threads.h"
+
+#include "lld/Common/Threads.h"
 
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/raw_ostream.h"
 #include <mutex>
 
 #if !defined(_MSC_VER) && !defined(__MINGW32__)

lld/trunk/ELF/Filesystem.cpp

 //===- Filesystem.cpp -----------------------------------------------------===//
 //
 //                             The LLVM Linker
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains a few utility functions to handle files.
 //
 //===----------------------------------------------------------------------===//
 
 #include "Filesystem.h"
 #include "Config.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 
 using namespace llvm;
 
 using namespace lld;
 using namespace lld::elf;
 
 // Removes a given file asynchronously. This is a performance hack,
 // so remove this when operating systems are improved.
 //
 // On Linux (and probably on other Unix-like systems), unlink(2) is a
 // noticeably slow system call. As of 2016, unlink takes 250
 // milliseconds to remove a 1 GB file on ext4 filesystem on my machine.
 //
 // To create a new result file, we first remove existing file. So, if
 // you repeatedly link a 1 GB program in a regular compile-link-debug
 // cycle, every cycle wastes 250 milliseconds only to remove a file.
 // Since LLD can link a 1 GB binary in about 5 seconds, that waste
 // actually counts.
 //
 // This function spawns a background thread to call unlink.
 // The calling thread returns almost immediately.
 void elf::unlinkAsync(StringRef Path) {
-  if (!Config->Threads || !sys::fs::exists(Config->OutputFile) ||
+  if (!ThreadsEnabled || !sys::fs::exists(Config->OutputFile) ||
       !sys::fs::is_regular_file(Config->OutputFile))
     return;
 
   // First, rename Path to avoid race condition. We cannot remove
   // Path from a different thread because we are now going to create
   // Path as a new file. If we do that in a different thread, the new
   // thread can remove the new file.
   SmallString<128> TempPath;

lld/trunk/ELF/ICF.cpp

 // in the Gold Linker
 // http://static.googleusercontent.com/media/research.google.com/en//pubs/archive/36912.pdf
 //
 //===----------------------------------------------------------------------===//
 
 #include "ICF.h"
 #include "Config.h"
 #include "SymbolTable.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Object/ELF.h"
 #include <algorithm>
 #include <atomic>
 
 using namespace lld;
 using namespace lld::elf;
void ICF<ELFT>::forEachClassRange(size_t Begin, size_t End,
   }
 }
 
 // Call Fn on each equivalence class.
 template <class ELFT>
 void ICF<ELFT>::forEachClass(std::function<void(size_t, size_t)> Fn) {
   // If threading is disabled or the number of sections are
   // too small to use threading, call Fn sequentially.
-  if (!Config->Threads || Sections.size() < 1024) {
+  if (!ThreadsEnabled || Sections.size() < 1024) {
     forEachClassRange(0, Sections.size(), Fn);
     ++Cnt;
     return;
   }
 
   Current = Cnt % 2;
   Next = (Cnt + 1) % 2;
 

lld/trunk/ELF/LinkerScript.cpp

 #include "InputSection.h"
 #include "Memory.h"
 #include "OutputSections.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
-#include "Threads.h"
 #include "Writer.h"
+#include "lld/Common/Threads.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"

lld/trunk/ELF/MapFile.cpp

 
 #include "MapFile.h"
 #include "InputFiles.h"
 #include "LinkerScript.h"
 #include "OutputSections.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "SyntheticSections.h"
-#include "Threads.h"
+
+#include "lld/Common/Threads.h"
 
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 using namespace llvm::object;
 
 using namespace lld;
 using namespace lld::elf;

lld/trunk/ELF/OutputSections.cpp

 #include "OutputSections.h"
 #include "Config.h"
 #include "LinkerScript.h"
 #include "Memory.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "SyntheticSections.h"
 #include "Target.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/SHA1.h"
 
 using namespace llvm;
 using namespace llvm::dwarf;

lld/trunk/ELF/SyntheticSections.cpp

 #include "Error.h"
 #include "InputFiles.h"
 #include "LinkerScript.h"
 #include "Memory.h"
 #include "OutputSections.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "Target.h"
-#include "Threads.h"
 #include "Writer.h"
+#include "lld/Common/Threads.h"
 #include "lld/Common/Version.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugPubTable.h"
 #include "llvm/Object/Decompressor.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/RandomNumberGenerator.h"

 void MergeNoTailSection::finalizeContents() {
   // Initializes string table builders.
   for (size_t I = 0; I < NumShards; ++I)
     Shards.emplace_back(StringTableBuilder::RAW, Alignment);
 
   // Concurrency level. Must be a power of 2 to avoid expensive modulo
   // operations in the following tight loop.
   size_t Concurrency = 1;
-  if (Config->Threads)
+  if (ThreadsEnabled)
     Concurrency =
         std::min<size_t>(PowerOf2Floor(hardware_concurrency()), NumShards);
 
   // Add section pieces to the builders.
   parallelForEachN(0, Concurrency, [&](size_t ThreadId) {
     for (MergeInputSection *Sec : Sections) {
       for (size_t I = 0, E = Sec->Pieces.size(); I != E; ++I) {
         if (!Sec->Pieces[I].Live)

lld/trunk/ELF/Threads.h

-//===- Threads.h ------------------------------------------------*- C++ -*-===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// LLD supports threads to distribute workloads to multiple cores. Using
-// multicore is most effective when more than one core are idle. At the
-// last step of a build, it is often the case that a linker is the only
-// active process on a computer. So, we are naturally interested in using
-// threads wisely to reduce latency to deliver results to users.
-//
-// That said, we don't want to do "too clever" things using threads.
-// Complex multi-threaded algorithms are sometimes extremely hard to
-// reason about and can easily mess up the entire design.
-//
-// Fortunately, when a linker links large programs (when the link time is
-// most critical), it spends most of the time to work on massive number of
-// small pieces of data of the same kind, and there are opportunities for
-// large parallelism there. Here are examples:
-//
-//  - We have hundreds of thousands of input sections that need to be
-//    copied to a result file at the last step of link. Once we fix a file
-//    layout, each section can be copied to its destination and its
-//    relocations can be applied independently.
-//
-//  - We have tens of millions of small strings when constructing a
-//    mergeable string section.
-//
-// For the cases such as the former, we can just use parallelForEach
-// instead of std::for_each (or a plain for loop). Because tasks are
-// completely independent from each other, we can run them in parallel
-// without any coordination between them. That's very easy to understand
-// and reason about.
-//
-// For the cases such as the latter, we can use parallel algorithms to
-// deal with massive data. We have to write code for a tailored algorithm
-// for each problem, but the complexity of multi-threading is isolated in
-// a single pass and doesn't affect the linker's overall design.
-//
-// The above approach seems to be working fairly well. As an example, when
-// linking Chromium (output size 1.6 GB), using 4 cores reduces latency to
-// 75% compared to single core (from 12.66 seconds to 9.55 seconds) on my
-// Ivy Bridge Xeon 2.8 GHz machine. Using 40 cores reduces it to 63% (from
-// 12.66 seconds to 7.95 seconds). Because of the Amdahl's law, the
-// speedup is not linear, but as you add more cores, it gets faster.
-//
-// On a final note, if you are trying to optimize, keep the axiom "don't
-// guess, measure!" in mind. Some important passes of the linker are not
-// that slow. For example, resolving all symbols is not a very heavy pass,
-// although it would be very hard to parallelize it. You want to first
-// identify a slow pass and then optimize it.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_ELF_THREADS_H
-#define LLD_ELF_THREADS_H
-
-#include "Config.h"
-
-#include "llvm/Support/Parallel.h"
-#include <functional>
-
-namespace lld {
-namespace elf {
-
-template <typename R, class FuncTy> void parallelForEach(R &&Range, FuncTy Fn) {
-  if (Config->Threads)
-    for_each(llvm::parallel::par, std::begin(Range), std::end(Range), Fn);
-  else
-    for_each(llvm::parallel::seq, std::begin(Range), std::end(Range), Fn);
-}
-
-inline void parallelForEachN(size_t Begin, size_t End,
-                             std::function<void(size_t)> Fn) {
-  if (Config->Threads)
-    for_each_n(llvm::parallel::par, Begin, End, Fn);
-  else
-    for_each_n(llvm::parallel::seq, Begin, End, Fn);
-}
-
-void runBackground(std::function<void()> Fn);
-void waitForBackgroundThreads();
-
-} // namespace elf
-} // namespace lld
-
-#endif

lld/trunk/ELF/Threads.cpp

-//===- Threads.cpp --------------------------------------------------------===//
-//
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Threads.h"
-#include <thread>
-
-static std::vector<std::thread> Threads;
-
-// Runs a given function in a new thread.
-void lld::elf::runBackground(std::function<void()> Fn) {
-  Threads.emplace_back(Fn);
-}
-
-// Wait for all threads spawned for runBackground() to finish.
-//
-// You need to call this function from the main thread before exiting
-// because it is not defined what will happen to non-main threads when
-// the main thread exits.
-void lld::elf::waitForBackgroundThreads() {
-  for (std::thread &T : Threads)
-    if (T.joinable())
-      T.join();
-}

lld/trunk/ELF/Writer.cpp

 #include "MapFile.h"
 #include "Memory.h"
 #include "OutputSections.h"
 #include "Relocations.h"
 #include "Strings.h"
 #include "SymbolTable.h"
 #include "SyntheticSections.h"
 #include "Target.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include <climits>
 
 using namespace llvm;
 using namespace llvm::ELF;
 using namespace llvm::object;

lld/trunk/include/lld/Common/Threads.h

+//===- Threads.h ------------------------------------------------*- C++ -*-===//
+//
+//                             The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// LLD supports threads to distribute workloads to multiple cores. Using
+// multicore is most effective when more than one core are idle. At the
+// last step of a build, it is often the case that a linker is the only
+// active process on a computer. So, we are naturally interested in using
+// threads wisely to reduce latency to deliver results to users.
+//
+// That said, we don't want to do "too clever" things using threads.
+// Complex multi-threaded algorithms are sometimes extremely hard to
+// reason about and can easily mess up the entire design.
+//
+// Fortunately, when a linker links large programs (when the link time is
+// most critical), it spends most of the time to work on massive number of
+// small pieces of data of the same kind, and there are opportunities for
+// large parallelism there. Here are examples:
+//
+//  - We have hundreds of thousands of input sections that need to be
+//    copied to a result file at the last step of link. Once we fix a file
+//    layout, each section can be copied to its destination and its
+//    relocations can be applied independently.
+//
+//  - We have tens of millions of small strings when constructing a
+//    mergeable string section.
+//
+// For the cases such as the former, we can just use parallelForEach
+// instead of std::for_each (or a plain for loop). Because tasks are
+// completely independent from each other, we can run them in parallel
+// without any coordination between them. That's very easy to understand
+// and reason about.
+//
+// For the cases such as the latter, we can use parallel algorithms to
+// deal with massive data. We have to write code for a tailored algorithm
+// for each problem, but the complexity of multi-threading is isolated in
+// a single pass and doesn't affect the linker's overall design.
+//
+// The above approach seems to be working fairly well. As an example, when
+// linking Chromium (output size 1.6 GB), using 4 cores reduces latency to
+// 75% compared to single core (from 12.66 seconds to 9.55 seconds) on my
+// Ivy Bridge Xeon 2.8 GHz machine. Using 40 cores reduces it to 63% (from
+// 12.66 seconds to 7.95 seconds). Because of the Amdahl's law, the
+// speedup is not linear, but as you add more cores, it gets faster.
+//
+// On a final note, if you are trying to optimize, keep the axiom "don't
+// guess, measure!" in mind. Some important passes of the linker are not
+// that slow. For example, resolving all symbols is not a very heavy pass,
+// although it would be very hard to parallelize it. You want to first
+// identify a slow pass and then optimize it.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COMMON_THREADS_H
+#define LLD_COMMON_THREADS_H
+
+#include "llvm/Support/Parallel.h"
+#include <functional>
+
+namespace lld {
+
+extern bool ThreadsEnabled;
+
+template <typename R, class FuncTy> void parallelForEach(R &&Range, FuncTy Fn) {
+  if (ThreadsEnabled)
+    for_each(llvm::parallel::par, std::begin(Range), std::end(Range), Fn);
+  else
+    for_each(llvm::parallel::seq, std::begin(Range), std::end(Range), Fn);
+}
+
+inline void parallelForEachN(size_t Begin, size_t End,
+                             std::function<void(size_t)> Fn) {
+  if (ThreadsEnabled)
+    for_each_n(llvm::parallel::par, Begin, End, Fn);
+  else
+    for_each_n(llvm::parallel::seq, Begin, End, Fn);
+}
+
+void runBackground(std::function<void()> Fn);
+void waitForBackgroundThreads();
+
+} // namespace lld
+
+#endif