[LLD][COFF] Support /threads[:no] like the ELF driver

[aganea]

Summary

This is mostly for debugging for now. With the upcoming parallel ghashes, I need a way to disable iterating in parallel.

[rnk]

lgtm

At first I was thinking, maybe we should extend this to /threads:N, so some build systems could reduce the parallelism when links run concurrently, but I think for Windows it's best to just let concrt/the system do its thing.

The other big source of expensive concurrency is ThinLTO, which logically we should wire up to /cgthreads.

[ruiu]

LGTM

[aganea]

Closed by rL355029

COFF/Driver.cpp

 #include "MinGW.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "Writer.h"
 #include "lld/Common/Args.h"
 #include "lld/Common/Driver.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Threads.h"
 #include "lld/Common/Timer.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/COFFModuleDefinition.h"
void LinkerDriver::link(ArrayRef<const char *> ArgsArr) {
   }
 
   // Handle /help
   if (Args.hasArg(OPT_help)) {
     printHelp(ArgsArr[0]);
     return;
   }
 
+  lld::ThreadsEnabled = Args.hasFlag(OPT_threads, OPT_threads_no, true);
+
   if (Args.hasArg(OPT_show_timing))
     Config->ShowTiming = true;
 
   ScopedTimer T(Timer::root());
   // Handle --version, which is an lld extension. This option is a bit odd
   // because it doesn't start with "/", but we deliberately chose "--" to
   // avoid conflict with /version and for compatibility with clang-cl.
   if (Args.hasArg(OPT_dash_dash_version)) {

COFF/MapFile.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "MapFile.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "Writer.h"
 #include "lld/Common/ErrorHandler.h"
-#include "llvm/Support/Parallel.h"
+#include "lld/Common/Threads.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 using namespace llvm::object;
 
 using namespace lld;
 using namespace lld::coff;
 
static SymbolMapTy getSectionSyms(ArrayRef<DefinedRegular *> Syms) {
   }
   return Ret;
 }
 
 // Construct a map from symbols to their stringified representations.
 static DenseMap<DefinedRegular *, std::string>
 getSymbolStrings(ArrayRef<DefinedRegular *> Syms) {
   std::vector<std::string> Str(Syms.size());
-  for_each_n(parallel::par, (size_t)0, Syms.size(), [&](size_t I) {
+  parallelForEachN((size_t)0, Syms.size(), [&](size_t I) {
     raw_string_ostream OS(Str[I]);
     writeHeader(OS, Syms[I]->getRVA(), 0, 0);
     OS << Indent16 << toString(*Syms[I]);
   });
 
   DenseMap<DefinedRegular *, std::string> Ret;
   for (size_t I = 0, E = Syms.size(); I < E; ++I)
     Ret[Syms[I]] = std::move(Str[I]);

COFF/Options.td

 def lldmingw : F<"lldmingw">;
 def output_def : Joined<["/", "-"], "output-def:">;
 def pdb_source_path : P<"pdbsourcepath",
                         "Base path used to make relative source file path absolute in PDB">;
 def rsp_quoting : Joined<["--"], "rsp-quoting=">,
   HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">;
 def dash_dash_version : Flag<["--"], "version">,
   HelpText<"Print version information">;
+defm threads: B<"threads",
+    "Run the linker multi-threaded (default)",
+    "Do not run the linker multi-threaded">;
 
 // Flags for debugging
 def lldmap : F<"lldmap">;
 def lldmap_file : Joined<["/", "-"], "lldmap:">;
 def show_timing : F<"time">;
 
 //==============================================================================
 // The flags below do nothing. They are defined only for link.exe compatibility.

COFF/PDB.cpp

 #include "Chunks.h"
 #include "Config.h"
 #include "Driver.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "Writer.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Timer.h"
+#include "lld/Common/Threads.h"
 #include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugSubsectionRecord.h"
 #include "llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h"
 #include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/RecordName.h"
 #include "llvm/DebugInfo/CodeView/SymbolDeserializer.h"
 #include "llvm/DebugInfo/CodeView/SymbolRecordHelpers.h"

 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Object/CVDebugRecord.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/JamCRC.h"
-#include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include <memory>
 
 using namespace lld;
 using namespace lld::coff;
 using namespace llvm;
 using namespace llvm::codeview;
Symtab->forEachSymbol([&Publics](Symbol *S) {
     // Only emit defined, live symbols that have a chunk.
     auto *Def = dyn_cast<Defined>(S);
     if (Def && Def->isLive() && Def->getChunk())
       Publics.push_back(createPublic(Def));
   });
 
   if (!Publics.empty()) {
     // Sort the public symbols and add them to the stream.
-    sort(parallel::par, Publics.begin(), Publics.end(),
-         [](const PublicSym32 &L, const PublicSym32 &R) {
-           return L.Name < R.Name;
-         });
+    parallelSort(Publics, [](const PublicSym32 &L, const PublicSym32 &R) {
     for (const PublicSym32 &Pub : Publics)
       GsiBuilder.addPublicSymbol(Pub);
   }
 }
 
 void PDBLinker::addNatvisFiles() {
   for (StringRef File : Config->NatvisFiles) {
     ErrorOr<std::unique_ptr<MemoryBuffer>> DataOrErr =

COFF/Writer.cpp

 #include "DLL.h"
 #include "InputFiles.h"
 #include "MapFile.h"
 #include "PDB.h"
 #include "SymbolTable.h"
 #include "Symbols.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Threads.h"
 #include "lld/Common/Timer.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/BinaryStreamReader.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/FileOutputBuffer.h"
for (auto &P : Config->Merge) {
     }
     To->merge(From);
   }
 }
 
 // Visits all sections to initialize their relocation targets.
 void Writer::readRelocTargets() {
   for (OutputSection *Sec : OutputSections)
-    for_each(parallel::par, Sec->Chunks.begin(), Sec->Chunks.end(),
-             [&](Chunk *C) { C->readRelocTargets(); });
+    parallelForEach(Sec->Chunks, [&](Chunk *C) { C->readRelocTargets(); });
 }
 
 // Visits all sections to assign incremental, non-overlapping RVAs and
 // file offsets.
 void Writer::assignAddresses() {
   SizeOfHeaders = DOSStubSize + sizeof(PEMagic) + sizeof(coff_file_header) +
                   sizeof(data_directory) * NumberOfDataDirectory +
                   sizeof(coff_section) * OutputSections.size();
void Writer::writeSections() {
   uint8_t *Buf = Buffer->getBufferStart();
   for (OutputSection *Sec : OutputSections) {
     uint8_t *SecBuf = Buf + Sec->getFileOff();
     // Fill gaps between functions in .text with INT3 instructions
     // instead of leaving as NUL bytes (which can be interpreted as
     // ADD instructions).
     if (Sec->Header.Characteristics & IMAGE_SCN_CNT_CODE)
       memset(SecBuf, 0xCC, Sec->getRawSize());
-    for_each(parallel::par, Sec->Chunks.begin(), Sec->Chunks.end(),
-             [&](Chunk *C) { C->writeTo(SecBuf); });
+    parallelForEach(Sec->Chunks, [&](Chunk *C) { C->writeTo(SecBuf); });
   }
 }
 
 void Writer::writeBuildId() {
   // There are two important parts to the build ID.
   // 1) If building with debug info, the COFF debug directory contains a
   //    timestamp as well as a Guid and Age of the PDB.
   // 2) In all cases, the PE COFF file header also contains a timestamp.
void Writer::sortExceptionTable() {
   auto BufAddr = [&](Chunk *C) {
     return Buffer->getBufferStart() + C->getOutputSection()->getFileOff() +
            C->getRVA() - C->getOutputSection()->getRVA();
   };
   uint8_t *Begin = BufAddr(FirstPdata);
   uint8_t *End = BufAddr(LastPdata) + LastPdata->getSize();
   if (Config->Machine == AMD64) {
     struct Entry { ulittle32_t Begin, End, Unwind; };
-    sort(parallel::par, (Entry *)Begin, (Entry *)End,
-         [](const Entry &A, const Entry &B) { return A.Begin < B.Begin; });
+    parallelSort(
+        MutableArrayRef<Entry>((Entry *)Begin, (Entry *)End),
     return;
   }
   if (Config->Machine == ARMNT || Config->Machine == ARM64) {
     struct Entry { ulittle32_t Begin, Unwind; };
-    sort(parallel::par, (Entry *)Begin, (Entry *)End,
-         [](const Entry &A, const Entry &B) { return A.Begin < B.Begin; });
+    parallelSort(
+        MutableArrayRef<Entry>((Entry *)Begin, (Entry *)End),
     return;
   }
   errs() << "warning: don't know how to handle .pdata.\n";
 }
 
 // The CRT section contains, among other things, the array of function
 // pointers that initialize every global variable that is not trivially
 // constructed. The CRT calls them one after the other prior to invoking

include/lld/Common/Threads.h

 inline void parallelForEachN(size_t Begin, size_t End,
                              llvm::function_ref<void(size_t)> Fn) {
   if (ThreadsEnabled)
     for_each_n(llvm::parallel::par, Begin, End, Fn);
   else
     for_each_n(llvm::parallel::seq, Begin, End, Fn);
 }
 
+template <typename R, class FuncTy> void parallelSort(R &&Range, FuncTy Fn) {
+  if (ThreadsEnabled)
+    sort(llvm::parallel::par, std::begin(Range), std::end(Range), Fn);
+  else
+    sort(llvm::parallel::seq, std::begin(Range), std::end(Range), Fn);
+}
+
 } // namespace lld
 
 #endif