[lld-macho] Support --thinlto-jobs

[int3]

Summary

The test is loosely based off LLD-ELF's thinlto.ll. However, I
found that test questionable because the the -save_temps behavior it
checks for is identical regardless of whether we are running in single-
or multi-threaded mode. I tried writing a test based on --time-trace
but couldn't get it to run deterministically... so I've opted to just
skip checking that behavior for now.

[alexander-shaposhnikov]

lld/test/MachO/Inputs/time_trace_tid_count.py

16 ↗

(On Diff #333355)

print(len(set(event['tid'] for event in events if event['name'] == event_name)))

[MaskRay]

LG.. I can port it to test/ELF..

[int3]

python one-liner

[int3]

is the test flaky?

[int3]

Welp, seems like this test is nondeterministic, even though it passes reliably on my local machine. Any ideas? Otherwise I guess we can just land this without tests

[int3]

I'm just going to remove the FileChecks for now

[int3]

nerf the test

[int3]

tweak comment to emphasize determinism

[gkm]

LGTM

lld/MachO/Config.h

struct Configuration {
   uint32_t dylibCompatibilityVersion = 0;
   uint32_t dylibCurrentVersion = 0;
   uint32_t timeTraceGranularity = 500;
   std::string progName;
   llvm::StringRef installName;
   llvm::StringRef mapFile;
   llvm::StringRef outputFile;
   llvm::StringRef ltoObjPath;
+  llvm::StringRef thinLTOJobs;
   bool demangle = false;
   llvm::MachO::Target target;
   PlatformInfo platformInfo;
   NamespaceKind namespaceKind = NamespaceKind::twolevel;
   UndefinedSymbolTreatment undefinedSymbolTreatment =
       UndefinedSymbolTreatment::error;
   llvm::MachO::HeaderFileType outputType;
   std::vector<llvm::StringRef> systemLibraryRoots;

lld/MachO/Driver.cpp

bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
 
   if (auto *arg = args.getLastArg(OPT_threads_eq)) {
     StringRef v(arg->getValue());
     unsigned threads = 0;
     if (!llvm::to_integer(v, threads, 0) || threads == 0)
       error(arg->getSpelling() + ": expected a positive integer, but got '" +
             arg->getValue() + "'");
     parallel::strategy = hardware_concurrency(threads);
-    // FIXME: use this to configure ThinLTO concurrency too
+    config->thinLTOJobs = v;
   }
+  if (auto *arg = args.getLastArg(OPT_thinlto_jobs_eq))
+    config->thinLTOJobs = arg->getValue();
+  if (!get_threadpool_strategy(config->thinLTOJobs))
+    error("--thinlto-jobs: invalid job count: " + config->thinLTOJobs);
 
   config->entry = symtab->addUndefined(args.getLastArgValue(OPT_e, "_main"),
                                        /*file=*/nullptr,
                                        /*isWeakRef=*/false);
   for (const Arg *arg : args.filtered(OPT_u)) {
     config->explicitUndefineds.push_back(symtab->addUndefined(
         arg->getValue(), /*file=*/nullptr, /*isWeakRef=*/false));
   }

lld/MachO/LTO.cpp

static lto::Config createConfig() {
   c.TimeTraceGranularity = config->timeTraceGranularity;
   if (config->saveTemps)
     checkError(c.addSaveTemps(config->outputFile.str() + ".",
                               /*UseInputModulePath=*/true));
   return c;
 }
 
 BitcodeCompiler::BitcodeCompiler() {
-  lto::ThinBackend backend =
-      lto::createInProcessThinBackend(heavyweight_hardware_concurrency());
+  lto::ThinBackend backend = lto::createInProcessThinBackend(
+      heavyweight_hardware_concurrency(config->thinLTOJobs));
   ltoObj = std::make_unique<lto::LTO>(createConfig(), backend);
 }
 
 void BitcodeCompiler::add(BitcodeFile &f) {
   ArrayRef<lto::InputFile::Symbol> objSyms = f.obj->symbols();
   std::vector<lto::SymbolResolution> resols;
   resols.reserve(objSyms.size());
 

lld/MachO/Options.td

def no_color_diagnostics: Flag<["--"], "no-color-diagnostics">,
     Group<grp_lld>;
 def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
     HelpText<"Use colors in diagnostics (default: auto)">,
     MetaVarName<"[auto,always,never]">,
     Group<grp_lld>;
 def threads_eq : Joined<["--"], "threads=">,
     HelpText<"Number of threads. '1' disables multi-threading. By default all available hardware threads are used">,
     Group<grp_lld>;
+def thinlto_jobs_eq : Joined<["--"], "thinlto-jobs=">,
+    HelpText<"Number of ThinLTO jobs. Default to --threads=">,
+    Group<grp_lld>;
 def reproduce: Separate<["--"], "reproduce">,
     Group<grp_lld>;
 def reproduce_eq: Joined<["--"], "reproduce=">,
     Alias<!cast<Separate>(reproduce)>,
     HelpText<"Write tar file containing inputs and command to reproduce link">,
     Group<grp_lld>;
 def version: Flag<["--"], "version">,
     HelpText<"Display the version number and exit">,

lld/test/MachO/thinlto-jobs.ll

+; REQUIRES: x86
+; RUN: rm -rf %t; split-file %s %t
+
+;; I'm not aware of a deterministic way to verify whether LTO is running in
+;; single- or multi-threaded mode. So this test simply checks that we can parse
+;; the --thinlto-jobs flag correctly, but doesn't verify its effect.
+
+; RUN: opt -module-summary %t/f.s -o %t/f.o
+; RUN: opt -module-summary %t/g.s -o %t/g.o
+
+; RUN: %lld --time-trace --thinlto-jobs=1 -dylib %t/f.o %t/g.o -o %t/out
+; RUN: %lld --time-trace --thinlto-jobs=2 -dylib %t/f.o %t/g.o -o %t/out
+; RUN: %lld --thinlto-jobs=all -dylib %t/f.o %t/g.o -o /dev/null
+
+;; Test with a bad value
+; RUN: not %lld --thinlto-jobs=foo -dylib %t/f.o %t/g.o -o /dev/null 2>&1 | FileCheck %s
+; CHECK: error: --thinlto-jobs: invalid job count: foo
+
+;--- f.s
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @g(...)
+
+define void @f() {
+entry:
+  call void (...) @g()
+  ret void
+}
+
+;--- g.s
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @g() {
+entry:
+  ret void
+}