Index: clang/include/clang/Driver/Driver.h =================================================================== --- clang/include/clang/Driver/Driver.h +++ clang/include/clang/Driver/Driver.h @@ -247,6 +247,9 @@ /// stored in it, and will clean them up when torn down. mutable llvm::StringMap> ToolChains; + /// Number of parallel jobs. + unsigned NumParallelJobs; + private: /// TranslateInputArgs - Create a new derived argument list from the input /// arguments, after applying the standard argument translations. @@ -549,6 +552,12 @@ /// Get the specific kind of LTO being performed. LTOKind getLTOMode() const { return LTOMode; } + /// Get the number of parallel jobs. + unsigned getNumberOfParallelJobs() const { return NumParallelJobs; } + + /// Set the number of parallel jobs. + void setNumberOfParallelJobs(unsigned N) { NumParallelJobs = N; } + private: /// Tries to load options from configuration file. Index: clang/include/clang/Driver/Job.h =================================================================== --- clang/include/clang/Driver/Job.h +++ clang/include/clang/Driver/Job.h @@ -73,6 +73,9 @@ /// See Command::setEnvironment std::vector Environment; + /// Dependent actions + llvm::SmallVector DependentActions; + /// When a response file is needed, we try to put most arguments in an /// exclusive file, while others remains as regular command line arguments. /// This functions fills a vector with the regular command line arguments, @@ -130,6 +133,10 @@ /// Set whether to print the input filenames when executing. void setPrintInputFilenames(bool P) { PrintInputFilenames = P; } + + const llvm::SmallVector &getDependentActions() const { + return DependentActions; + } }; /// Like Command, but with a fallback which is executed in case Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -401,6 +401,8 @@ def Ofast : Joined<["-"], "Ofast">, Group, Flags<[CC1Option]>; def P : Flag<["-"], "P">, Flags<[CC1Option]>, Group, HelpText<"Disable linemarker output in -E mode">; +def parallel_jobs_EQ : Joined<["-"], "parallel-jobs=">, Flags<[DriverOption]>, + HelpText<"Number of parallel jobs">; def Qy : Flag<["-"], "Qy">, Flags<[CC1Option]>, HelpText<"Emit metadata containing compiler name and version">; def Qn : Flag<["-"], "Qn">, Flags<[CC1Option]>, Index: clang/lib/Driver/Compilation.cpp =================================================================== --- clang/lib/Driver/Compilation.cpp +++ clang/lib/Driver/Compilation.cpp @@ -15,6 +15,7 @@ #include "clang/Driver/Options.h" #include "clang/Driver/ToolChain.h" #include "clang/Driver/Util.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -25,8 +26,11 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/raw_ostream.h" #include +#include +#include #include #include +#include #include using namespace clang; @@ -220,22 +224,134 @@ return !ActionFailed(&C.getSource(), FailingCommands); } +namespace { +class JobScheduler { +public: + enum JobState { JS_WAIT, JS_RUN, JS_DONE, JS_FAIL }; + JobScheduler(const JobList &Jobs, size_t NJobs = 1) + : Jobs(Jobs), NumJobs(NJobs) { +#if !LLVM_ENABLE_THREADS + NumJobs = 1; +#endif + for (auto &Job : Jobs) { + JState[&Job] = JS_WAIT; + for (const auto *AI : Job.getDependentActions()) { + for (const auto *CI : ActToCmds[AI]) { + DependentCmds[&Job].push_back(CI); + } + } + for (const auto *CI : ActToCmds[&Job.getSource()]) { + DependentCmds[&Job].push_back(CI); + } + ActToCmds[&Job.getSource()].push_back(&Job); + } + } + /// \return true if all jobs are done. Otherwise, \p Next contains the + /// the next job ready to be executed if it is not null pointer. Otherwise + /// all jobs are running or waiting. + bool IsDone(const Command *&Next) { + std::lock_guard lock(Mutex); + Next = nullptr; + unsigned Done = 0; + unsigned Running = 0; + for (auto &Cmd : Jobs) { + switch (JState[&Cmd]) { + case JS_RUN: + ++Running; + break; + case JS_DONE: + case JS_FAIL: + ++Done; + break; + case JS_WAIT: { + bool InputsReady = true; + for (const auto *CI : DependentCmds[&Cmd]) { + if (JState[CI] == JS_FAIL) { + JState[&Cmd] = JS_FAIL; + ++Done; + InputsReady = false; + break; + } + if (JState[CI] != JS_DONE) { + InputsReady = false; + break; + } + } + if (!Next && InputsReady) { + Next = &Cmd; + } + break; + } + } + } + if (Running >= NumJobs) + Next = nullptr; + return Done == Jobs.size(); + } + + void setJobState(const Command *Cmd, JobState JS) { + std::lock_guard lock(Mutex); + JState[Cmd] = JS; + } + + void launch(std::function Work) { +#if LLVM_ENABLE_THREADS + if (NumJobs == 1) { + Work(); + return; + } + std::thread Th(Work); + Th.detach(); +#else + Work(); +#endif + } + +private: + std::mutex Mutex; + const JobList &Jobs; + llvm::DenseMap JState; + llvm::DenseMap> + ActToCmds; + llvm::DenseMap> + DependentCmds; + size_t NumJobs; // Number of parallel jobs to run +}; +} // namespace void Compilation::ExecuteJobs(const JobList &Jobs, FailingCommandList &FailingCommands) const { // According to UNIX standard, driver need to continue compiling all the // inputs on the command line even one of them failed. // In all but CLMode, execute all the jobs unless the necessary inputs for the // job is missing due to previous failures. - for (const auto &Job : Jobs) { - if (!InputsOk(Job, FailingCommands)) + JobScheduler JS(Jobs, getDriver().getNumberOfParallelJobs()); + + const Command *Next = nullptr; + while (!JS.IsDone(Next)) { + if (!Next) { + std::this_thread::yield(); continue; - const Command *FailingCommand = nullptr; - if (int Res = ExecuteCommand(Job, FailingCommand)) { - FailingCommands.push_back(std::make_pair(Res, FailingCommand)); + } + + if (!InputsOk(*Next, FailingCommands)) { + JS.setJobState(Next, JobScheduler::JS_FAIL); // Bail as soon as one command fails in cl driver mode. if (TheDriver.IsCLMode()) return; + continue; } + + JS.setJobState(Next, JobScheduler::JS_RUN); + auto Work = [&, Next]() { + const Command *FailingCommand = nullptr; + if (int Res = ExecuteCommand(*Next, FailingCommand)) { + JS.setJobState(Next, JobScheduler::JS_FAIL); + FailingCommands.push_back(std::make_pair(Res, FailingCommand)); + } else { + JS.setJobState(Next, JobScheduler::JS_DONE); + } + }; + JS.launch(Work); } } Index: clang/lib/Driver/Driver.cpp =================================================================== --- clang/lib/Driver/Driver.cpp +++ clang/lib/Driver/Driver.cpp @@ -38,13 +38,14 @@ #include "ToolChains/NaCl.h" #include "ToolChains/NetBSD.h" #include "ToolChains/OpenBSD.h" -#include "ToolChains/PS4CPU.h" #include "ToolChains/PPCLinux.h" +#include "ToolChains/PS4CPU.h" #include "ToolChains/RISCVToolchain.h" #include "ToolChains/Solaris.h" #include "ToolChains/TCE.h" #include "ToolChains/WebAssembly.h" #include "ToolChains/XCore.h" +#include "clang/Basic/OptionUtils.h" #include "clang/Basic/Version.h" #include "clang/Config/config.h" #include "clang/Driver/Action.h" @@ -55,6 +56,7 @@ #include "clang/Driver/SanitizerArgs.h" #include "clang/Driver/Tool.h" #include "clang/Driver/ToolChain.h" +#include "clang/Driver/Util.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" @@ -130,7 +132,7 @@ CCLogDiagnostics(false), CCGenDiagnostics(false), TargetTriple(TargetTriple), CCCGenericGCCName(""), Saver(Alloc), CheckInputsExist(true), GenReproducer(false), - SuppressMissingInputWarning(false) { + SuppressMissingInputWarning(false), NumParallelJobs(1) { // Provide a sane fallback if no VFS is specified. if (!this->VFS) @@ -1103,6 +1105,9 @@ BitcodeEmbed = static_cast(Model); } + setNumberOfParallelJobs( + getLastArgIntValue(Args, options::OPT_parallel_jobs_EQ, 1, Diags)); + std::unique_ptr UArgs = std::make_unique(std::move(Args)); Index: clang/lib/Driver/Job.cpp =================================================================== --- clang/lib/Driver/Job.cpp +++ clang/lib/Driver/Job.cpp @@ -39,9 +39,11 @@ ArrayRef Inputs) : Source(Source), Creator(Creator), Executable(Executable), Arguments(Arguments) { - for (const auto &II : Inputs) + for (const auto &II : Inputs) { if (II.isFilename()) InputFilenames.push_back(II.getFilename()); + DependentActions.push_back(II.getAction()); + } } /// Check if the compiler flag in question should be skipped when Index: clang/lib/Driver/ToolChains/Clang.cpp =================================================================== --- clang/lib/Driver/ToolChains/Clang.cpp +++ clang/lib/Driver/ToolChains/Clang.cpp @@ -6483,7 +6483,7 @@ C.addCommand(std::make_unique( JA, *this, TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())), - CmdArgs, None)); + CmdArgs, Inputs)); } void OffloadBundler::ConstructJobMultipleOutputs( @@ -6549,7 +6549,7 @@ C.addCommand(std::make_unique( JA, *this, TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())), - CmdArgs, None)); + CmdArgs, Inputs)); } void OffloadWrapper::ConstructJob(Compilation &C, const JobAction &JA,