diff --git a/clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json b/clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json new file mode 100644 --- /dev/null +++ b/clang/test/ClangScanDeps/Inputs/num-workers-1cmd.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E -fsyntax-only DIR/file.cpp", + "file": "DIR/file.cpp" +} +] diff --git a/clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json b/clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json new file mode 100644 --- /dev/null +++ b/clang/test/ClangScanDeps/Inputs/num-workers-2cmds.json @@ -0,0 +1,12 @@ +[ +{ + "directory": "DIR", + "command": "clang -E -fsyntax-only DIR/file.cpp", + "file": "DIR/file.cpp" +}, +{ + "directory": "DIR", + "command": "clang -E -fsyntax-only DIR/file2.cpp", + "file": "DIR/file2.cpp" +} +] diff --git a/clang/test/ClangScanDeps/num-workers.cpp b/clang/test/ClangScanDeps/num-workers.cpp new file mode 100644 --- /dev/null +++ b/clang/test/ClangScanDeps/num-workers.cpp @@ -0,0 +1,31 @@ +// REQUIRES: thread_support +// RUN: rm -rf %t.dir +// RUN: rm -f %t.*.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/file.cpp +// RUN: cp %s %t.dir/file2.cpp +// RUN: echo > %t.dir/header.h + +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/num-workers-1cmd.json > %t.1cmd.cdb +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/num-workers-2cmds.json > %t.2cmds.cdb +// +// RUN: clang-scan-deps -compilation-database %t.1cmd.cdb -j 2 -v | \ +// RUN: FileCheck --check-prefixes=CHECK,CHECK1F1W %s +// +// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 1 -v | \ +// RUN: FileCheck --check-prefixes=CHECK,CHECK2F1W %s +// +// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 2 -v | \ +// RUN: FileCheck --check-prefixes=CHECK,CHECK2F2W %s +// +// RUN: clang-scan-deps -compilation-database %t.2cmds.cdb -j 10 -v | \ +// RUN: FileCheck --check-prefixes=CHECK,CHECK2F2W %s + +#include "header.h" + +// CHECK1F1W: Running clang-scan-deps on 1 files using 1 workers +// CHECK2F1W: Running clang-scan-deps on 2 files using 1 workers +// CHECK2F2W: Running clang-scan-deps on 2 files using 2 workers + +// CHECK: file.cpp +// CHECK-NEXT: header.h diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -20,7 +20,6 @@ #include "llvm/Support/JSON.h" #include "llvm/Support/Program.h" #include "llvm/Support/Signals.h" -#include "llvm/Support/ThreadPool.h" #include "llvm/Support/Threading.h" #include #include @@ -555,16 +554,29 @@ DependencyScanningService Service(ScanMode, Format, ReuseFileManager, SkipExcludedPPRanges); - llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads)); - std::vector> WorkerTools; - for (unsigned I = 0; I < Pool.getThreadCount(); ++I) - WorkerTools.push_back(std::make_unique(Service)); std::vector Inputs; - for (tooling::CompileCommand Cmd : - AdjustingCompilations->getAllCompileCommands()) - Inputs.emplace_back(Cmd); + { + auto AdjustedCommands = AdjustingCompilations->getAllCompileCommands(); + Inputs.reserve(AdjustedCommands.size()); + for (tooling::CompileCommand Cmd : AdjustedCommands) + Inputs.emplace_back(Cmd); + } + unsigned NumWorkers = + llvm::hardware_concurrency(NumThreads).compute_thread_count(); + + if (NumWorkers > Inputs.size()) + NumWorkers = static_cast(Inputs.size()); + + std::vector> WorkerTools; + WorkerTools.reserve(NumWorkers); + for (unsigned I = 0; I < NumWorkers; ++I) + WorkerTools.push_back(std::make_unique(Service)); + + std::vector WorkerThreads; + if (NumWorkers > 1) + WorkerThreads.reserve(NumWorkers - 1); std::atomic HadErrors(false); FullDeps FD; std::mutex Lock; @@ -572,11 +584,11 @@ if (Verbose) { llvm::outs() << "Running clang-scan-deps on " << Inputs.size() - << " files using " << Pool.getThreadCount() << " workers\n"; + << " files using " << NumWorkers << " workers\n"; } - for (unsigned I = 0; I < Pool.getThreadCount(); ++I) { - Pool.async([I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, - &DependencyOS, &Errs]() { + for (unsigned I = 0; I < NumWorkers; ++I) { + auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &FD, &WorkerTools, + &DependencyOS, &Errs]() { llvm::StringSet<> AlreadySeenModules; while (true) { const SingleCommandCompilationDatabase *Input; @@ -608,9 +620,17 @@ HadErrors = true; } } - }); + }; + // Leave the last worker for the main thread. + if (I < NumWorkers - 1) { + assert(llvm::llvm_is_multithreaded()); + WorkerThreads.emplace_back(std::move(Worker)); + } else { + Worker(); + } } - Pool.wait(); + for (auto &W : WorkerThreads) + W.join(); if (Format == ScanningOutputFormat::Full) FD.printFullOutput(llvm::outs());