[find-all-symbols] Parallelize the merge step.

[bkramer]

Summary

There is still more parallelism to get here because we synchonize on the
actual uniquing but just doing YAML parsing in parallel already gives a
significant speedup.

Merging all symbols in LLVM+clang+compiler-rt+lld+libc++, 48 cores.
before: 201.55s user 1.47s system 99% cpu 3:23.04 total
after: 279.13s user 7.53s system 929% cpu 30.838 total

[bkramer]

Drain the pool, fix typo.

[hokein]

include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp

107

Don't we need to all 'Pool.wait()' to wait for all working threads complete?

[hokein]

Just see you updated code. LGTM.

include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp

 //===-- FindAllSymbolsMain.cpp --------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #include "FindAllSymbols.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/Path.h"
-
+#include "llvm/Support/ThreadPool.h"
 #include <map>
+#include <mutex>
 #include <string>
 #include <vector>
 
 using namespace clang::tooling;
 using namespace llvm;
 using SymbolInfo = clang::find_all_symbols::SymbolInfo;
 
 // Apply a custom category to all command-line options so that they are the
void Write(const std::string &Dir) {
     }
   }
 
 private:
   std::map<std::string, std::set<SymbolInfo>> Symbols;
 };
 
 bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) {
-  std::error_code EC;
   std::set<SymbolInfo> UniqueSymbols;
+  std::mutex SymbolMutex;
+  auto AddSymbols = [&](ArrayRef<SymbolInfo> Symbols) {
+    // Synchronize set accesses.
+    std::unique_lock<std::mutex> LockGuard(SymbolMutex);
+    for (const SymbolInfo &Symbol : Symbols)
+      UniqueSymbols.insert(Symbol);
+  };
+
   // Load all symbol files in MergeDir.
+  llvm::ThreadPool Pool;
+  std::error_code EC;
   for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd;
        Dir != DirEnd && !EC; Dir.increment(EC)) {
-    int ReadFD = 0;
-    if (llvm::sys::fs::openFileForRead(Dir->path(), ReadFD)) {
-      llvm::errs() << "Cann't open " << Dir->path() << "\n";
-      continue;
-    }
-    auto Buffer = llvm::MemoryBuffer::getOpenFile(ReadFD, Dir->path(), -1);
-    if (!Buffer)
-      continue;
+    // Do the YAML file parsing in parallel.
+    Pool.async(
+        [&AddSymbols](std::string Path) {
+          auto Buffer = llvm::MemoryBuffer::getFile(Path);
+          if (!Buffer) {
+            llvm::errs() << "Cann't open " << Path << "\n";
+            return;
+          }
           std::vector<SymbolInfo> Symbols =
               ReadSymbolInfosFromYAML(Buffer.get()->getBuffer());
-    for (const auto &Symbol : Symbols)
-      UniqueSymbols.insert(Symbol);
+          // FIXME: Merge without creating such a heavy contention point.
+          AddSymbols(Symbols);
+        },
+        Dir->path());
   }
 
   llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::F_None);
   if (EC) {
     llvm::errs() << "Cann't open '" << OutputFile << "': " << EC.message()
                  << '\n';
     return false;
   }
   WriteSymbolInfosToStream(OS, UniqueSymbols);
   return true;