[Tooling] Support src/buildroot to find the build-root, and expose it.

[sammccall]

Summary

Prototype implementation, no tests yet.
See http://lists.llvm.org/pipermail/cfe-dev/2018-October/059752.html

include/clang/Tooling/CompilationDatabase.h

 
 #ifndef LLVM_CLANG_TOOLING_COMPILATIONDATABASE_H
 #define LLVM_CLANG_TOOLING_COMPILATIONDATABASE_H
 
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Support/VirtualFileSystem.h"
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 namespace clang {
 namespace tooling {
 
+class CompilationDatabase;
+
+struct Codebase {
+  // The directory containing build artifacts.
+  std::string BuildRoot;
+  // Describes how to build the files in the codebase.
+  std::unique_ptr<CompilationDatabase> CompilationDatabase;
+
+  // TODO: make the factory functions vfs-aware.
+
+  // Detect the codebase containing the specified file or directory.
+  // Simply traverses upward until fromDirectory() returns a valid codebase.
+  static llvm::Optional<Codebase> detect(StringRef Path);
+
+  // Returns information about the codebase rooted at a given directory.
+  // Root may be a build root or a source root (build root will be detected).
+  static llvm::Expected<Codebase> inDirectory(StringRef Root);
+};
+
 /// Specifies the working directory and command of a compilation.
 struct CompileCommand {
   CompileCommand() = default;
   CompileCommand(Twine Directory, Twine Filename,
                  std::vector<std::string> CommandLine, Twine Output)
       : Directory(Directory.str()), Filename(Filename.str()),
         CommandLine(std::move(CommandLine)), Output(Output.str()){}
 

 /// for the files in a project.
 ///
 /// Many implementations are enumerable, allowing all command lines to be
 /// retrieved. These can be used to run clang tools over a subset of the files
 /// in a project.
 class CompilationDatabase {
 public:
   virtual ~CompilationDatabase();
-
-  /// Loads a compilation database from a build directory.
-  ///
-  /// Looks at the specified 'BuildDirectory' and creates a compilation database
-  /// that allows to query compile commands for source files in the
-  /// corresponding source tree.
-  ///
-  /// Returns NULL and sets ErrorMessage if we were not able to build up a
-  /// compilation database for the build directory.
-  ///
-  /// FIXME: Currently only supports JSON compilation databases, which
-  /// are named 'compile_commands.json' in the given directory. Extend this
-  /// for other build types (like ninja build files).
-  static std::unique_ptr<CompilationDatabase>
-  loadFromDirectory(StringRef BuildDirectory, std::string &ErrorMessage);
-
-  /// Tries to detect a compilation database location and load it.
-  ///
-  /// Looks for a compilation database in all parent paths of file 'SourceFile'
-  /// by calling loadFromDirectory.
-  static std::unique_ptr<CompilationDatabase>
-  autoDetectFromSource(StringRef SourceFile, std::string &ErrorMessage);
-
-  /// Tries to detect a compilation database location and load it.
-  ///
-  /// Looks for a compilation database in directory 'SourceDir' and all
-  /// its parent paths by calling loadFromDirectory.
-  static std::unique_ptr<CompilationDatabase>
-  autoDetectFromDirectory(StringRef SourceDir, std::string &ErrorMessage);
-
   /// Returns all compile commands in which the specified file was
   /// compiled.
   ///
   /// This includes compile commands that span multiple source files.
   /// For example, consider a project with the following compilations:
   /// $ clang++ -o test a.cc b.cc t.cc
   /// $ clang++ -o production a.cc b.cc -DPRODUCTION
   /// A compilation database representing the project would return both command

include/clang/Tooling/CompilationDatabasePluginRegistry.h

 /// X("my-compilation-database", "Reads my own compilation database");
 /// \endcode
 class CompilationDatabasePlugin {
 public:
   virtual ~CompilationDatabasePlugin();
 
   /// Loads a compilation database from a build directory.
   ///
-  /// \see CompilationDatabase::loadFromDirectory().
+  /// If the database detects that Root is a source directory and the build
+  /// directory is external, it should replace Root with the build directory.
+  /// e.g. if compile_commands.json is a symlink.
   virtual std::unique_ptr<CompilationDatabase>
-  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) = 0;
+  loadFromDirectory(StringRef &Root, std::string &ErrorMessage) = 0;
 };
 
 using CompilationDatabasePluginRegistry =
     llvm::Registry<CompilationDatabasePlugin>;
 
 } // namespace tooling
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_COMPILATIONDATABASEPLUGINREGISTRY_H

lib/Tooling/CommonOptionsParser.cpp

llvm::Error CommonOptionsParser::init(
 
   cl::PrintOptionValues();
 
   SourcePathList = SourcePaths;
   if ((OccurrencesFlag == cl::ZeroOrMore || OccurrencesFlag == cl::Optional) &&
       SourcePathList.empty())
     return llvm::Error::success();
   if (!Compilations) {
-    if (!BuildPath.empty()) {
-      Compilations =
-          CompilationDatabase::autoDetectFromDirectory(BuildPath, ErrorMessage);
+    if (auto Codebase =
+        Codebase::detect(BuildPath.empty() ? SourcePaths[0] : BuildPath)) {
+      Compilations = std::move(Codebase->CompilationDatabase);
     } else {
-      Compilations = CompilationDatabase::autoDetectFromSource(SourcePaths[0],
-                                                               ErrorMessage);
-    }
-    if (!Compilations) {
-      llvm::errs() << "Error while trying to load a compilation database:\n"
-                   << ErrorMessage << "Running without flags.\n";
+      llvm::errs()
+          << "Failed to detect compilation database. Running without flags.\n";
       Compilations.reset(
           new FixedCompilationDatabase(".", std::vector<std::string>()));
     }
   }
   auto AdjustingCompilations =
       llvm::make_unique<ArgumentsAdjustingCompilations>(
           std::move(Compilations));
   Adjuster =

lib/Tooling/CompilationDatabase.cpp

 #include <string>
 #include <system_error>
 #include <utility>
 #include <vector>
 
 using namespace clang;
 using namespace tooling;
 
-LLVM_INSTANTIATE_REGISTRY(CompilationDatabasePluginRegistry)
-
-CompilationDatabase::~CompilationDatabase() = default;
-
-std::unique_ptr<CompilationDatabase>
-CompilationDatabase::loadFromDirectory(StringRef BuildDirectory,
-                                       std::string &ErrorMessage) {
-  llvm::raw_string_ostream ErrorStream(ErrorMessage);
-  for (CompilationDatabasePluginRegistry::iterator
-       It = CompilationDatabasePluginRegistry::begin(),
-       Ie = CompilationDatabasePluginRegistry::end();
-       It != Ie; ++It) {
-    std::string DatabaseErrorMessage;
-    std::unique_ptr<CompilationDatabasePlugin> Plugin(It->instantiate());
-    if (std::unique_ptr<CompilationDatabase> DB =
-            Plugin->loadFromDirectory(BuildDirectory, DatabaseErrorMessage))
-      return DB;
-    ErrorStream << It->getName() << ": " << DatabaseErrorMessage << "\n";
-  }
-  return nullptr;
-}
-
-static std::unique_ptr<CompilationDatabase>
-findCompilationDatabaseFromDirectory(StringRef Directory,
-                                     std::string &ErrorMessage) {
-  std::stringstream ErrorStream;
-  bool HasErrorMessage = false;
-  while (!Directory.empty()) {
-    std::string LoadErrorMessage;
-
-    if (std::unique_ptr<CompilationDatabase> DB =
-            CompilationDatabase::loadFromDirectory(Directory, LoadErrorMessage))
-      return DB;
-
-    if (!HasErrorMessage) {
-      ErrorStream << "No compilation database found in " << Directory.str()
-                  << " or any parent directory\n" << LoadErrorMessage;
-      HasErrorMessage = true;
-    }
-
-    Directory = llvm::sys::path::parent_path(Directory);
-  }
-  ErrorMessage = ErrorStream.str();
-  return nullptr;
+llvm::Optional<Codebase>
+Codebase::detect(StringRef Path) {
+  for (; !Path.empty(); Path = llvm::sys::path::parent_path(Path))
+    if (auto Info = inDirectory(Path))
+      return std::move(*Info);
+    else
+      consumeError(Info.takeError());
+  return llvm::None;
+}
+
+llvm::Expected<Codebase> Codebase::inDirectory(StringRef Root) {
+  SmallString<256> BuildRoot = Root;
+  llvm::sys::path::append(BuildRoot, "buildroot");
+  if (llvm::sys::fs::is_directory(BuildRoot))
+    Root = BuildRoot;
+
+  std::string Errs;
+  llvm::raw_string_ostream ErrsStream(Errs);
+  for (const auto& Plugin : CompilationDatabasePluginRegistry::entries()) {
+    std::string Err;
+    if (auto CDB = Plugin.instantiate()->loadFromDirectory(Root, Err)) {
+      Codebase Info;
+      Info.BuildRoot = Root;
+      Info.CompilationDatabase = std::move(CDB);
+      return Info;
+    } else
+      ErrsStream << Plugin.getName() << ": " << Err << "\n";
+  return llvm::make_error<llvm::StringError>(ErrsStream.str(),
+                                             llvm::inconvertibleErrorCode());
 }
 
-std::unique_ptr<CompilationDatabase>
-CompilationDatabase::autoDetectFromSource(StringRef SourceFile,
-                                          std::string &ErrorMessage) {
-  SmallString<1024> AbsolutePath(getAbsolutePath(SourceFile));
-  StringRef Directory = llvm::sys::path::parent_path(AbsolutePath);
-
-  std::unique_ptr<CompilationDatabase> DB =
-      findCompilationDatabaseFromDirectory(Directory, ErrorMessage);
-
-  if (!DB)
-    ErrorMessage = ("Could not auto-detect compilation database for file \"" +
-                   SourceFile + "\"\n" + ErrorMessage).str();
-  return DB;
-}
+LLVM_INSTANTIATE_REGISTRY(CompilationDatabasePluginRegistry)
 
-std::unique_ptr<CompilationDatabase>
-CompilationDatabase::autoDetectFromDirectory(StringRef SourceDir,
-                                             std::string &ErrorMessage) {
-  SmallString<1024> AbsolutePath(getAbsolutePath(SourceDir));
-
-  std::unique_ptr<CompilationDatabase> DB =
-      findCompilationDatabaseFromDirectory(AbsolutePath, ErrorMessage);
-
-  if (!DB)
-    ErrorMessage = ("Could not auto-detect compilation database from directory \"" +
-                   SourceDir + "\"\n" + ErrorMessage).str();
-  return DB;
-}
+CompilationDatabase::~CompilationDatabase() = default;
 
 std::vector<CompileCommand> CompilationDatabase::getAllCompileCommands() const {
   std::vector<CompileCommand> Result;
   for (const auto &File : getAllFiles()) {
     auto C = getCompileCommands(File);
     std::move(C.begin(), C.end(), std::back_inserter(Result));
   }
   return Result;
FixedCompilationDatabase::getCompileCommands(StringRef FilePath) const {
   Result[0].Filename = FilePath;
   return Result;
 }
 
 namespace {
 
 class FixedCompilationDatabasePlugin : public CompilationDatabasePlugin {
   std::unique_ptr<CompilationDatabase>
-  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
+  loadFromDirectory(StringRef &Directory, std::string &ErrorMessage) override {
     SmallString<1024> DatabasePath(Directory);
     llvm::sys::path::append(DatabasePath, "compile_flags.txt");
     return FixedCompilationDatabase::loadFromFile(DatabasePath, ErrorMessage);
   }
 };
 
 } // namespace
 

lib/Tooling/JSONCompilationDatabase.cpp

 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <memory>
std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
   CommandLineArgumentParser parser(EscapedCommandLine);
   return parser.parse();
 }
 
 // This plugin locates a nearby compile_command.json file, and also infers
 // compile commands for files not present in the database.
 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
   std::unique_ptr<CompilationDatabase>
-  loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
+  loadFromDirectory(StringRef &Directory, std::string &ErrorMessage) override {
     SmallString<1024> JSONDatabasePath(Directory);
     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
     auto Base = JSONCompilationDatabase::loadFromFile(
         JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
-    return Base ? inferMissingCompileCommands(std::move(Base)) : nullptr;
+    if (!Base)
+      return nullptr;
+    // If compile_commands.json is a symlink, target's parent is the build root.
+    if (llvm::sys::fs::is_symlink_file(JSONDatabasePath)) {
+      llvm::sys::fs::real_path(JSONDatabasePath, JSONDatabasePath);
+      Directory = llvm::sys::path::parent_path(JSONDatabasePath);
+    }
+    return inferMissingCompileCommands(std::move(Base));
   }
 };
 
 } // namespace
 
 // Register the JSONCompilationDatabasePlugin with the
 // CompilationDatabasePluginRegistry using this statically initialized variable.
 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>

tools/libclang/CXCompilationDatabase.cpp

 #include "clang-c/CXCompilationDatabase.h"
 #include "CXString.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include <cstdio>
 
 using namespace clang;
 using namespace clang::tooling;
 
 // FIXME: do something more useful with the error message
 CXCompilationDatabase
 clang_CompilationDatabase_fromDirectory(const char *BuildDir,
                                         CXCompilationDatabase_Error *ErrorCode)
 {
   std::string ErrorMsg;
   CXCompilationDatabase_Error Err = CXCompilationDatabase_NoError;
 
-  std::unique_ptr<CompilationDatabase> db =
-      CompilationDatabase::loadFromDirectory(BuildDir, ErrorMsg);
+  auto Codebase = Codebase::inDirectory(BuildDir);
 
-  if (!db) {
-    fprintf(stderr, "LIBCLANG TOOLING ERROR: %s\n", ErrorMsg.c_str());
+  if (!Codebase) {
+    fprintf(stderr, "LIBCLANG TOOLING ERROR: %s\n",
+            llvm::toString(Codebase.takeError()).c_str());
     Err = CXCompilationDatabase_CanNotLoadDatabase;
   }
 
   if (ErrorCode)
     *ErrorCode = Err;
 
-  return db.release();
+  return Codebase->CompilationDatabase.release();
 }
 
 void
 clang_CompilationDatabase_dispose(CXCompilationDatabase CDb)
 {
   delete static_cast<CompilationDatabase *>(CDb);
 }