[clang][Tooling] Infer target and mode from argv[0] when using JSONCompilationDatabase

[kadircet]

Summary

Wraps JSON compilation database with a target and mode adding database
wrapper. So that driver can correctly figure out which toolchain to use.

Note that clients that wants to make use of this target discovery mechanism
needs to link in TargetsInfos and initialize them at startup.

[ilya-biryukov]

LGTM

[kadircet]

• Rename file
• Fix tests

cfe/trunk/bindings/python/tests/cindex/test_cdb.py

class TestCDB(unittest.TestCase):
     def test_all_compilecommand(self):
         """Check we get all results from the db"""
         cdb = CompilationDatabase.fromDirectory(kInputsDir)
         cmds = cdb.getAllCompileCommands()
         self.assertEqual(len(cmds), 3)
         expected = [
             { 'wd': '/home/john.doe/MyProject',
               'file': '/home/john.doe/MyProject/project.cpp',
-              'line': ['clang++', '-o', 'project.o', '-c',
+              'line': ['clang++', '--driver-mode=g++', '-o', 'project.o', '-c',
                        '/home/john.doe/MyProject/project.cpp']},
             { 'wd': '/home/john.doe/MyProjectA',
               'file': '/home/john.doe/MyProject/project2.cpp',
-              'line': ['clang++', '-o', 'project2.o', '-c',
+              'line': ['clang++', '--driver-mode=g++', '-o', 'project2.o', '-c',
                        '/home/john.doe/MyProject/project2.cpp']},
             { 'wd': '/home/john.doe/MyProjectB',
               'file': '/home/john.doe/MyProject/project2.cpp',
-              'line': ['clang++', '-DFEATURE=1', '-o', 'project2-feature.o', '-c',
+              'line': ['clang++', '--driver-mode=g++', '-DFEATURE=1', '-o',
+                       'project2-feature.o', '-c',
                        '/home/john.doe/MyProject/project2.cpp']},
 
             ]
         for i in range(len(cmds)):
             self.assertEqual(cmds[i].directory, expected[i]['wd'])
             self.assertEqual(cmds[i].filename, expected[i]['file'])
             for arg, exp in zip(cmds[i].arguments, expected[i]['line']):
                 self.assertEqual(arg, exp)
 
     def test_1_compilecommand(self):
         """Check file with single compile command"""
         cdb = CompilationDatabase.fromDirectory(kInputsDir)
         file = '/home/john.doe/MyProject/project.cpp'
         cmds = cdb.getCompileCommands(file)
         self.assertEqual(len(cmds), 1)
         self.assertEqual(cmds[0].directory, os.path.dirname(file))
         self.assertEqual(cmds[0].filename, file)
-        expected = [ 'clang++', '-o', 'project.o', '-c',
+        expected = [ 'clang++', '--driver-mode=g++', '-o', 'project.o', '-c',
                      '/home/john.doe/MyProject/project.cpp']
         for arg, exp in zip(cmds[0].arguments, expected):
             self.assertEqual(arg, exp)
 
     def test_2_compilecommand(self):
         """Check file with 2 compile commands"""
         cdb = CompilationDatabase.fromDirectory(kInputsDir)
         cmds = cdb.getCompileCommands('/home/john.doe/MyProject/project2.cpp')
         self.assertEqual(len(cmds), 2)
         expected = [
             { 'wd': '/home/john.doe/MyProjectA',
-              'line': ['clang++', '-o', 'project2.o', '-c',
+              'line': ['clang++', '--driver-mode=g++', '-o', 'project2.o', '-c',
                        '/home/john.doe/MyProject/project2.cpp']},
             { 'wd': '/home/john.doe/MyProjectB',
-              'line': ['clang++', '-DFEATURE=1', '-o', 'project2-feature.o', '-c',
+              'line': ['clang++', '--driver-mode=g++', '-DFEATURE=1', '-o',
+                       'project2-feature.o', '-c',
                        '/home/john.doe/MyProject/project2.cpp']}
             ]
         for i in range(len(cmds)):
             self.assertEqual(cmds[i].directory, expected[i]['wd'])
             for arg, exp in zip(cmds[i].arguments, expected[i]['line']):
                 self.assertEqual(arg, exp)
 
     def test_compilecommand_iterator_stops(self):

cfe/trunk/include/clang/Tooling/CompilationDatabase.h

 
 /// Returns a wrapped CompilationDatabase that defers to the provided one,
 /// but getCompileCommands() will infer commands for unknown files.
 /// The return value of getAllFiles() or getAllCompileCommands() is unchanged.
 /// See InterpolatingCompilationDatabase.cpp for details on heuristics.
 std::unique_ptr<CompilationDatabase>
     inferMissingCompileCommands(std::unique_ptr<CompilationDatabase>);
 
+/// Returns a wrapped CompilationDatabase that will add -target and -mode flags
+/// to commandline when they can be deduced from argv[0] of commandline returned
+/// by underlying database.
+std::unique_ptr<CompilationDatabase>
+inferTargetAndDriverMode(std::unique_ptr<CompilationDatabase> Base);
+
 } // namespace tooling
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_COMPILATIONDATABASE_H

cfe/trunk/lib/Tooling/CMakeLists.txt

 add_clang_library(clangTooling
   AllTUsExecution.cpp
   ArgumentsAdjusters.cpp
   CommonOptionsParser.cpp
   CompilationDatabase.cpp
   Execution.cpp
   FileMatchTrie.cpp
   FixIt.cpp
+  GuessTargetAndModeCompilationDatabase.cpp
   InterpolatingCompilationDatabase.cpp
   JSONCompilationDatabase.cpp
   Refactoring.cpp
   RefactoringCallbacks.cpp
   StandaloneExecution.cpp
   Tooling.cpp
 
   DEPENDS

cfe/trunk/lib/Tooling/GuessTargetAndModeCompilationDatabase.cpp

+//===- GuessTargetAndModeCompilationDatabase.cpp --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include <memory>
+
+namespace clang {
+namespace tooling {
+
+namespace {
+class TargetAndModeAdderDatabase : public CompilationDatabase {
+public:
+  TargetAndModeAdderDatabase(std::unique_ptr<CompilationDatabase> Base)
+      : Base(std::move(Base)) {
+    assert(this->Base != nullptr);
+  }
+
+  std::vector<std::string> getAllFiles() const override {
+    return Base->getAllFiles();
+  }
+
+  std::vector<CompileCommand> getAllCompileCommands() const override {
+    return addTargetAndMode(Base->getAllCompileCommands());
+  }
+
+  std::vector<CompileCommand>
+  getCompileCommands(StringRef FilePath) const override {
+    return addTargetAndMode(Base->getCompileCommands(FilePath));
+  }
+
+private:
+  std::vector<CompileCommand>
+  addTargetAndMode(std::vector<CompileCommand> Cmds) const {
+    for (auto &Cmd : Cmds) {
+      if (Cmd.CommandLine.empty())
+        continue;
+      addTargetAndModeForProgramName(Cmd.CommandLine, Cmd.CommandLine.front());
+    }
+    return Cmds;
+  }
+  std::unique_ptr<CompilationDatabase> Base;
+};
+} // namespace
+
+std::unique_ptr<CompilationDatabase>
+inferTargetAndDriverMode(std::unique_ptr<CompilationDatabase> Base) {
+  return llvm::make_unique<TargetAndModeAdderDatabase>(std::move(Base));
+}
+
+} // namespace tooling
+} // namespace clang

cfe/trunk/lib/Tooling/JSONCompilationDatabase.cpp

 //===- JSONCompilationDatabase.cpp ----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file contains the implementation of the JSONCompilationDatabase.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/JSONCompilationDatabase.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
+#include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorOr.h"

 // compile commands for files not present in the database.
 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
   std::unique_ptr<CompilationDatabase>
   loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
     SmallString<1024> JSONDatabasePath(Directory);
     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
     auto Base = JSONCompilationDatabase::loadFromFile(
         JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
-    return Base ? inferMissingCompileCommands(std::move(Base)) : nullptr;
+    return Base ? inferTargetAndDriverMode(
+                      inferMissingCompileCommands(std::move(Base)))
+                : nullptr;
   }
 };
 
 } // namespace
 
 // Register the JSONCompilationDatabasePlugin with the
 // CompilationDatabasePluginRegistry using this statically initialized variable.
 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>

cfe/trunk/unittests/Tooling/CompilationDatabaseTest.cpp

 //===- unittest/Tooling/CompilationDatabaseTest.cpp -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclGroup.h"
 #include "clang/Frontend/FrontendAction.h"
+#include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/FileMatchTrie.h"
 #include "clang/Tooling/JSONCompilationDatabase.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TargetSelect.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 namespace clang {
 namespace tooling {
 
 using testing::ElementsAre;
 using testing::EndsWith;
struct MemCDB : public CompilationDatabase {
   std::vector<std::string> getAllFiles() const override {
     std::vector<std::string> Result;
     for (const auto &Entry : Entries)
       Result.push_back(Entry.first());
     return Result;
   }
 };
 
-class InterpolateTest : public ::testing::Test {
+class MemDBTest : public ::testing::Test {
 protected:
   // Adds an entry to the underlying compilation database.
   // A flag is injected: -D <File>, so the command used can be identified.
   void add(StringRef File, StringRef Clang, StringRef Flags) {
     SmallVector<StringRef, 8> Argv = {Clang, File, "-D", File};
     llvm::SplitString(Flags, Argv);
 
     SmallString<32> Dir;
std::string path(llvm::SmallString<32> File) {
     llvm::SmallString<32> Dir;
     llvm::sys::path::system_temp_directory(false, Dir);
     llvm::sys::path::native(File);
     llvm::SmallString<64> Result;
     llvm::sys::path::append(Result, Dir, File);
     return Result.str();
   }
 
+  MemCDB::EntryMap Entries;
+};
+
+class InterpolateTest : public MemDBTest {
+protected:
   // Look up the command from a relative path, and return it in string form.
   // The input file is not included in the returned command.
   std::string getCommand(llvm::StringRef F) {
     auto Results =
         inferMissingCompileCommands(llvm::make_unique<MemCDB>(Entries))
             ->getCompileCommands(path(F));
     if (Results.empty())
       return "none";
std::string getProxy(llvm::StringRef F) {
     llvm::SmallString<32> TempDir;
     llvm::sys::path::system_temp_directory(false, TempDir);
     Proxy.consume_front(TempDir);
     Proxy.consume_front(llvm::sys::path::get_separator());
     llvm::SmallString<32> Result = Proxy;
     llvm::sys::path::native(Result, llvm::sys::path::Style::posix);
     return Result.str();
   }
-
-  MemCDB::EntryMap Entries;
 };
 
 TEST_F(InterpolateTest, Nearby) {
   add("dir/foo.cpp");
   add("dir/bar.cpp");
   add("an/other/foo.cpp");
 
   // great: dir and name both match (prefix or full, case insensitive)
TEST(CompileCommandTest, EqualityOperator) {
   EXPECT_TRUE(CCRef != CCTest);
 
   CCTest = CCRef;
   CCTest.Output = "bonjour.o";
   EXPECT_FALSE(CCRef == CCTest);
   EXPECT_TRUE(CCRef != CCTest);
 }
 
+class TargetAndModeTest : public MemDBTest {
+public:
+  TargetAndModeTest() { llvm::InitializeAllTargetInfos(); }
+
+protected:
+  // Look up the command from a relative path, and return it in string form.
+  std::string getCommand(llvm::StringRef F) {
+    auto Results = inferTargetAndDriverMode(llvm::make_unique<MemCDB>(Entries))
+                       ->getCompileCommands(path(F));
+    if (Results.empty())
+      return "none";
+    return llvm::join(Results[0].CommandLine, " ");
+  }
+};
+
+TEST_F(TargetAndModeTest, TargetAndMode) {
+  add("foo.cpp", "clang-cl", "");
+  add("bar.cpp", "x86_64-linux-clang", "");
+
+  EXPECT_EQ(getCommand("foo.cpp"),
+            "clang-cl --driver-mode=cl foo.cpp -D foo.cpp");
+  EXPECT_EQ(getCommand("bar.cpp"),
+            "x86_64-linux-clang -target x86_64-linux bar.cpp -D bar.cpp");
+}
+
 } // end namespace tooling
 } // end namespace clang