Fix unittests after windows command line parsing

[zturner]

Summary

After submitting the windows command line parsing patch, it broke all the command line parsing unittests, because now they were trying parse gnu command lines using the windows command line parser. The patch is reverted for now until we get these unit tests fixed.

I'm not sure what the best solution is. One option is to disable those tests on Windows. But that seems undesirable because it was only some of the tests, and others worked fine. Not all the tests even depended on a command line in the first place. So I decided to expose the command syntax through the API. You can either force it to Gnu, force it to Windows, and let it auto-detect. The unit tests all force it to Gnu for the sake of simplicity, since the Windows command line tokenizer is already rigorously tested in LLVM.

This also kind of lines up with the idea I had in the previous code review for exposing a command line option in CommonOptions.cpp to override the command syntax so we don't have to guess. So if someone decides to do that in the future that should be easier now.

[alexfh]

LG

include/clang/Tooling/JSONCompilationDatabase.h

 /// considered to be the main source file for the translation unit.
 ///
 /// 'command' is a full command line that will be unescaped.
 ///
 /// 'arguments' is a list of command line arguments that will not be unescaped.
 ///
 /// JSON compilation databases can for example be generated in CMake projects
 /// by setting the flag -DCMAKE_EXPORT_COMPILE_COMMANDS.
+enum class JSONCommandLineSyntax { Windows, Gnu, AutoDetect };
 class JSONCompilationDatabase : public CompilationDatabase {
 public:
   /// \brief Loads a JSON compilation database from the specified file.
   ///
   /// Returns NULL and sets ErrorMessage if the database could not be
   /// loaded from the given file.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromFile(StringRef FilePath, std::string &ErrorMessage);
+  loadFromFile(StringRef FilePath, std::string &ErrorMessage,
+               JSONCommandLineSyntax Syntax);
 
   /// \brief Loads a JSON compilation database from a data buffer.
   ///
   /// Returns NULL and sets ErrorMessage if the database could not be loaded.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage);
+  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage,
+                 JSONCommandLineSyntax Syntax);
 
   /// \brief Returns all compile comamnds in which the specified file was
   /// compiled.
   ///
   /// FIXME: Currently FilePath must be an absolute path inside the
   /// source directory which does not have symlinks resolved.
   std::vector<CompileCommand>
   getCompileCommands(StringRef FilePath) const override;
 
   /// \brief Returns the list of all files available in the compilation database.
   ///
   /// These are the 'file' entries of the JSON objects.
   std::vector<std::string> getAllFiles() const override;
 
   /// \brief Returns all compile commands for all the files in the compilation
   /// database.
   std::vector<CompileCommand> getAllCompileCommands() const override;
 
 private:
   /// \brief Constructs a JSON compilation database on a memory buffer.
-  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database)
-      : Database(std::move(Database)),
+  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database,
+                          JSONCommandLineSyntax Syntax)
+      : Database(std::move(Database)), Syntax(Syntax),
         YAMLStream(this->Database->getBuffer(), SM) {}
 
   /// \brief Parses the database file and creates the index.
   ///
   /// Returns whether parsing succeeded. Sets ErrorMessage if parsing
   /// failed.
   bool parse(std::string &ErrorMessage);
 
private:
 
   /// All the compile commands in the order that they were provided in the
   /// JSON stream.
   std::vector<CompileCommandRef> AllCommands;
 
   FileMatchTrie MatchTrie;
 
   std::unique_ptr<llvm::MemoryBuffer> Database;
+  JSONCommandLineSyntax Syntax;
   llvm::SourceMgr SM;
   llvm::yaml::Stream YAMLStream;
 };
 
 } // end namespace tooling
 } // end namespace clang
 
 #endif

lib/Tooling/JSONCompilationDatabase.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/JSONCompilationDatabase.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 #include <system_error>
 
 namespace clang {
 namespace tooling {
 
 namespace {
 
 /// \brief A parser for escaped strings of command line arguments.
bool next() {
     return Position != Input.end();
   }
 
   const StringRef Input;
   StringRef::iterator Position;
   std::vector<std::string> CommandLine;
 };
 
-std::vector<std::string> unescapeCommandLine(
+std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
                                              StringRef EscapedCommandLine) {
+  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
+    llvm::Triple Triple(llvm::sys::getProcessTriple());
+    if (Triple.getOS() == llvm::Triple::OSType::Win32) {
+      // Assume Windows command line parsing on Win32 unless the triple
+      // explicitly
+      // tells us otherwise.
+      if (!Triple.hasEnvironment() ||
+          Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
+        Syntax = JSONCommandLineSyntax::Windows;
+      else
+        Syntax = JSONCommandLineSyntax::Gnu;
+    }
+  }
+
+  if (Syntax == JSONCommandLineSyntax::Windows) {
+    llvm::BumpPtrAllocator Alloc;
+    llvm::StringSaver Saver(Alloc);
+    llvm::SmallVector<const char *, 64> T;
+    llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
+    std::vector<std::string> Result(T.begin(), T.end());
+    return Result;
+  }
+  assert(Syntax == JSONCommandLineSyntax::Gnu);
   CommandLineArgumentParser parser(EscapedCommandLine);
   return parser.parse();
 }
 
 class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
   std::unique_ptr<CompilationDatabase>
   loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
     SmallString<1024> JSONDatabasePath(Directory);
     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
     std::unique_ptr<CompilationDatabase> Database(
-        JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
+        JSONCompilationDatabase::loadFromFile(
+            JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect));
     if (!Database)
       return nullptr;
     return Database;
   }
 };
 
 } // end namespace
 
 // Register the JSONCompilationDatabasePlugin with the
 // CompilationDatabasePluginRegistry using this statically initialized variable.
 static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
 X("json-compilation-database", "Reads JSON formatted compilation databases");
 
 // This anchor is used to force the linker to link in the generated object file
 // and thus register the JSONCompilationDatabasePlugin.
 volatile int JSONAnchorSource = 0;
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromFile(StringRef FilePath,
-                                      std::string &ErrorMessage) {
+                                      std::string &ErrorMessage,
+                                      JSONCommandLineSyntax Syntax) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
       llvm::MemoryBuffer::getFile(FilePath);
   if (std::error_code Result = DatabaseBuffer.getError()) {
     ErrorMessage = "Error while opening JSON database: " + Result.message();
     return nullptr;
   }
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(*DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
 }
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
-                                        std::string &ErrorMessage) {
+                                        std::string &ErrorMessage,
+                                        JSONCommandLineSyntax Syntax) {
   std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
       llvm::MemoryBuffer::getMemBuffer(DatabaseString));
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
 }
 
 std::vector<CompileCommand>
 JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
   SmallString<128> NativeFilePath;

 std::vector<CompileCommand>
 JSONCompilationDatabase::getAllCompileCommands() const {
   std::vector<CompileCommand> Commands;
   getCommands(AllCommands, Commands);
   return Commands;
 }
 
 static std::vector<std::string>
-nodeToCommandLine(const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
+nodeToCommandLine(JSONCommandLineSyntax Syntax,
+                  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
   SmallString<1024> Storage;
   if (Nodes.size() == 1) {
-    return unescapeCommandLine(Nodes[0]->getValue(Storage));
+    return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
   }
   std::vector<std::string> Arguments;
   for (auto *Node : Nodes) {
     Arguments.push_back(Node->getValue(Storage));
   }
   return Arguments;
 }
 
 void JSONCompilationDatabase::getCommands(
     ArrayRef<CompileCommandRef> CommandsRef,
     std::vector<CompileCommand> &Commands) const {
   for (int I = 0, E = CommandsRef.size(); I != E; ++I) {
     SmallString<8> DirectoryStorage;
     SmallString<32> FilenameStorage;
     Commands.emplace_back(
         std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
         std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
-      nodeToCommandLine(std::get<2>(CommandsRef[I])));
+        nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I])));
   }
 }
 
 bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
   llvm::yaml::document_iterator I = YAMLStream.begin();
   if (I == YAMLStream.end()) {
     ErrorMessage = "Error while parsing YAML.";
     return false;

unittests/Tooling/CompilationDatabaseTest.cpp

 #include "llvm/Support/Path.h"
 #include "gtest/gtest.h"
 
 namespace clang {
 namespace tooling {
 
 static void expectFailure(StringRef JSONDatabase, StringRef Explanation) {
   std::string ErrorMessage;
-  EXPECT_EQ(nullptr, JSONCompilationDatabase::loadFromBuffer(JSONDatabase,
-                                                             ErrorMessage))
+  EXPECT_EQ(nullptr,
+            JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                                    JSONCommandLineSyntax::Gnu))
       << "Expected an error because of: " << Explanation.str();
 }
 
 TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) {
   expectFailure("", "Empty database");
   expectFailure("{", "Invalid JSON");
   expectFailure("[[]]", "Array instead of object");
   expectFailure("[{\"a\":[]}]", "Array instead of value");
   expectFailure("[{\"a\":\"b\"}]", "Unknown key");
   expectFailure("[{[]:\"\"}]", "Incorrectly typed entry");
   expectFailure("[{}]", "Empty entry");
   expectFailure("[{\"directory\":\"\",\"command\":\"\"}]", "Missing file");
   expectFailure("[{\"directory\":\"\",\"file\":\"\"}]", "Missing command or arguments");
   expectFailure("[{\"command\":\"\",\"file\":\"\"}]", "Missing directory");
   expectFailure("[{\"directory\":\"\",\"arguments\":[]}]", "Missing file");
   expectFailure("[{\"arguments\":\"\",\"file\":\"\"}]", "Missing directory");
   expectFailure("[{\"directory\":\"\",\"arguments\":\"\",\"file\":\"\"}]", "Arguments not array");
   expectFailure("[{\"directory\":\"\",\"command\":[],\"file\":\"\"}]", "Command not string");
   expectFailure("[{\"directory\":\"\",\"arguments\":[[]],\"file\":\"\"}]",
                 "Arguments contain non-string");
 }
 
 static std::vector<std::string> getAllFiles(StringRef JSONDatabase,
-                                            std::string &ErrorMessage) {
+                                            std::string &ErrorMessage,
+                                            JSONCommandLineSyntax Syntax) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<std::string>();
   }
   return Database->getAllFiles();
 }
 
-static std::vector<CompileCommand> getAllCompileCommands(StringRef JSONDatabase,
-                                                    std::string &ErrorMessage) {
+static std::vector<CompileCommand>
+getAllCompileCommands(JSONCommandLineSyntax Syntax, StringRef JSONDatabase,
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<CompileCommand>();
   }
   return Database->getAllCompileCommands();
 }
 
 TEST(JSONCompilationDatabase, GetAllFiles) {
   std::string ErrorMessage;
   EXPECT_EQ(std::vector<std::string>(),
-            getAllFiles("[]", ErrorMessage)) << ErrorMessage;
+            getAllFiles("[]", ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 
   std::vector<std::string> expected_files;
   SmallString<16> PathStorage;
   llvm::sys::path::native("//net/dir/file1", PathStorage);
   expected_files.push_back(PathStorage.str());
   llvm::sys::path::native("//net/dir/file2", PathStorage);
   expected_files.push_back(PathStorage.str());
-  EXPECT_EQ(expected_files, getAllFiles(
-    "[{\"directory\":\"//net/dir\","
+  EXPECT_EQ(expected_files,
+            getAllFiles("[{\"directory\":\"//net/dir\","
                         "\"command\":\"command\","
                         "\"file\":\"file1\"},"
                         " {\"directory\":\"//net/dir\","
                         "\"command\":\"command\","
                         "\"file\":\"file2\"}]",
-    ErrorMessage)) << ErrorMessage;
+                        ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 }
 
 TEST(JSONCompilationDatabase, GetAllCompileCommands) {
   std::string ErrorMessage;
-  EXPECT_EQ(0u,
-            getAllCompileCommands("[]", ErrorMessage).size()) << ErrorMessage;
+  EXPECT_EQ(
+      0u, getAllCompileCommands(JSONCommandLineSyntax::Gnu, "[]", ErrorMessage)
+              .size())
+      << ErrorMessage;
 
   StringRef Directory1("//net/dir1");
   StringRef FileName1("file1");
   StringRef Command1("command1");
   StringRef Directory2("//net/dir2");
   StringRef FileName2("file2");
   StringRef Command2("command2");
 
   std::vector<CompileCommand> Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"},"
-       " {\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory1 + "\"," + "\"command\":\"" + Command1 +
+       "\","
+       "\"file\":\"" +
+       FileName1 + "\"},"
+                   " {\"directory\":\"" +
+       Directory2 + "\"," + "\"command\":\"" + Command2 + "\","
+                                                          "\"file\":\"" +
+       FileName2 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory1, Commands[0].Directory) << ErrorMessage;
   EXPECT_EQ(FileName1, Commands[0].Filename) << ErrorMessage;
   ASSERT_EQ(1u, Commands[0].CommandLine.size());
   EXPECT_EQ(Command1, Commands[0].CommandLine[0]) << ErrorMessage;
   EXPECT_EQ(Directory2, Commands[1].Directory) << ErrorMessage;
   EXPECT_EQ(FileName2, Commands[1].Filename) << ErrorMessage;
   ASSERT_EQ(1u, Commands[1].CommandLine.size());
   EXPECT_EQ(Command2, Commands[1].CommandLine[0]) << ErrorMessage;
 
   // Check that order is preserved.
   Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"},"
-       " {\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory2 + "\"," + "\"command\":\"" + Command2 +
+       "\","
+       "\"file\":\"" +
+       FileName2 + "\"},"
+                   " {\"directory\":\"" +
+       Directory1 + "\"," + "\"command\":\"" + Command1 + "\","
+                                                          "\"file\":\"" +
+       FileName1 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory2, Commands[0].Directory) << ErrorMessage;
   EXPECT_EQ(FileName2, Commands[0].Filename) << ErrorMessage;
   ASSERT_EQ(1u, Commands[0].CommandLine.size());
   EXPECT_EQ(Command2, Commands[0].CommandLine[0]) << ErrorMessage;
   EXPECT_EQ(Directory1, Commands[1].Directory) << ErrorMessage;
   EXPECT_EQ(FileName1, Commands[1].Filename) << ErrorMessage;
   ASSERT_EQ(1u, Commands[1].CommandLine.size());
   EXPECT_EQ(Command1, Commands[1].CommandLine[0]) << ErrorMessage;
 }
 
 static CompileCommand findCompileArgsInJsonDatabase(StringRef FileName,
                                                     StringRef JSONDatabase,
                                                     std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              JSONCommandLineSyntax::Gnu));
   if (!Database)
     return CompileCommand();
   std::vector<CompileCommand> Commands = Database->getCompileCommands(FileName);
   EXPECT_LE(Commands.size(), 1u);
   if (Commands.empty())
     return CompileCommand();
   return Commands[0];
 }