diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp --- a/clang-tools-extra/clangd/index/Background.cpp +++ b/clang-tools-extra/clangd/index/Background.cpp @@ -25,6 +25,7 @@ #include "index/SymbolCollector.h" #include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/StringMap.h" @@ -373,6 +374,11 @@ Shard.Relations = RelS.get(); Shard.Sources = IG.get(); + // Only store command line hash for main files of the TU, since our + // current model keeps only one version of a header file. + if (Path == MainFile) + Shard.Cmd = Index.Cmd.getPointer(); + if (auto Error = IndexStorage->storeShard(Path, Shard)) elog("Failed to write background-index shard for file {0}: {1}", Path, std::move(Error)); @@ -479,6 +485,7 @@ Action->EndSourceFile(); + Index.Cmd = Inputs.CompileCommand; assert(Index.Symbols && Index.Refs && Index.Sources && "Symbols, Refs and Sources must be set."); diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h --- a/clang-tools-extra/clangd/index/Serialization.h +++ b/clang-tools-extra/clangd/index/Serialization.h @@ -27,6 +27,7 @@ #include "Headers.h" #include "Index.h" #include "index/Symbol.h" +#include "clang/Tooling/CompilationDatabase.h" #include "llvm/Support/Error.h" namespace clang { @@ -44,6 +45,8 @@ llvm::Optional Relations; // Keys are URIs of the source files. llvm::Optional Sources; + // This contains only the Directory and CommandLine. + llvm::Optional Cmd; }; // Parse an index file. The input must be a RIFF or YAML file. llvm::Expected readIndexFile(llvm::StringRef); @@ -57,13 +60,15 @@ const IncludeGraph *Sources = nullptr; // TODO: Support serializing Dex posting lists. IndexFileFormat Format = IndexFileFormat::RIFF; + const tooling::CompileCommand *Cmd = nullptr; IndexFileOut() = default; IndexFileOut(const IndexFileIn &I) : Symbols(I.Symbols ? I.Symbols.getPointer() : nullptr), Refs(I.Refs ? I.Refs.getPointer() : nullptr), Relations(I.Relations ? I.Relations.getPointer() : nullptr), - Sources(I.Sources ? I.Sources.getPointer() : nullptr) {} + Sources(I.Sources ? I.Sources.getPointer() : nullptr), + Cmd(I.Cmd ? I.Cmd.getPointer() : nullptr) {} }; // Serializes an index file. llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O); diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp --- a/clang-tools-extra/clangd/index/Serialization.cpp +++ b/clang-tools-extra/clangd/index/Serialization.cpp @@ -13,9 +13,13 @@ #include "SymbolOrigin.h" #include "Trace.h" #include "dex/Dex.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include namespace clang { namespace clangd { @@ -403,6 +407,30 @@ return {Subject, Predicate, Object}; } +struct InternedCompileCommand { + llvm::StringRef Directory; + std::vector CommandLine; +}; + +void writeCompileCommand(const InternedCompileCommand &Cmd, + const StringTableOut &Strings, + llvm::raw_ostream &CmdOS) { + writeVar(Strings.index(Cmd.Directory), CmdOS); + writeVar(Cmd.CommandLine.size(), CmdOS); + for (llvm::StringRef C : Cmd.CommandLine) + writeVar(Strings.index(C), CmdOS); +} + +InternedCompileCommand +readCompileCommand(Reader CmdReader, llvm::ArrayRef Strings) { + InternedCompileCommand Cmd; + Cmd.Directory = CmdReader.consumeString(Strings); + Cmd.CommandLine.resize(CmdReader.consumeVar()); + for (llvm::StringRef &C : Cmd.CommandLine) + C = CmdReader.consumeString(Strings); + return Cmd; +} + // FILE ENCODING // A file is a RIFF chunk with type 'CdIx'. // It contains the sections: @@ -415,7 +443,7 @@ // The current versioning scheme is simple - non-current versions are rejected. // If you make a breaking change, bump this version number to invalidate stored // data. Later we may want to support some backward compatibility. -constexpr static uint32_t Version = 10; +constexpr static uint32_t Version = 11; llvm::Expected readRIFF(llvm::StringRef Data) { auto RIFF = riff::readFile(Data); @@ -490,6 +518,18 @@ return makeError("malformed or truncated relations"); Result.Relations = std::move(Relations).build(); } + if (Chunks.count("cmdl")) { + Reader CmdReader(Chunks.lookup("cmdl")); + if (CmdReader.err()) + return makeError("malformed or truncated commandline section"); + InternedCompileCommand Cmd = + readCompileCommand(CmdReader, Strings->Strings); + Result.Cmd.emplace(); + Result.Cmd->Directory = Cmd.Directory; + Result.Cmd->CommandLine.reserve(Cmd.CommandLine.size()); + for (llvm::StringRef C : Cmd.CommandLine) + Result.Cmd->CommandLine.emplace_back(C.str()); + } return std::move(Result); } @@ -547,6 +587,17 @@ } } + InternedCompileCommand Cmd; + if (Data.Cmd) { + Cmd.CommandLine.reserve(Data.Cmd->CommandLine.size()); + Cmd.Directory = Data.Cmd->Directory; + Strings.intern(Cmd.Directory); + for (llvm::StringRef C : Data.Cmd->CommandLine) { + Cmd.CommandLine.emplace_back(C); + Strings.intern(Cmd.CommandLine.back()); + } + } + std::string StringSection; { llvm::raw_string_ostream StringOS(StringSection); @@ -592,6 +643,15 @@ RIFF.Chunks.push_back({riff::fourCC("srcs"), SrcsSection}); } + std::string CmdlSection; + if (Data.Cmd) { + { + llvm::raw_string_ostream CmdOS(CmdlSection); + writeCompileCommand(Cmd, Strings, CmdOS); + } + RIFF.Chunks.push_back({riff::fourCC("cmdl"), CmdlSection}); + } + OS << RIFF; } diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp --- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp +++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp @@ -2,6 +2,7 @@ #include "TestFS.h" #include "TestTU.h" #include "index/Background.h" +#include "clang/Tooling/CompilationDatabase.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/Threading.h" #include "gmock/gmock.h" @@ -526,5 +527,49 @@ "unittest:///B.h")); } +TEST_F(BackgroundIndexTest, CmdLineHash) { + MockFSProvider FS; + llvm::StringMap Storage; + size_t CacheHits = 0; + MemoryShardStorage MSS(Storage, CacheHits); + OverlayCDB CDB(/*Base=*/nullptr, /*FallbackFlags=*/{}, + /*ResourceDir=*/std::string("")); + BackgroundIndex Idx(Context::empty(), FS, CDB, + [&](llvm::StringRef) { return &MSS; }); + + tooling::CompileCommand Cmd; + FS.Files[testPath("A.cc")] = "#include \"A.h\""; + FS.Files[testPath("A.h")] = ""; + Cmd.Filename = "../A.cc"; + Cmd.Directory = testPath("build"); + Cmd.CommandLine = {"clang++", "../A.cc", "-fsyntax-only"}; + CDB.setCompileCommand(testPath("build/../A.cc"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + + EXPECT_THAT(Storage.keys(), ElementsAre(testPath("A.cc"), testPath("A.h"))); + // Make sure we only store the Cmd for main file. + EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd); + + { + tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd; + EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine); + EXPECT_EQ(CmdStored.Directory, Cmd.Directory); + } + + // FIXME: Changing compile commands should be enough to invalidate the cache. + FS.Files[testPath("A.cc")] = " "; + Cmd.CommandLine = {"clang++", "../A.cc", "-Dfoo", "-fsyntax-only"}; + CDB.setCompileCommand(testPath("build/../A.cc"), Cmd); + ASSERT_TRUE(Idx.blockUntilIdleForTest()); + + EXPECT_FALSE(MSS.loadShard(testPath("A.h"))->Cmd); + + { + tooling::CompileCommand CmdStored = *MSS.loadShard(testPath("A.cc"))->Cmd; + EXPECT_EQ(CmdStored.CommandLine, Cmd.CommandLine); + EXPECT_EQ(CmdStored.Directory, Cmd.Directory); + } +} + } // namespace clangd } // namespace clang diff --git a/clang-tools-extra/clangd/unittests/SerializationTests.cpp b/clang-tools-extra/clangd/unittests/SerializationTests.cpp --- a/clang-tools-extra/clangd/unittests/SerializationTests.cpp +++ b/clang-tools-extra/clangd/unittests/SerializationTests.cpp @@ -8,6 +8,7 @@ #include "index/Index.h" #include "index/Serialization.h" +#include "clang/Tooling/CompilationDatabase.h" #include "llvm/Support/SHA1.h" #include "llvm/Support/ScopedPrinter.h" #include "gmock/gmock.h" @@ -240,6 +241,36 @@ } } +TEST(SerializationTest, CmdlTest) { + auto In = readIndexFile(YAML); + EXPECT_TRUE(bool(In)) << In.takeError(); + + tooling::CompileCommand Cmd; + Cmd.Directory = "testdir"; + Cmd.CommandLine.push_back("cmd1"); + Cmd.CommandLine.push_back("cmd2"); + Cmd.Filename = "ignored"; + Cmd.Heuristic = "ignored"; + Cmd.Output = "ignored"; + + IndexFileOut Out(*In); + Out.Format = IndexFileFormat::RIFF; + Out.Cmd = &Cmd; + { + std::string Serialized = llvm::to_string(Out); + + auto In = readIndexFile(Serialized); + ASSERT_TRUE(bool(In)) << In.takeError(); + ASSERT_TRUE(In->Cmd); + + const tooling::CompileCommand &SerializedCmd = In->Cmd.getValue(); + EXPECT_EQ(SerializedCmd.CommandLine, Cmd.CommandLine); + EXPECT_EQ(SerializedCmd.Directory, Cmd.Directory); + EXPECT_NE(SerializedCmd.Filename, Cmd.Filename); + EXPECT_NE(SerializedCmd.Heuristic, Cmd.Heuristic); + EXPECT_NE(SerializedCmd.Output, Cmd.Output); + } +} } // namespace } // namespace clangd } // namespace clang