Index: clangd/CMakeLists.txt =================================================================== --- clangd/CMakeLists.txt +++ clangd/CMakeLists.txt @@ -14,6 +14,7 @@ DraftStore.cpp FuzzyMatch.cpp GlobalCompilationDatabase.cpp + IncludeScanner.cpp JSONExpr.cpp JSONRPCDispatcher.cpp Logger.cpp Index: clangd/Compiler.cpp =================================================================== --- clangd/Compiler.cpp +++ clangd/Compiler.cpp @@ -29,17 +29,18 @@ IntrusiveRefCntPtr VFS, DiagnosticConsumer &DiagsClient) { assert(VFS && "VFS is null"); - assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers && - "Setting RetainRemappedFileBuffers to true will cause a memory leak " - "of ContentsBuffer"); - - // NOTE: we use Buffer.get() when adding remapped files, so we have to make - // sure it will be released if no error is emitted. - if (Preamble) { - Preamble->AddImplicitPreamble(*CI, VFS, Buffer.get()); - } else { - CI->getPreprocessorOpts().addRemappedFile( - CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get()); + if (Buffer != nullptr) { + assert(!CI->getPreprocessorOpts().RetainRemappedFileBuffers && + "Setting RetainRemappedFileBuffers to true will cause a memory leak " + "of Buffer"); + // NOTE: we use Buffer.get() when adding remapped files, so we have to make + // sure it will be released if no error is emitted. + if (Preamble) { + Preamble->AddImplicitPreamble(*CI, VFS, Buffer.get()); + } else { + CI->getPreprocessorOpts().addRemappedFile( + CI->getFrontendOpts().Inputs[0].getFile(), Buffer.get()); + } } auto Clang = llvm::make_unique(PCHs); @@ -58,7 +59,8 @@ // RemappedFileBuffers will handle the lifetime of the Buffer pointer, // release it. - Buffer.release(); + if (Buffer != nullptr) + Buffer.release(); return Clang; } Index: clangd/IncludeScanner.h =================================================================== --- /dev/null +++ clangd/IncludeScanner.h @@ -0,0 +1,94 @@ +//===--- IncludeScanner.h - Infer compile commands for headers --*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +// Typical compilation databases don't list commands for headers. But headers +// can be compiled with the same flags as the files that include them. +// So when we find a database, we scan through the commands it do +// preprocessor to find #included files that command is valid for. +//===---------------------------------------------------------------------===// +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INCLUDESCANNER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INCLUDESCANNER_H +#include "Context.h" +#include "Path.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include +#include +#include +#include +#include +#include + +namespace clang { +class PCHContainerOperations; +namespace vfs { +class FileSystem; +} +namespace tooling { +struct CompileCommand; +} // namespace tooling +namespace clangd { + +// IncludeScanner runs a background thread that scans files for which compile +// commands are known, recording the headers for which that command is valid. +// +// It supports lookup by header name. As scanning doesn't block, it's always +// possible we won't have scanned the right compile command yet. +// This means the compilation database should also have heuristics for headers. +// +// This class is threadsafe. +class IncludeScanner { +public: + IncludeScanner(); + + // The destructor may wait for the current file to finish scanning. + ~IncludeScanner(); + + // If we've scanned some file that #includes Header, return the inferred + // compile command. This will have the same flags, but the filename replaced. + llvm::Optional lookup(PathRef Header) const; + + // Adds compile commands to scan. Files we've already scanned will be ignored. + void enqueue(std::vector Cmds, + llvm::IntrusiveRefCntPtr VFS); + + // Blocks until the scanner is idle. Mostly useful for tests. + void wait() const; + +private: + struct QueueEntry; + + // Worker thread body: read commands off the queue and process them. + void + consumeQueue(); + + // Scan a compile command and record included headers. Runs on worker thread. + void process(QueueEntry Cmd); + bool canonicalize(tooling::CompileCommand &Cmd); + + // Commands we've detected that are available for lookup. + mutable std::mutex CommandsMu; + llvm::StringMap Commands; + + // The BFS state is locked by a mutex so other threads can enqueue. + mutable std::mutex QueueMu; + std::deque Queue; + // CompileCommand.Filenames we've processed, so we can skip duplicates. + llvm::StringSet SeenFiles; + bool Done = false; // Indicates the queue is closing down. + mutable std::condition_variable QueueCV; // Notified on queue writes. + + std::shared_ptr PCHContainerOps; + std::thread Worker; + Context Background; +}; + +} // namespace clangd +} // namespace clang +#endif Index: clangd/IncludeScanner.cpp =================================================================== --- /dev/null +++ clangd/IncludeScanner.cpp @@ -0,0 +1,197 @@ +//===--- IncludeScanner.cpp - Infer compile commands for headers *- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===---------------------------------------------------------------------===// +#include "IncludeScanner.h" +#include "Compiler.h" +#include "Logger.h" +#include "clang/Lex/PPCallbacks.h" +#include "clang/Lex/PreprocessorOptions.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Frontend/PCHContainerOperations.h" +#include "clang/Tooling/ArgumentsAdjusters.h" +#include "clang/Tooling/CompilationDatabase.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +namespace clang { +namespace clangd { +using namespace llvm; +using tooling::CompileCommand; + +struct IncludeScanner::QueueEntry : public CompileCommand { + QueueEntry(const CompileCommand &Cmd, IntrusiveRefCntPtr VFS) + : CompileCommand(Cmd), VFS(std::move(VFS)) {} + IntrusiveRefCntPtr VFS; +}; + +IncludeScanner::IncludeScanner() + : PCHContainerOps(std::make_shared()), + Worker([this] { consumeQueue(); }) {} + +IncludeScanner::~IncludeScanner() { + { + std::lock_guard Lock(QueueMu); + Done = true; + } + // Wake up the thread if it's blocking on data. + QueueCV.notify_all(); + // Don't destroy members until the thread exits. + Worker.join(); +} + +Optional IncludeScanner::lookup(PathRef File) const { + std::lock_guard Lock(CommandsMu); + auto I = Commands.find(File); + if (I == Commands.end()) + return None; + return I->getValue(); +} + +// Ensures that a compile command we're going to scan is in canonical form, +// with the filename at the end, so we can replace it with a header filename. +bool IncludeScanner::canonicalize(CompileCommand &C) { + using sys::path::filename; + if (C.CommandLine.empty() || + filename(C.Filename) != filename(C.CommandLine.back())) { + log(Background, "Compile command not canonical [" + C.Filename + "] " + + join(C.CommandLine, " ")); + return false; + } + C.CommandLine = + tooling::getClangStripOutputAdjuster()(C.CommandLine, C.Filename); + return true; +} + +void IncludeScanner::enqueue(std::vector Cmds, + IntrusiveRefCntPtr VFS) { + std::lock_guard Lock(QueueMu); + bool WasEmpty = Queue.empty(); + for (const auto &Cmd : Cmds) { + QueueEntry E(Cmd, VFS); + if (canonicalize(E) && SeenFiles.insert(E.Filename).second) { + Queue.push_back(std::move(E)); + } + } + if (WasEmpty && !Queue.empty()) + QueueCV.notify_all(); +} + +void IncludeScanner::wait() const { + std::unique_lock Lock(QueueMu); + QueueCV.wait(Lock, [&] { return Done || Queue.empty(); }); +} + +void IncludeScanner::consumeQueue() { + while (true) { + std::unique_lock Lock(QueueMu); + QueueCV.wait(Lock, [&] { return Done || !Queue.empty(); }); + if (Done) + return; + auto &Entry = Queue.front(); + Lock.unlock(); + process(std::move(Entry)); + Lock.lock(); + Queue.pop_front(); + if (Queue.empty()) { + log(Background, "Finished scanning headers"); + QueueCV.notify_all(); + } + } +} + +void IncludeScanner::process(QueueEntry Cmd) { + class RecordHeaders : public PPCallbacks { + public: + RecordHeaders(std::vector &Headers) : Headers(Headers) {} + + void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok, + StringRef FileName, bool IsAngled, + CharSourceRange FilenameRange, + const FileEntry *File, StringRef SearchPath, + StringRef RelativePath, + const Module *Imported) override { + if (File != nullptr && !File->tryGetRealPathName().empty()) + Headers.push_back(File->tryGetRealPathName()); + } + + private: + std::vector &Headers; + }; + + std::vector Argv; + for (const auto &S : Cmd.CommandLine) + Argv.push_back(S.c_str()); + IgnoringDiagConsumer IgnoreDiags; + // XXX the VFS working directory is global state, this is unsafe. + Cmd.VFS->setCurrentWorkingDirectory(Cmd.Directory); + auto CI = clang::createInvocationFromCommandLine( + Argv, + CompilerInstance::createDiagnostics(new DiagnosticOptions(), &IgnoreDiags, + false), + Cmd.VFS); + if (!CI) + return; + CI->getFrontendOpts().DisableFree = false; + CI->getPreprocessorOpts().SingleFileParseMode = true; + auto Clang = prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr, + /*Buffer=*/nullptr, PCHContainerOps, + Cmd.VFS, IgnoreDiags); + PreprocessOnlyAction Action; + if (Clang->getFrontendOpts().Inputs.empty() || + !Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) + return; + std::vector Headers; + Clang->getPreprocessor().addPPCallbacks( + make_unique(Headers)); + if (!Action.Execute()) + return; + + // Now we turn Cmd into a template that we can use for each included header. + Cmd.CommandLine.pop_back(); // Remove the filename. + // `clang foo.cpp` builds as C++, but `clang foo.h` builds as C. + // When reusing flags, we add an explicit `-x cpp` to override the extension. + // TODO: copying CompilerInvocation would avoid this, and is more robust. + if (std::find(Cmd.CommandLine.begin(), Cmd.CommandLine.end(), "-x") == + Cmd.CommandLine.end()) { + Cmd.CommandLine.push_back("-x"); + switch (Clang->getFrontendOpts().Inputs[0].getKind().getLanguage()) { + case InputKind::C: + Cmd.CommandLine.push_back("c"); + break; + case InputKind::CXX: + Cmd.CommandLine.push_back("c++"); + break; + case InputKind::ObjC: + Cmd.CommandLine.push_back("objective-c"); + break; + case InputKind::ObjCXX: + Cmd.CommandLine.push_back("objective-c++"); + break; + default: + Cmd.CommandLine.pop_back(); + break; + } + } + std::lock_guard Lock(CommandsMu); + std::vector NextCommands; + for (const auto &S : Headers) { + auto R = Commands.try_emplace(S, Cmd); + if (!R.second) + continue; // Already seen this header. + auto &NewCmd = R.first->getValue(); + NewCmd.Filename = S; + NewCmd.CommandLine.push_back(S); + NextCommands.push_back(NewCmd); + } + if (!NextCommands.empty()) + enqueue(std::move(NextCommands), std::move(Cmd.VFS)); +} + +} // namespace clangd +} // namespace clang Index: unittests/clangd/CMakeLists.txt =================================================================== --- unittests/clangd/CMakeLists.txt +++ unittests/clangd/CMakeLists.txt @@ -16,6 +16,7 @@ ContextTests.cpp FileIndexTests.cpp FuzzyMatchTests.cpp + IncludeScannerTests.cpp IndexTests.cpp JSONExprTests.cpp TestFS.cpp Index: unittests/clangd/IncludeScannerTests.cpp =================================================================== --- /dev/null +++ unittests/clangd/IncludeScannerTests.cpp @@ -0,0 +1,61 @@ +//===-- IncludeScannerTests.cpp ----------------------*- C++ -*-----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "IncludeScanner.h" +#include "TestFS.h" +#include "llvm/ADT/StringMap.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "Logger.h" + +namespace clang { +namespace tooling { +void PrintTo(const llvm::Optional &S, std::ostream *OS) { + if (S) + *OS << llvm::join(S->CommandLine, " "); + else + *OS << ""; +} +} // namespace tooling +namespace clangd { +namespace { + +MATCHER_P2(HasCmd, Args, Filename, "") { + return arg && + arg->CommandLine == Args && + arg->Filename == Filename; +} + +testing::Matcher> +Cmd(std::vector Args) { + return HasCmd(Args, Args.back()); +} + +TEST(IncludeScanner, FindsFiles) { + llvm::StringMap Files; + llvm::StringRef One = getVirtualTestFilePath("one.cc"), + Two = getVirtualTestFilePath("dir/two.h"), + Three = getVirtualTestFilePath("dir/three.h"); + Files[One] = R"cpp(#include "dir/two.h")cpp"; + Files[Two] = R"cpp(#include "three.h")cpp"; + Files[Three] = ""; + auto FS = buildTestFS(Files); + + IncludeScanner Scanner; + Scanner.enqueue({ + {".", One, {"clang", "-DX", One}, ""}, + }, FS); + Scanner.wait(); + EXPECT_THAT(Scanner.lookup(Two), Cmd({"clang", "-DX", "-x", "c++", Two})); + EXPECT_THAT(Scanner.lookup(Three), Cmd({"clang", "-DX", "-x", "c++", Three})); +} + +} // namespace +} // namespace clangd +} // namespace clang