Index: clangd/CMakeLists.txt =================================================================== --- clangd/CMakeLists.txt +++ clangd/CMakeLists.txt @@ -47,3 +47,4 @@ add_subdirectory(fuzzer) endif() add_subdirectory(tool) +add_subdirectory(global-symbol-builder) Index: clangd/global-symbol-builder/CMakeLists.txt =================================================================== --- /dev/null +++ clangd/global-symbol-builder/CMakeLists.txt @@ -0,0 +1,19 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../) + +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_executable(global-symbol-builder + GlobalSymbolBuilderMain.cpp + ) + +target_link_libraries(global-symbol-builder + PRIVATE + clangAST + clangIndex + clangDaemon + clangBasic + clangFrontend + clangTooling +) Index: clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp =================================================================== --- /dev/null +++ clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp @@ -0,0 +1,192 @@ +//===--- IndexSourceBuilderMain.cpp ------------------------------*- C++-*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// GlobalSymbolBuilder is a tool to generate YAML-format symbols across the +// whole project. This tools is for **experimental** only. Don't use it in +// production code. +// +//===---------------------------------------------------------------------===// + +#include "index/Index.h" +#include "index/SymbolCollector.h" +#include "index/SymbolYAML.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Index/IndexingAction.h" +#include "clang/Index/IndexDataConsumer.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Execution.h" +#include "clang/Tooling/StandaloneExecution.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ThreadPool.h" + +using namespace llvm; +using clang::clangd::SymbolSlab; + +namespace clang { +namespace clangd { + +class YAMLSymbolCallback : public SymbolCollector::SymbolCallback { +public: + YAMLSymbolCallback(tooling::ExecutionContext& Context) : Context(Context) {} + + ~YAMLSymbolCallback() = default; + + void onFinish(const ASTContext &ASTCtx, const SymbolSlab &Symbols) override { + auto FID = ASTCtx.getSourceManager().getMainFileID(); + const auto *Entry = ASTCtx.getSourceManager().getFileEntryForID(FID); + Context.reportResult(Entry->tryGetRealPathName(), SymbolToYAML(Symbols)); + } + +private: + tooling::ExecutionContext& Context; +}; + +class SymbolIndexActionFactory : public tooling::FrontendActionFactory { + public: + SymbolIndexActionFactory(tooling::ExecutionContext &Context) + : Context(Context) {} + + clang::FrontendAction *create() override { + index::IndexingOptions IndexOpts; + IndexOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::All; + IndexOpts.IndexFunctionLocals = false; + Callback = llvm::make_unique(Context); + Collector = std::make_shared(); + Collector->addSymbolCallback(Callback.get()); + return index::createIndexingAction(Collector, IndexOpts, nullptr) + .release(); + } + + tooling::ExecutionContext &Context; + std::shared_ptr Collector; + std::unique_ptr Callback; +}; + +} // namespace clangd +} // namespace clang + +static cl::OptionCategory IndexSourceCategory("index-source-builder options"); + +static cl::opt OutputDir("output-dir", cl::desc(R"( +The output directory for saving the results.)"), + cl::init("."), + cl::cat(IndexSourceCategory)); + +static cl::opt MergeDir("merge-dir", cl::desc(R"( +The directory for merging symbols.)"), + cl::init(""), + cl::cat(IndexSourceCategory)); + +bool WriteFile(llvm::StringRef OutputFile, const SymbolSlab& Symbols) { + std::error_code EC; + llvm::raw_fd_ostream OS(OutputFile, EC, llvm::sys::fs::F_None); + if (EC) { + llvm::errs() << "Can't open '" << OutputFile << "': " << EC.message() + << '\n'; + return false; + } + OS << clang::clangd::SymbolToYAML(Symbols); + return true; +} + +bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) { + std::error_code EC; + SymbolSlab Result; + std::mutex SymbolMutex; + auto AddSymbols = [&](const SymbolSlab& NewSymbols) { + // Synchronize set accesses. + std::unique_lock LockGuard(SymbolMutex); + for (const auto &Symbol : NewSymbols) { + auto it = Result.find(Symbol.second.ID); + if (it == Result.end()) + Result.insert(Symbol.second); + } + }; + + // Load all symbol files in MergeDir. + { + llvm::ThreadPool Pool; + for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd; + Dir != DirEnd && !EC; Dir.increment(EC)) { + // Parse YAML files in parallel. + Pool.async( + [&AddSymbols](std::string Path) { + auto Buffer = llvm::MemoryBuffer::getFile(Path); + if (!Buffer) { + llvm::errs() << "Can't open " << Path << "\n"; + return; + } + auto Symbols = + clang::clangd::SymbolFromYAML(Buffer.get()->getBuffer()); + // FIXME: Merge without creating such a heavy contention point. + AddSymbols(Symbols); + }, + Dir->path()); + } + } + WriteFile(OutputFile, Result); + return true; +} + + +int main(int argc, const char **argv) { + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + auto Executor = clang::tooling::createExecutorFromCommandLineArgs( + argc, argv, IndexSourceCategory, + "This is an **experimental** tool to generate YAML-format " + "project-wide symbols for clangd (global code completion). It would be " + "changed and deprecated eventually. Don't use it in production code!"); + + if (!Executor) { + llvm::errs() << llvm::toString(Executor.takeError()) << "\n"; + return 1; + } + + if (!MergeDir.empty()) { + // FIXME: createExecutorFromCommandLineArgs will print "Eror while trying to + // load a compilation databse" for the `merge` mode, we don't want this + // warning during merging. + llvm::out() << "Merging symbols now\n"; + // FIXME: use a safer way to do downcast -- ToolExecutor doesn't support + // LLVM-style RTTI yet. StandaloneToolExecutor is the expected executor of + // global-symbol-builder. + auto *STE = + static_cast(Executor->get()); + assert(!STE->getSourcePaths().empty()); + Merge(MergeDir, STE->getSourcePaths()[0]); + return 0; + } + + std::unique_ptr T( + new clang::clangd::SymbolIndexActionFactory( + *Executor->get()->getExecutionContext())); + auto Err = Executor->get()->execute(std::move(T)); + if (Err) { + llvm::errs() << llvm::toString(std::move(Err)) << "\n"; + return 1; + } + + Executor->get()->getToolResults()->forEachResult( + [](llvm::StringRef Key, llvm::StringRef Value) { + int FD; + SmallString<128> ResultPath; + llvm::sys::fs::createUniqueFile( + OutputDir + "/" + llvm::sys::path::filename(Key) + "-%%%%%%.yaml", + FD, ResultPath); + llvm::raw_fd_ostream OS(FD, true); + OS << Value; + }); + return 0; +} Index: clangd/global-symbol-builder/run-global-symbol-builder.py =================================================================== --- /dev/null +++ clangd/global-symbol-builder/run-global-symbol-builder.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# +#=- run-global-symbol-builder.py -------------------------------*- python -*-=# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===------------------------------------------------------------------------===# + +""" +Parallel global-symbol-builder runner +==================================== + +Runs global-symbol-builder over all files in a compilation database. + +Example invocations. +- Run global-symbol-builder on all files in the current working directory. + run-global-symbol-builder.py + +Compilation database setup: +http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html +""" + +import argparse +import json +import multiprocessing +import os +import Queue +import shutil +import subprocess +import sys +import tempfile +import threading + + +def find_compilation_database(path): + """Adjusts the directory until a compilation database is found.""" + result = './' + while not os.path.isfile(os.path.join(result, path)): + if os.path.realpath(result) == '/': + print 'Error: could not find compilation database.' + sys.exit(1) + result += '../' + return os.path.realpath(result) + + +def MergeSymbols(directory, args): + """Merge all symbol files (yaml) in a given directory into a single file.""" + invocation = [args.binary, '-merge-dir='+directory, args.saving_path] + subprocess.call(invocation) + print 'Merge is finished. Saving results in ' + args.saving_path + + +def run_global_symbol_builder(args, tmpdir, build_path, queue): + """Takes filenames out of queue and runs global-symbol-builder on them.""" + while True: + name = queue.get() + invocation = [args.binary, name, '-output-dir='+tmpdir, '-p='+build_path] + sys.stdout.write(' '.join(invocation) + '\n') + subprocess.call(invocation) + queue.task_done() + + +def main(): + parser = argparse.ArgumentParser(description='Runs global-symbol-builder over' + ' all files in a compilation database.') + parser.add_argument('-binary', metavar='PATH', + default='./bin/global-symbol-builder', + help='path to global-symbol-builder binary') + parser.add_argument('-j', type=int, default=0, + help='number of instances to be run in parallel.') + parser.add_argument('-p', dest='build_path', + help='path used to read a compilation database.') + parser.add_argument('-saving-path', default='./clangd-global-symbol.yaml', + help='path of saving the temporary results') + parser.add_argument('-tmp-path', default='/tmp', + help='path of saving the temporary results') + args = parser.parse_args() + + db_path = 'compile_commands.json' + + if args.build_path is not None: + build_path = args.build_path + else: + build_path = find_compilation_database(db_path) + + + # Load the database and extract all files. + database = json.load(open(os.path.join(build_path, db_path))) + files = [entry['file'] for entry in database] + + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + try: + # Spin up a bunch of launching threads. + queue = Queue.Queue(max_task) + for _ in range(max_task): + t = threading.Thread(target=run_global_symbol_builder, + args=(args, args.tmp_path, build_path, queue)) + t.daemon = True + t.start() + + # Fill the queue with files. + for name in files: + queue.put(name) + + # Wait for all threads to be done. + queue.join() + + MergeSymbols(args.tmp_path, args) + + + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print '\nCtrl-C detected, goodbye.' + os.kill(0, 9) + + +if __name__ == '__main__': + main() Index: clangd/index/SymbolCollector.h =================================================================== --- clangd/index/SymbolCollector.h +++ clangd/index/SymbolCollector.h @@ -10,6 +10,7 @@ #include "Index.h" #include "clang/Index/IndexDataConsumer.h" #include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/SmallPtrSet.h" namespace clang { namespace clangd { @@ -21,8 +22,24 @@ // changed. class SymbolCollector : public index::IndexDataConsumer { public: + // Callback to get AST/Symbol information when collecting symbols. + class SymbolCallback { + public: + virtual ~SymbolCallback() = default; + // Called when SymbolCollector finish collecting all symbols. + // Note: Symbols is a immutable object. + virtual void onFinish(const ASTContext &ASTCtx, const SymbolSlab &Symbols) { + } + }; + SymbolCollector() = default; + void addSymbolCallback(SymbolCallback* Callback) { + AllCallbacks.insert(Callback); + } + + void initialize(ASTContext &Ctx) override; + bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, ArrayRef Relations, FileID FID, @@ -36,6 +53,11 @@ private: // All Symbols collected from the AST. SymbolSlab Symbols; + + ASTContext *ASTCtx; + + // All symbol callbacks. + llvm::SmallPtrSet AllCallbacks; }; } // namespace clangd Index: clangd/index/SymbolCollector.cpp =================================================================== --- clangd/index/SymbolCollector.cpp +++ clangd/index/SymbolCollector.cpp @@ -107,7 +107,15 @@ return true; } -void SymbolCollector::finish() { Symbols.freeze(); } +void SymbolCollector::initialize(ASTContext &Ctx) { + ASTCtx = &Ctx; +} + +void SymbolCollector::finish() { + Symbols.freeze(); + for (auto *CB : AllCallbacks) + CB->onFinish(*ASTCtx, Symbols); +} } // namespace clangd } // namespace clang