Index: clang-tools-extra/trunk/include-fixer/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/include-fixer/CMakeLists.txt +++ clang-tools-extra/trunk/include-fixer/CMakeLists.txt @@ -18,3 +18,4 @@ ) add_subdirectory(tool) +add_subdirectory(find-all-symbols) Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/CMakeLists.txt +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/CMakeLists.txt @@ -0,0 +1,18 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_library(findAllSymbols + FindAllSymbols.cpp + SymbolInfo.cpp + + LINK_LIBS + clangAST + clangASTMatchers + clangBasic + clangFrontend + clangTooling + clangToolingCore + ) + +add_subdirectory(tool) Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.h =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.h +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.h @@ -0,0 +1,56 @@ +//===-- FindAllSymbols.h - find all symbols----------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H +#define LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H + +#include "SymbolInfo.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include + +namespace clang { +namespace find_all_symbols { + +/// \brief FindAllSymbols collects all classes, free standing functions and +/// global variables with some extra information such as the path of the header +/// file, the namespaces they are contained in, the type of variables and the +/// parameter types of functions. +/// +/// NOTE: +/// - Symbols declared in main files are not collected since they can not be +/// included. +/// - Member functions are not collected because accessing them must go +/// through the class. #include fixer only needs the class name to find +/// headers. +/// +class FindAllSymbols : public clang::ast_matchers::MatchFinder::MatchCallback { +public: + class ResultReporter { + public: + virtual ~ResultReporter() = default; + + virtual void reportResult(llvm::StringRef FileName, + const SymbolInfo &Symbol) = 0; + }; + + explicit FindAllSymbols(ResultReporter *Reporter) : Reporter(Reporter) {} + + void registerMatchers(clang::ast_matchers::MatchFinder *MatchFinder); + + void + run(const clang::ast_matchers::MatchFinder::MatchResult &result) override; + +private: + ResultReporter *const Reporter; +}; + +} // namespace find_all_symbols +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_FIND_ALL_SYMBOLS_SYMBOL_MATCHER_H Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.cpp =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.cpp +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/FindAllSymbols.cpp @@ -0,0 +1,193 @@ +//===-- FindAllSymbols.cpp - find all symbols -----------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FindAllSymbols.h" +#include "SymbolInfo.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclCXX.h" +#include "clang/AST/Type.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/FileSystem.h" + +using namespace clang::ast_matchers; + +namespace clang { +namespace find_all_symbols { +namespace { +void SetContext(const NamedDecl *ND, SymbolInfo *Symbol) { + for (const auto *Context = ND->getDeclContext(); Context; + Context = Context->getParent()) { + if (llvm::isa(Context) || + llvm::isa(Context)) + break; + + assert(llvm::isa(Context) && + "Expect Context to be a NamedDecl"); + if (const auto *NSD = dyn_cast(Context)) { + Symbol->Contexts.emplace_back( + SymbolInfo::Namespace, + NSD->isAnonymousNamespace() ? "" : NSD->getName().str()); + } else { + const auto *RD = cast(Context); + Symbol->Contexts.emplace_back(SymbolInfo::Record, RD->getName().str()); + } + } +} + +bool SetCommonInfo(const MatchFinder::MatchResult &Result, + const NamedDecl *ND, SymbolInfo *Symbol) { + SetContext(ND, Symbol); + + Symbol->Name = ND->getNameAsString(); + SourceLocation Loc = Result.SourceManager->getExpansionLoc(ND->getLocation()); + if (!Loc.isValid()) { + llvm::errs() << "Declaration " << ND->getNameAsString() << "(" + << ND->getDeclKindName() + << ") has invalid declaration location."; + return false; + } + std::string FilePath = Result.SourceManager->getFilename(Loc).str(); + if (FilePath.empty()) + return false; + + Symbol->FilePath = FilePath; + Symbol->LineNumber = Result.SourceManager->getExpansionLineNumber(Loc); + return true; +} +} // namespace + +void FindAllSymbols::registerMatchers(MatchFinder *MatchFinder) { + // FIXME: Handle specialization. + auto IsInSpecialization = hasAncestor( + decl(anyOf(cxxRecordDecl(isExplicitTemplateSpecialization()), + functionDecl(isExplicitTemplateSpecialization())))); + + // Matchers for both C and C++. + // We only match symbols from header files, i.e. not from main files (see + // function's comment for detailed explanation). + auto CommonFilter = + allOf(unless(isImplicit()), unless(isExpansionInMainFile())); + + auto HasNSOrTUCtxMatcher = + hasDeclContext(anyOf(namespaceDecl(), translationUnitDecl())); + + // We need seperate rules for C record types and C++ record types since some + // template related matchers are inapplicable on C record declarations. + // + // Matchers specific to C++ code. + // All declarations should be in namespace or translation unit. + auto CCMatcher = + allOf(HasNSOrTUCtxMatcher, unless(IsInSpecialization), + unless(ast_matchers::isTemplateInstantiation()), + unless(isInstantiated()), unless(classTemplateSpecializationDecl()), + unless(isExplicitTemplateSpecialization())); + + // Matchers specific to code in extern "C" {...}. + auto ExternCMatcher = hasDeclContext(linkageSpecDecl()); + + // Matchers for variable declarations. + // + // In most cases, `ParmVarDecl` is filtered out by hasDeclContext(...) + // matcher since the declaration context is usually `MethodDecl`. However, + // this assumption does not hold for parameters of a function pointer + // parameter. + // For example, consider a function declaration: + // void Func(void (*)(float), int); + // The float parameter of the function pointer has an empty name, and its + // declaration context is an anonymous namespace; therefore, it won't be + // filtered out by our matchers above. + MatchFinder->addMatcher(varDecl(CommonFilter, + anyOf(ExternCMatcher, CCMatcher), + unless(parmVarDecl())) + .bind("decl"), + this); + + // Matchers for C-style record declarations in extern "C" {...}. + MatchFinder->addMatcher( + recordDecl(CommonFilter, ExternCMatcher, isDefinition()).bind("decl"), + this); + + // Matchers for C++ record declarations. + auto CxxRecordDecl = + cxxRecordDecl(CommonFilter, CCMatcher, isDefinition(), + unless(isExplicitTemplateSpecialization())); + MatchFinder->addMatcher(CxxRecordDecl.bind("decl"), this); + + // Matchers for function declarations. + MatchFinder->addMatcher( + functionDecl(CommonFilter, anyOf(ExternCMatcher, CCMatcher)).bind("decl"), + this); + + // Matcher for typedef and type alias declarations. + // + // typedef and type alias can come from C-style headers and C++ heaeders. + // For C-style header, `DeclContxet` can be either `TranslationUnitDecl` + // or `LinkageSpecDecl`. + // For C++ header, `DeclContext ` can be one of `TranslationUnitDecl`, + // `NamespaceDecl`. + // With the following context matcher, we can match `typedefNameDecl` from + // both C-style header and C++ header (except for those in classes). + // "cc_matchers" are not included since template-related matchers are not + // applicable on `TypedefNameDecl`. + MatchFinder->addMatcher( + typedefNameDecl(CommonFilter, anyOf(HasNSOrTUCtxMatcher, + hasDeclContext(linkageSpecDecl()))) + .bind("decl"), + this); +} + +void FindAllSymbols::run(const MatchFinder::MatchResult &Result) { + // Ignore Results in failing TUs. + if (Result.Context->getDiagnostics().hasErrorOccurred()) { + return; + } + + const NamedDecl *ND = Result.Nodes.getNodeAs("decl"); + assert(ND && "Matched declaration must be a NamedDecl!"); + const SourceManager *SM = Result.SourceManager; + + SymbolInfo Symbol; + if (!SetCommonInfo(Result, ND, &Symbol)) + return; + + if (const auto *VD = llvm::dyn_cast(ND)) { + Symbol.Type = SymbolInfo::Variable; + SymbolInfo::VariableInfo VI; + VI.Type = VD->getType().getAsString(); + Symbol.VariableInfos = VI; + } else if (const auto *FD = llvm::dyn_cast(ND)) { + Symbol.Type = SymbolInfo::Function; + SymbolInfo::FunctionInfo FI; + FI.ReturnType = FD->getReturnType().getAsString(); + for (const auto *Param : FD->params()) + FI.ParameterTypes.push_back(Param->getType().getAsString()); + Symbol.FunctionInfos = FI; + } else if (const auto *TD = llvm::dyn_cast(ND)) { + Symbol.Type = SymbolInfo::TypedefName; + SymbolInfo::TypedefNameInfo TI; + TI.UnderlyingType = TD->getUnderlyingType().getAsString(); + Symbol.TypedefNameInfos = TI; + } else { + assert( + llvm::isa(ND) && + "Matched decl must be one of VarDecl, FunctionDecl, and RecordDecl!"); + // C-style record decl can have empty name, e.g "struct { ... } var;". + if (ND->getName().empty()) + return; + Symbol.Type = SymbolInfo::Class; + } + + const FileEntry *FE = SM->getFileEntryForID(SM->getMainFileID()); + Reporter->reportResult(FE->getName(), Symbol); +} + +} // namespace find_all_symbols +} // namespace clang Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.h =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.h +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.h @@ -0,0 +1,100 @@ +//===-- SymbolInfo.h - find all symbols--------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_INCLUDE_FIXER_FIND_ALL_SYMBOLS_SYMBOL_INFO_H +#define LLVM_CLANG_TOOLS_EXTRA_INCLUDE_FIXER_FIND_ALL_SYMBOLS_SYMBOL_INFO_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include +#include +#include + +namespace clang { +namespace find_all_symbols { + +/// \brief Contains all information for a Symbol. +struct SymbolInfo { + enum SymbolKind { + Function, + Class, + Variable, + TypedefName, + }; + + enum ContextType { + Namespace, // Symbols declared in a namespace. + Record, // Symbols declared in a class. + }; + + /// \brief Identifier name. + std::string Name; + + /// \brief Symbol type. + SymbolKind Type; + + /// \brief The file path where the symbol comes from. + std::string FilePath; + + /// \brief A pair of . + typedef std::pair Context; + + /// \brief Contains information about symbol contexts. Context information is + /// stored from the inner-most level to outer-most level. + /// + /// For example, if a symbol 'x' is declared as: + /// namespace na { namespace nb { class A { int x; } } } + /// The contexts would be { {RECORD, "A"}, {NAMESPACE, "nb"}, {NAMESPACE, + /// "na"} }. + /// The name of an anonymous namespace is "". + /// + /// If the symbol is declared in `TranslationUnitDecl`, it has no context. + std::vector Contexts; + + /// \brief The 1-based line number of of the symbol's declaration. + int LineNumber; + + struct FunctionInfo { + std::string ReturnType; + std::vector ParameterTypes; + }; + + struct TypedefNameInfo { + std::string UnderlyingType; + }; + + struct VariableInfo { + std::string Type; + }; + + /// \brief The function information. + llvm::Optional FunctionInfos; + + /// \brief The typedef information. + llvm::Optional TypedefNameInfos; + + /// \brief The variable information. + llvm::Optional VariableInfos; + + bool operator==(const SymbolInfo &Symbol) const; + + bool operator<(const SymbolInfo &Symbol) const; +}; + +/// \brief Write SymbolInfos to a single file (YAML format). +bool WriteSymboInfosToFile(llvm::StringRef FilePath, + const std::set &Symbols); + +/// \brief Read SymbolInfos from a YAML document. +std::vector ReadSymbolInfosFromYAML(llvm::StringRef Yaml); + +} // namespace find_all_symbols +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_INCLUDE_FIXER_FIND_ALL_SYMBOLS_SYMBOL_INFO_H Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.cpp =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.cpp +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/SymbolInfo.cpp @@ -0,0 +1,121 @@ +//===-- SymbolInfo.cpp ----------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "SymbolInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" + +using llvm::yaml::MappingTraits; +using llvm::yaml::IO; +using llvm::yaml::Input; +using ContextType = clang::find_all_symbols::SymbolInfo::ContextType; +using clang::find_all_symbols::SymbolInfo; +using SymbolKind = clang::find_all_symbols::SymbolInfo::SymbolKind; + +LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(SymbolInfo) +LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(std::string) +LLVM_YAML_IS_SEQUENCE_VECTOR(SymbolInfo::Context) + +namespace llvm { +namespace yaml { +template <> struct MappingTraits { + static void mapping(IO &io, SymbolInfo &Symbol) { + io.mapRequired("Name", Symbol.Name); + io.mapRequired("Contexts", Symbol.Contexts); + io.mapRequired("FilePath", Symbol.FilePath); + io.mapRequired("LineNumber", Symbol.LineNumber); + io.mapRequired("Type", Symbol.Type); + io.mapOptional("Variable", Symbol.VariableInfos); + io.mapOptional("Function", Symbol.FunctionInfos); + io.mapOptional("TypedefName", Symbol.TypedefNameInfos); + } +}; + +template <> struct ScalarEnumerationTraits { + static void enumeration(IO &io, ContextType &value) { + io.enumCase(value, "Record", ContextType::Record); + io.enumCase(value, "Namespace", ContextType::Namespace); + } +}; + +template <> struct ScalarEnumerationTraits { + static void enumeration(IO &io, SymbolKind &value) { + io.enumCase(value, "Variable", SymbolKind::Variable); + io.enumCase(value, "Function", SymbolKind::Function); + io.enumCase(value, "Class", SymbolKind::Class); + io.enumCase(value, "TypedefName", SymbolKind::TypedefName); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &io, SymbolInfo::Context &Context) { + io.mapRequired("ContextType", Context.first); + io.mapRequired("ContextName", Context.second); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &io, SymbolInfo::FunctionInfo &Value) { + io.mapRequired("ReturnType", Value.ReturnType); + io.mapRequired("ParameterTypes", Value.ParameterTypes); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &io, SymbolInfo::VariableInfo &Value) { + io.mapRequired("VariableType", Value.Type); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &io, SymbolInfo::TypedefNameInfo &Value) { + io.mapRequired("TypedefNameType", Value.UnderlyingType); + } +}; + +} // namespace yaml +} // namespace llvm + +namespace clang { +namespace find_all_symbols { + +bool SymbolInfo::operator==(const SymbolInfo &Symbol) const { + return Name == Symbol.Name && FilePath == Symbol.FilePath && + LineNumber == Symbol.LineNumber && Contexts == Symbol.Contexts; +} + +bool SymbolInfo::operator<(const SymbolInfo &Symbol) const { + return std::tie(Name, FilePath, LineNumber) < + std::tie(Symbol.Name, Symbol.FilePath, Symbol.LineNumber); +} + +bool WriteSymboInfosToFile(llvm::StringRef FilePath, + const std::set &Symbols) { + int FD = 0; + if (llvm::sys::fs::openFileForWrite(FilePath, FD, llvm::sys::fs::F_None)) + return false; + llvm::raw_fd_ostream OS(FD, true); + llvm::yaml::Output yout(OS); + for (auto Symbol : Symbols) + yout << Symbol; + OS.close(); + return true; +} + +std::vector ReadSymbolInfosFromYAML(llvm::StringRef Yaml) { + std::vector Symbols; + llvm::yaml::Input yin(Yaml); + yin >> Symbols; + return Symbols; +} + +} // namespace find_all_symbols +} // namespace clang Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/CMakeLists.txt +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/CMakeLists.txt @@ -0,0 +1,13 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) + +add_clang_executable(find-all-symbols FindAllSymbolsMain.cpp) +target_link_libraries(find-all-symbols + + clangASTMatchers + clangBasic + clangDynamicASTMatchers + clangFrontend + clangQuery + clangTooling + findAllSymbols + ) Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/FindAllSymbolsMain.cpp @@ -0,0 +1,122 @@ +//===-- FindAllSymbolsMain.cpp --------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FindAllSymbols.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/Path.h" + +#include +#include +#include + +using namespace clang::tooling; +using namespace llvm; +using SymbolInfo = clang::find_all_symbols::SymbolInfo; + +// Apply a custom category to all command-line options so that they are the +// only ones displayed. +static cl::OptionCategory FindAllSymbolsCategory("find_all_symbols options"); + +// CommonOptionsParser declares HelpMessage with a description of the common +// command-line options related to the compilation database and input files. +// It's nice to have this help message in all tools. +static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage); + +// A help message for this specific tool can be added afterwards. +static cl::extrahelp MoreHelp("\nMore help text..."); + +static cl::opt OutputDir("output-dir", cl::desc(R"( +The output directory for saving the results.)"), + cl::init("."), + cl::cat(FindAllSymbolsCategory)); + +static cl::opt MergeDir("merge-dir", cl::desc(R"( +The directory for merging symbols.)"), + cl::init(""), + cl::cat(FindAllSymbolsCategory)); + +namespace clang { +namespace find_all_symbols { + +class YamlReporter + : public clang::find_all_symbols::FindAllSymbols::ResultReporter { +public: + ~YamlReporter() {} + + void reportResult(StringRef FileName, const SymbolInfo &Symbol) override { + Symbols[FileName].insert(Symbol); + } + + void Write(const std::string &Dir) { + for (const auto &Symbol : Symbols) { + SmallString<256> FilePath(Dir); + llvm::sys::path::append( + FilePath, llvm::sys::path::filename(Symbol.first) + ".yaml"); + WriteSymboInfosToFile(FilePath, Symbol.second); + } + } + +private: + std::map> Symbols; +}; + +bool Merge(llvm::StringRef MergeDir, llvm::StringRef OutputFile) { + std::error_code EC; + std::set UniqueSymbols; + // Load all symbol files in MergeDir. + for (llvm::sys::fs::directory_iterator Dir(MergeDir, EC), DirEnd; + Dir != DirEnd && !EC; Dir.increment(EC)) { + int ReadFD = 0; + if (llvm::sys::fs::openFileForRead(Dir->path(), ReadFD)) { + llvm::errs() << "Cann't open " << Dir->path() << "\n"; + continue; + } + auto Buffer = llvm::MemoryBuffer::getOpenFile(ReadFD, Dir->path(), -1); + if (!Buffer) + continue; + std::vector Symbols = + ReadSymbolInfosFromYAML(Buffer.get()->getBuffer()); + for (const auto &Symbol : Symbols) + UniqueSymbols.insert(Symbol); + } + + WriteSymboInfosToFile(OutputFile, UniqueSymbols); + return true; +} + +} // namespace clang +} // namespace find_all_symbols + +int main(int argc, const char **argv) { + CommonOptionsParser OptionsParser(argc, argv, FindAllSymbolsCategory); + ClangTool Tool(OptionsParser.getCompilations(), + OptionsParser.getSourcePathList()); + + std::vector sources = OptionsParser.getSourcePathList(); + if (sources.empty()) { + llvm::errs() << "Must specify at least one one source file.\n"; + return 1; + } + if (!MergeDir.empty()) { + clang::find_all_symbols::Merge(MergeDir, sources[0]); + return 0; + } + + clang::find_all_symbols::YamlReporter Reporter; + clang::find_all_symbols::FindAllSymbols Matcher(&Reporter); + clang::ast_matchers::MatchFinder MatchFinder; + Matcher.registerMatchers(&MatchFinder); + Tool.run(newFrontendActionFactory(&MatchFinder).get()); + Reporter.Write(OutputDir); + return 0; +} Index: clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/run-find-all-symbols.py =================================================================== --- clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/run-find-all-symbols.py +++ clang-tools-extra/trunk/include-fixer/find-all-symbols/tool/run-find-all-symbols.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +# +#=- run-find-all-symbols.py - Parallel find-all-symbols runner -*- python -*-=# +# +# The LLVM Compiler Infrastructure +# +# This file is distributed under the University of Illinois Open Source +# License. See LICENSE.TXT for details. +# +#===------------------------------------------------------------------------===# + +""" +Parallel find-all-symbols runner +================================ + +Runs find-all-symbols over all files in a compilation database. + +Example invocations. +- Run find-all-symbols on all files in the current working directory. + run-find-all-symbols.py + +Compilation database setup: +http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html +""" + +import argparse +import json +import multiprocessing +import os +import Queue +import shutil +import subprocess +import sys +import tempfile +import threading + + +def find_compilation_database(path): + """Adjusts the directory until a compilation database is found.""" + result = './' + while not os.path.isfile(os.path.join(result, path)): + if os.path.realpath(result) == '/': + print 'Error: could not find compilation database.' + sys.exit(1) + result += '../' + return os.path.realpath(result) + + +def MergeSymbols(directory, args): + """Merge all symbol files (yaml) in a given directaory into a single file.""" + invocation = [args.binary, '-merge-dir='+directory, args.saving_path] + subprocess.call(invocation) + print 'Merge is finished. Saving results in ' + args.saving_path + + +def run_find_all_symbols(args, tmpdir, build_path, queue): + """Takes filenames out of queue and runs find-all-symbols on them.""" + while True: + name = queue.get() + invocation = [args.binary, name, '-output-dir='+tmpdir, '-p='+build_path] + sys.stdout.write(' '.join(invocation) + '\n') + subprocess.call(invocation) + queue.task_done() + + +def main(): + parser = argparse.ArgumentParser(description='Runs find-all-symbols over all' + 'files in a compilation database.') + parser.add_argument('-binary', metavar='PATH', + default='./bin/find-all-symbols', + help='path to find-all-symbols binary') + parser.add_argument('-j', type=int, default=0, + help='number of instances to be run in parallel.') + parser.add_argument('-p', dest='build_path', + help='path used to read a compilation database.') + parser.add_argument('-saving-path', default='./find_all_symbols_db.yaml', + help='result saving path') + args = parser.parse_args() + + db_path = 'compile_commands.json' + + if args.build_path is not None: + build_path = args.build_path + else: + build_path = find_compilation_database(db_path) + + tmpdir = tempfile.mkdtemp() + + # Load the database and extract all files. + database = json.load(open(os.path.join(build_path, db_path))) + files = [entry['file'] for entry in database] + + max_task = args.j + if max_task == 0: + max_task = multiprocessing.cpu_count() + + try: + # Spin up a bunch of tidy-launching threads. + queue = Queue.Queue(max_task) + for _ in range(max_task): + t = threading.Thread(target=run_find_all_symbols, + args=(args, tmpdir, build_path, queue)) + t.daemon = True + t.start() + + # Fill the queue with files. + for name in files: + queue.put(name) + + # Wait for all threads to be done. + queue.join() + + MergeSymbols(tmpdir, args) + + + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print '\nCtrl-C detected, goodbye.' + os.kill(0, 9) + + +if __name__ == '__main__': + main() Index: clang-tools-extra/trunk/unittests/include-fixer/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/unittests/include-fixer/CMakeLists.txt +++ clang-tools-extra/trunk/unittests/include-fixer/CMakeLists.txt @@ -23,3 +23,5 @@ clangTooling clangToolingCore ) + +add_subdirectory(find-all-symbols) Index: clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/CMakeLists.txt =================================================================== --- clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/CMakeLists.txt +++ clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/CMakeLists.txt @@ -0,0 +1,21 @@ +set(LLVM_LINK_COMPONENTS + support + ) + +get_filename_component(INCLUDE_FIXER_SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include-fixer/find-all-symbols REALPATH) +include_directories( + ${INCLUDE_FIXER_SOURCE_DIR} + ) + +add_extra_unittest(FindAllSymbolsTests + FindAllSymbolsTests.cpp + ) + +target_link_libraries(FindAllSymbolsTests + clangBasic + clangFrontend + clangTooling + clangToolingCore + findAllSymbols + ) Index: clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/FindAllSymbolsTests.cpp =================================================================== --- clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/FindAllSymbolsTests.cpp +++ clang-tools-extra/trunk/unittests/include-fixer/find-all-symbols/FindAllSymbolsTests.cpp @@ -0,0 +1,364 @@ +//===-- FindAllSymbolsTests.cpp - find all symbols unit tests -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FindAllSymbols.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/YAMLTraits.h" +#include "gtest/gtest.h" + +namespace clang { +namespace find_all_symbols { + +static const char HeaderName[] = "symbols.h"; + +class MockReporter + : public clang::find_all_symbols::FindAllSymbols::ResultReporter { +public: + ~MockReporter() {} + + void reportResult(llvm::StringRef FileName, + const SymbolInfo &Symbol) override { + Symbols.push_back(Symbol); + } + + bool hasSymbol(const SymbolInfo &Symbol) { + for (const auto &S : Symbols) { + if (S == Symbol) + return true; + } + return false; + } + + bool getSymbolExtraInfo(SymbolInfo *Symbol) { + for (const auto &S : Symbols) { + if (S == *Symbol) { + Symbol->FunctionInfos = S.FunctionInfos; + Symbol->TypedefNameInfos = S.TypedefNameInfos; + Symbol->VariableInfos = S.VariableInfos; + return true; + } + } + return false; + } + +private: + std::vector Symbols; +}; + +class FindAllSymbolsTest : public ::testing::Test { +public: + bool hasSymbol(const SymbolInfo &Symbol) { + return Reporter.hasSymbol(Symbol); + } + + bool getSymbolExtraInfo(SymbolInfo &Symbol) { + return Reporter.getSymbolExtraInfo(&Symbol); + } + + bool runFindAllSymbols(StringRef Code) { + FindAllSymbols matcher(&Reporter); + clang::ast_matchers::MatchFinder MatchFinder; + matcher.registerMatchers(&MatchFinder); + + llvm::IntrusiveRefCntPtr InMemoryFileSystem( + new vfs::InMemoryFileSystem); + llvm::IntrusiveRefCntPtr Files( + new FileManager(FileSystemOptions(), InMemoryFileSystem)); + + std::string FileName = "symbol.cc"; + std::unique_ptr Factory = + clang::tooling::newFrontendActionFactory(&MatchFinder); + tooling::ToolInvocation Invocation( + {std::string("find_all_symbols"), std::string("-fsyntax-only"), + FileName}, + Factory->create(), Files.get(), + std::make_shared()); + + InMemoryFileSystem->addFile(HeaderName, 0, + llvm::MemoryBuffer::getMemBuffer(Code)); + + std::string Content = "#include\"" + std::string(HeaderName) + "\""; + InMemoryFileSystem->addFile(FileName, 0, + llvm::MemoryBuffer::getMemBuffer(Content)); + Invocation.run(); + return true; + } + +private: + MockReporter Reporter; +}; + +SymbolInfo +CreateSymbolInfo(StringRef Name, SymbolInfo::SymbolKind Type, + const std::string FilePath, int LineNumber, + const std::vector &Contexts) { + SymbolInfo Symbol; + Symbol.Name = Name; + Symbol.Type = Type; + Symbol.FilePath = FilePath; + Symbol.LineNumber = LineNumber; + Symbol.Contexts = Contexts; + return Symbol; +} + +TEST_F(FindAllSymbolsTest, VariableSymbols) { + static const char Code[] = R"( + extern int xargc; + namespace na { + static bool SSSS = false; + namespace nb { const long long *XXXX; } + })"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("xargc", SymbolInfo::Variable, HeaderName, 2, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.VariableInfos.getValue().Type); + } + { + SymbolInfo Symbol = + CreateSymbolInfo("SSSS", SymbolInfo::Variable, HeaderName, 4, + {{SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("_Bool", Symbol.VariableInfos.getValue().Type); + } + { + SymbolInfo Symbol = CreateSymbolInfo( + "XXXX", SymbolInfo::Variable, HeaderName, 5, + {{SymbolInfo::Namespace, "nb"}, {SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("const long long *", Symbol.VariableInfos.getValue().Type); + } +} + +TEST_F(FindAllSymbolsTest, ExternCSymbols) { + static const char Code[] = R"( + extern "C" { + int C_Func() { return 0; } + struct C_struct { + int Member; + }; + })"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("C_Func", SymbolInfo::Function, HeaderName, 3, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_TRUE(Symbol.FunctionInfos.getValue().ParameterTypes.empty()); + } + { + SymbolInfo Symbol = + CreateSymbolInfo("C_struct", SymbolInfo::Class, HeaderName, 4, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + } +} + +TEST_F(FindAllSymbolsTest, CXXRecordSymbols) { + static const char Code[] = R"( + struct Glob {}; + struct A; // Not a defintion, ignored. + class NOP; // Not a defintion, ignored + namespace na { + struct A { + struct AAAA {}; + int x; + int y; + void f() {} + }; + }; // + )"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("Glob", SymbolInfo::Class, HeaderName, 2, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + } + { + SymbolInfo Symbol = CreateSymbolInfo("A", SymbolInfo::Class, HeaderName, 6, + {{SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + } +} + +TEST_F(FindAllSymbolsTest, CXXRecordSymbolsTemplate) { + static const char Code[] = R"( + template + class T_TEMP { + template + struct rebind { typedef T_TEMP<_Tp1> other; }; + }; + // Ignore specialization. + template class T_TEMP; + + template + class Observer { + }; + // Ignore specialization. + template <> class Observer {}; + )"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("T_TEMP", SymbolInfo::Class, HeaderName, 3, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + } +} + +TEST_F(FindAllSymbolsTest, FunctionSymbols) { + static const char Code[] = R"( + namespace na { + int gg(int); + int f(const int &a) { int Local; static int StaticLocal; return 0; } + static void SSSFFF() {} + } // namespace na + namespace na { + namespace nb { + template + void fun(T t) {}; + } // namespace nb + } // namespace na"; + )"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = CreateSymbolInfo("gg", SymbolInfo::Class, HeaderName, 3, + {{SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_EQ(1, Symbol.FunctionInfos.getValue().ParameterTypes.size()); + EXPECT_EQ("int", Symbol.FunctionInfos.getValue().ParameterTypes[0]); + } + { + SymbolInfo Symbol = CreateSymbolInfo("f", SymbolInfo::Class, HeaderName, 4, + {{SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_EQ(1, Symbol.FunctionInfos.getValue().ParameterTypes.size()); + EXPECT_EQ("const int &", Symbol.FunctionInfos.getValue().ParameterTypes[0]); + } + { + SymbolInfo Symbol = + CreateSymbolInfo("SSSFFF", SymbolInfo::Class, HeaderName, 5, + {{SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("void", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_TRUE(Symbol.FunctionInfos.getValue().ParameterTypes.empty()); + } + { + SymbolInfo Symbol = CreateSymbolInfo( + "fun", SymbolInfo::Class, HeaderName, 10, + {{SymbolInfo::Namespace, "nb"}, {SymbolInfo::Namespace, "na"}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("void", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_EQ(1, Symbol.FunctionInfos.getValue().ParameterTypes.size()); + EXPECT_EQ("T", Symbol.FunctionInfos.getValue().ParameterTypes[0]); + } +} + +TEST_F(FindAllSymbolsTest, NamespaceTest) { + static const char Code[] = R"( + int X1; + namespace { int X2; } + namespace { namespace { int X3; } } + namespace { namespace nb { int X4;} } + )"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("X1", SymbolInfo::Variable, HeaderName, 2, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.VariableInfos.getValue().Type); + } + { + SymbolInfo Symbol = CreateSymbolInfo("X2", SymbolInfo::Variable, HeaderName, + 3, {{SymbolInfo::Namespace, ""}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.VariableInfos.getValue().Type); + } + { + SymbolInfo Symbol = CreateSymbolInfo( + "X3", SymbolInfo::Variable, HeaderName, 4, + {{SymbolInfo::Namespace, ""}, {SymbolInfo::Namespace, ""}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.VariableInfos.getValue().Type); + } + { + SymbolInfo Symbol = CreateSymbolInfo( + "X4", SymbolInfo::Variable, HeaderName, 5, + {{SymbolInfo::Namespace, "nb"}, {SymbolInfo::Namespace, ""}}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("int", Symbol.VariableInfos.getValue().Type); + } +} + +TEST_F(FindAllSymbolsTest, DecayedTypeTest) { + static const char Code[] = "void DecayedFunc(int x[], int y[10]) {}"; + runFindAllSymbols(Code); + SymbolInfo Symbol = + CreateSymbolInfo("DecayedFunc", SymbolInfo::Class, HeaderName, 1, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("void", Symbol.FunctionInfos.getValue().ReturnType); + EXPECT_EQ(2, Symbol.FunctionInfos.getValue().ParameterTypes.size()); + EXPECT_EQ("int *", Symbol.FunctionInfos.getValue().ParameterTypes[0]); + EXPECT_EQ("int *", Symbol.FunctionInfos.getValue().ParameterTypes[1]); +} + +TEST_F(FindAllSymbolsTest, CTypedefTest) { + static const char Code[] = R"( + typedef unsigned size_t; + typedef struct { int x; } X; + using XX = X; + )"; + runFindAllSymbols(Code); + + { + SymbolInfo Symbol = + CreateSymbolInfo("size_t", SymbolInfo::TypedefName, HeaderName, 2, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("unsigned int", + Symbol.TypedefNameInfos.getValue().UnderlyingType); + } + { + SymbolInfo Symbol = + CreateSymbolInfo("X", SymbolInfo::TypedefName, HeaderName, 3, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("struct X", Symbol.TypedefNameInfos.getValue().UnderlyingType); + } + { + SymbolInfo Symbol = + CreateSymbolInfo("XX", SymbolInfo::TypedefName, HeaderName, 4, {}); + EXPECT_TRUE(hasSymbol(Symbol)); + getSymbolExtraInfo(Symbol); + EXPECT_EQ("X", Symbol.TypedefNameInfos.getValue().UnderlyingType); + } +} + +} // namespace find_all_symbols +} // namespace clang