Index: include/clang/Basic/AllDiagnostics.h =================================================================== --- include/clang/Basic/AllDiagnostics.h +++ include/clang/Basic/AllDiagnostics.h @@ -21,6 +21,7 @@ #include "clang/CrossTU/CrossTUDiagnostic.h" #include "clang/Driver/DriverDiagnostic.h" #include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Index/IndexDiagnostic.h" #include "clang/Lex/LexDiagnostic.h" #include "clang/Parse/ParseDiagnostic.h" #include "clang/Sema/SemaDiagnostic.h" Index: include/clang/Basic/CMakeLists.txt =================================================================== --- include/clang/Basic/CMakeLists.txt +++ include/clang/Basic/CMakeLists.txt @@ -12,6 +12,7 @@ clang_diag_gen(CrossTU) clang_diag_gen(Driver) clang_diag_gen(Frontend) +clang_diag_gen(Index) clang_diag_gen(Lex) clang_diag_gen(Parse) clang_diag_gen(Refactoring) Index: include/clang/Basic/Diagnostic.td =================================================================== --- include/clang/Basic/Diagnostic.td +++ include/clang/Basic/Diagnostic.td @@ -136,6 +136,7 @@ include "DiagnosticCrossTUKinds.td" include "DiagnosticDriverKinds.td" include "DiagnosticFrontendKinds.td" +include "DiagnosticIndexKinds.td" include "DiagnosticLexKinds.td" include "DiagnosticParseKinds.td" include "DiagnosticRefactoringKinds.td" Index: include/clang/Basic/DiagnosticGroups.td =================================================================== --- include/clang/Basic/DiagnosticGroups.td +++ include/clang/Basic/DiagnosticGroups.td @@ -327,6 +327,7 @@ def ModuleBuild : DiagGroup<"module-build">; def ModuleConflict : DiagGroup<"module-conflict">; def ModuleFileExtension : DiagGroup<"module-file-extension">; +def IndexStore : DiagGroup<"index-store">; def NewlineEOF : DiagGroup<"newline-eof">; def Nullability : DiagGroup<"nullability">; def NullabilityDeclSpec : DiagGroup<"nullability-declspec">; Index: include/clang/Basic/DiagnosticIDs.h =================================================================== --- include/clang/Basic/DiagnosticIDs.h +++ include/clang/Basic/DiagnosticIDs.h @@ -40,6 +40,7 @@ DIAG_SIZE_SEMA = 3500, DIAG_SIZE_ANALYSIS = 100, DIAG_SIZE_REFACTORING = 1000, + DIAG_SIZE_INDEX = 100, }; // Start position for diagnostics. enum { @@ -55,7 +56,8 @@ DIAG_START_SEMA = DIAG_START_CROSSTU + DIAG_SIZE_COMMENT, DIAG_START_ANALYSIS = DIAG_START_SEMA + DIAG_SIZE_SEMA, DIAG_START_REFACTORING = DIAG_START_ANALYSIS + DIAG_SIZE_ANALYSIS, - DIAG_UPPER_LIMIT = DIAG_START_REFACTORING + DIAG_SIZE_REFACTORING + DIAG_START_INDEX = DIAG_START_REFACTORING + DIAG_SIZE_REFACTORING, + DIAG_UPPER_LIMIT = DIAG_START_INDEX + DIAG_SIZE_INDEX, }; class CustomDiagInfo; Index: include/clang/Basic/DiagnosticIndexKinds.td =================================================================== --- /dev/null +++ include/clang/Basic/DiagnosticIndexKinds.td @@ -0,0 +1,31 @@ +//==--- DiagnosticIndexKinds.td - indexing diagnostics --------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Indexing Diagnostics +//===----------------------------------------------------------------------===// + +let Component = "Index" in { + +let CategoryName = "Index Store Issue" in { + +def err_index_store_dir_create_failed : Error<"failed creating the index store " + "directory: %0">; +def err_index_store_file_status_failed : Error<"failed file status check: %0">; +def err_index_store_record_write_failed : Error<"failed writing record '%0': " + "%1">; +def err_index_store_unit_write_failed : Error<"failed writing unit data: %0">; + +def remark_index_producing_module_file_data : Remark<"producing index data for " + "module file '%0'">, + InGroup; + +} + +} // end of Indexing diagnostics Index: include/clang/Driver/Job.h =================================================================== --- include/clang/Driver/Job.h +++ include/clang/Driver/Job.h @@ -34,9 +34,11 @@ struct CrashReportInfo { StringRef Filename; StringRef VFSPath; + StringRef IndexStorePath; - CrashReportInfo(StringRef Filename, StringRef VFSPath) - : Filename(Filename), VFSPath(VFSPath) {} + CrashReportInfo(StringRef Filename, StringRef VFSPath, + StringRef IndexStorePath) + : Filename(Filename), VFSPath(VFSPath), IndexStorePath(IndexStorePath) {} }; /// Command - An executable path/name and argument vector to Index: include/clang/Driver/Options.td =================================================================== --- include/clang/Driver/Options.td +++ include/clang/Driver/Options.td @@ -324,6 +324,13 @@ def : Joined<["-"], "objcmt-white-list-dir-path=">, Flags<[CC1Option]>, Alias; +def index_store_path : Separate<["-"], "index-store-path">, Flags<[CC1Option]>, + HelpText<"Enable indexing with the specified data store path">; +def index_ignore_system_symbols : Flag<["-"], "index-ignore-system-symbols">, Flags<[CC1Option]>, + HelpText<"Ignore symbols from system headers">; +def index_record_codegen_name : Flag<["-"], "index-record-codegen-name">, Flags<[CC1Option]>, + HelpText<"Record the codegen name for symbols">; + // Make sure all other -ccc- options are rejected. def ccc_ : Joined<["-"], "ccc-">, Group, Flags<[Unsupported]>; Index: include/clang/Frontend/CompilerInstance.h =================================================================== --- include/clang/Frontend/CompilerInstance.h +++ include/clang/Frontend/CompilerInstance.h @@ -183,6 +183,14 @@ /// The list of active output files. std::list OutputFiles; + typedef std::function( + const FrontendOptions &opts, std::unique_ptr action)> + ActionWrapperTy; + + /// \brief An optional callback function used to wrap any + /// GenerateModuleActions created and executed when loading modules. + ActionWrapperTy GenModuleActionWrapper; + CompilerInstance(const CompilerInstance &) = delete; void operator=(const CompilerInstance &) = delete; public: @@ -796,6 +804,14 @@ bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override; + void setGenModuleActionWrapper(ActionWrapperTy Wrapper) { + GenModuleActionWrapper = Wrapper; + }; + + ActionWrapperTy getGenModuleActionWrapper() const { + return GenModuleActionWrapper; + } + void addDependencyCollector(std::shared_ptr Listener) { DependencyCollectors.push_back(std::move(Listener)); } Index: include/clang/Frontend/FrontendOptions.h =================================================================== --- include/clang/Frontend/FrontendOptions.h +++ include/clang/Frontend/FrontendOptions.h @@ -259,6 +259,13 @@ std::string MTMigrateDir; std::string ARCMTMigrateReportOut; + /// The path to write index data to + std::string IndexStorePath; + /// Whether to ignore system files when writing out index data + unsigned IndexIgnoreSystemSymbols : 1; + /// Whether to include the codegen name of symbols in the index data + unsigned IndexRecordCodegenName : 1; + /// The input files and their types. std::vector Inputs; @@ -336,8 +343,9 @@ SkipFunctionBodies(false), UseGlobalModuleIndex(true), GenerateGlobalModuleIndex(true), ASTDumpDecls(false), ASTDumpLookups(false), BuildingImplicitModule(false), ModulesEmbedAllFiles(false), - IncludeTimestamps(true), ARCMTAction(ARCMT_None), - ObjCMTAction(ObjCMT_None), ProgramAction(frontend::ParseSyntaxOnly) + IncludeTimestamps(true), ARCMTAction(ARCMT_None), ObjCMTAction(ObjCMT_None), + IndexIgnoreSystemSymbols(false), IndexRecordCodegenName(false), + ProgramAction(frontend::ParseSyntaxOnly) {} /// getInputKindForExtension - Return the appropriate input kind for a file Index: include/clang/Index/IndexDataConsumer.h =================================================================== --- include/clang/Index/IndexDataConsumer.h +++ include/clang/Index/IndexDataConsumer.h @@ -40,17 +40,18 @@ virtual bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, ArrayRef Relations, FileID FID, unsigned Offset, - ASTNodeInfo ASTNode); + bool IsInSystemFile, ASTNodeInfo ASTNode); /// \returns true to continue indexing, or false to abort. virtual bool handleMacroOccurence(const IdentifierInfo *Name, const MacroInfo *MI, SymbolRoleSet Roles, - FileID FID, unsigned Offset); + FileID FID, unsigned Offset, + bool IsInSystemFile); /// \returns true to continue indexing, or false to abort. virtual bool handleModuleOccurence(const ImportDecl *ImportD, - SymbolRoleSet Roles, - FileID FID, unsigned Offset); + SymbolRoleSet Roles, FileID FID, + unsigned Offset, bool IsInSystemFile); virtual void finish() {} Index: include/clang/Index/IndexDiagnostic.h =================================================================== --- /dev/null +++ include/clang/Index/IndexDiagnostic.h @@ -0,0 +1,29 @@ +//===--- IndexDiagnostic.h - ------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H +#define LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H + +#include "clang/Basic/Diagnostic.h" + +namespace clang { +namespace diag { +enum { +#define DIAG(ENUM, FLAGS, DEFAULT_MAPPING, DESC, GROUP, SFINAE, NOWERROR, \ + SHOWINSYSHEADER, CATEGORY) \ + ENUM, +#define INDEXSTART +#include "clang/Basic/DiagnosticIndexKinds.inc" +#undef DIAG + NUM_BUILTIN_INDEX_DIAGNOSTICS +}; +} // end namespace diag +} // end namespace clang + +#endif // LLVM_CLANG_INDEX_INDEXDIAGNOSTIC_H Index: include/clang/Index/IndexingAction.h =================================================================== --- include/clang/Index/IndexingAction.h +++ include/clang/Index/IndexingAction.h @@ -13,13 +13,17 @@ #include "clang/Basic/LLVM.h" #include "llvm/ADT/ArrayRef.h" #include +#include namespace clang { class ASTContext; class ASTReader; class ASTUnit; + class CompilerInstance; class Decl; class FrontendAction; + class FrontendOptions; + class Module; namespace serialization { class ModuleFile; @@ -27,6 +31,7 @@ namespace index { class IndexDataConsumer; + class IndexUnitWriter; struct IndexingOptions { enum class SystemSymbolFilterKind { @@ -35,11 +40,24 @@ All, }; - SystemSymbolFilterKind SystemSymbolFilter - = SystemSymbolFilterKind::DeclarationsOnly; + SystemSymbolFilterKind SystemSymbolFilter = + SystemSymbolFilterKind::DeclarationsOnly; bool IndexFunctionLocals = false; }; +struct RecordingOptions { + enum class IncludesRecordingKind { + None, + UserOnly, // only record includes inside non-system files. + All, + }; + + std::string DataDirPath; + bool RecordSymbolCodeGenName = false; + bool RecordSystemDependencies = true; + IncludesRecordingKind RecordIncludes = IncludesRecordingKind::UserOnly; +}; + /// \param WrappedAction another frontend action to wrap over or null. std::unique_ptr createIndexingAction(std::shared_ptr DataConsumer, @@ -58,6 +76,11 @@ std::shared_ptr DataConsumer, IndexingOptions Opts); +/// \param WrappedAction another frontend action to wrap over or null. +std::unique_ptr +createIndexDataRecordingAction(const FrontendOptions &FEOpts, + std::unique_ptr WrappedAction); + } // namespace index } // namespace clang Index: include/clang/module.modulemap =================================================================== --- include/clang/module.modulemap +++ include/clang/module.modulemap @@ -67,6 +67,7 @@ module Comment { header "AST/CommentDiagnostic.h" export * } module Driver { header "Driver/DriverDiagnostic.h" export * } module Frontend { header "Frontend/FrontendDiagnostic.h" export * } + module Index { header "Index/IndexDiagnostic.h" export * } module Lex { header "Lex/LexDiagnostic.h" export * } module Parse { header "Parse/ParseDiagnostic.h" export * } module Sema { header "Sema/SemaDiagnostic.h" export * } Index: lib/Basic/DiagnosticIDs.cpp =================================================================== --- lib/Basic/DiagnosticIDs.cpp +++ lib/Basic/DiagnosticIDs.cpp @@ -89,6 +89,7 @@ VALIDATE_DIAG_SIZE(SEMA) VALIDATE_DIAG_SIZE(ANALYSIS) VALIDATE_DIAG_SIZE(REFACTORING) +VALIDATE_DIAG_SIZE(INDEX) #undef VALIDATE_DIAG_SIZE #undef STRINGIFY_NAME @@ -114,6 +115,7 @@ #include "clang/Basic/DiagnosticSemaKinds.inc" #include "clang/Basic/DiagnosticAnalysisKinds.inc" #include "clang/Basic/DiagnosticRefactoringKinds.inc" +#include "clang/Basic/DiagnosticIndexKinds.inc" #undef DIAG }; @@ -153,6 +155,7 @@ CATEGORY(SEMA, CROSSTU) CATEGORY(ANALYSIS, SEMA) CATEGORY(REFACTORING, ANALYSIS) +CATEGORY(INDEX, REFACTORING) #undef CATEGORY // Avoid out of bounds reads. Index: lib/Driver/Driver.cpp =================================================================== --- lib/Driver/Driver.cpp +++ lib/Driver/Driver.cpp @@ -993,7 +993,9 @@ } // Assume associated files are based off of the first temporary file. - CrashReportInfo CrashInfo(TempFiles[0], VFS); + CrashReportInfo CrashInfo( + TempFiles[0], VFS, + C.getArgs().getLastArgValue(options::OPT_index_store_path)); std::string Script = CrashInfo.Filename.rsplit('.').first.str() + ".sh"; std::error_code EC; Index: lib/Driver/Job.cpp =================================================================== --- lib/Driver/Job.cpp +++ lib/Driver/Job.cpp @@ -68,6 +68,8 @@ .Default(false); if (IsInclude) return HaveCrashVFS ? false : true; + if (StringRef(Flag).startswith("-index-store-path")) + return true; // The remaining flags are treated as a single argument. @@ -220,6 +222,7 @@ } bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); + bool HaveIndexStorePath = CrashInfo && !CrashInfo->IndexStorePath.empty(); for (size_t i = 0, e = Args.size(); i < e; ++i) { const char *const Arg = Args[i]; @@ -283,6 +286,23 @@ printArg(OS, ModCachePath, Quote); } + if (CrashInfo && HaveIndexStorePath) { + SmallString<128> IndexStoreDir; + + if (HaveCrashVFS) { + IndexStoreDir = llvm::sys::path::parent_path( + llvm::sys::path::parent_path(CrashInfo->VFSPath)); + llvm::sys::path::append(IndexStoreDir, "index-store"); + } else { + IndexStoreDir = "index-store"; + } + + OS << ' '; + printArg(OS, "-index-store-path", Quote); + OS << ' '; + printArg(OS, IndexStoreDir.c_str(), Quote); + } + if (ResponseFile != nullptr) { OS << "\n Arguments passed via response file:\n"; writeResponseFile(OS); Index: lib/Driver/ToolChains/Clang.cpp =================================================================== --- lib/Driver/ToolChains/Clang.cpp +++ lib/Driver/ToolChains/Clang.cpp @@ -3582,6 +3582,18 @@ RenderARCMigrateToolOptions(D, Args, CmdArgs); + if (Args.hasArg(options::OPT_index_store_path)) { + Args.AddLastArg(CmdArgs, options::OPT_index_store_path); + Args.AddLastArg(CmdArgs, options::OPT_index_ignore_system_symbols); + Args.AddLastArg(CmdArgs, options::OPT_index_record_codegen_name); + + // If '-o' is passed along with '-fsyntax-only' pass it along the cc1 + // invocation so that the index action knows what the out file is. + if (isa(JA) && JA.getType() == types::TY_Nothing) { + Args.AddLastArg(CmdArgs, options::OPT_o); + } + } + // Add preprocessing options like -I, -D, etc. if we are using the // preprocessor. // Index: lib/Driver/ToolChains/Darwin.cpp =================================================================== --- lib/Driver/ToolChains/Darwin.cpp +++ lib/Driver/ToolChains/Darwin.cpp @@ -436,6 +436,10 @@ // more information. ArgStringList CmdArgs; + Args.ClaimAllArgs(options::OPT_index_store_path); + Args.ClaimAllArgs(options::OPT_index_ignore_system_symbols); + Args.ClaimAllArgs(options::OPT_index_record_codegen_name); + /// Hack(tm) to ignore linking errors when we are doing ARC migration. if (Args.hasArg(options::OPT_ccc_arcmt_check, options::OPT_ccc_arcmt_migrate)) { Index: lib/Frontend/CompilerInstance.cpp =================================================================== --- lib/Frontend/CompilerInstance.cpp +++ lib/Frontend/CompilerInstance.cpp @@ -28,6 +28,7 @@ #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" #include "clang/Frontend/VerifyDiagnosticConsumer.h" +#include "clang/Index/IndexingAction.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/PTHManager.h" #include "clang/Lex/Preprocessor.h" @@ -1148,6 +1149,10 @@ SourceMgr.pushModuleBuildStack(ModuleName, FullSourceLoc(ImportLoc, ImportingInstance.getSourceManager())); + // Pass along the GenModuleActionWrapper callback + auto WrapGenModuleAction = ImportingInstance.getGenModuleActionWrapper(); + Instance.setGenModuleActionWrapper(WrapGenModuleAction); + // If we're collecting module dependencies, we need to share a collector // between all of the module CompilerInstances. Other than that, we don't // want to produce any dependency output from the module build. @@ -1166,8 +1171,11 @@ llvm::CrashRecoveryContext CRC; CRC.RunSafelyOnThread( [&]() { - GenerateModuleFromModuleMapAction Action; - Instance.ExecuteAction(Action); + std::unique_ptr Action( + new GenerateModuleFromModuleMapAction); + if (WrapGenModuleAction) + Action = WrapGenModuleAction(FrontendOpts, std::move(Action)); + Instance.ExecuteAction(*Action); }, ThreadStackSize); Index: lib/Frontend/CompilerInvocation.cpp =================================================================== --- lib/Frontend/CompilerInvocation.cpp +++ lib/Frontend/CompilerInvocation.cpp @@ -1438,6 +1438,10 @@ << "ARC migration" << "ObjC migration"; } + Opts.IndexStorePath = Args.getLastArgValue(OPT_index_store_path); + Opts.IndexIgnoreSystemSymbols = Args.hasArg(OPT_index_ignore_system_symbols); + Opts.IndexRecordCodegenName = Args.hasArg(OPT_index_record_codegen_name); + InputKind DashX(InputKind::Unknown); if (const Arg *A = Args.getLastArg(OPT_x)) { StringRef XValue = A->getValue(); Index: lib/FrontendTool/CMakeLists.txt =================================================================== --- lib/FrontendTool/CMakeLists.txt +++ lib/FrontendTool/CMakeLists.txt @@ -8,6 +8,7 @@ clangCodeGen clangDriver clangFrontend + clangIndex clangRewriteFrontend ) Index: lib/FrontendTool/ExecuteCompilerInvocation.cpp =================================================================== --- lib/FrontendTool/ExecuteCompilerInvocation.cpp +++ lib/FrontendTool/ExecuteCompilerInvocation.cpp @@ -23,6 +23,7 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/FrontendPluginRegistry.h" #include "clang/Frontend/Utils.h" +#include "clang/Index/IndexingAction.h" #include "clang/Rewrite/Frontend/FrontendActions.h" #include "clang/StaticAnalyzer/Frontend/FrontendActions.h" #include "llvm/Option/OptTable.h" @@ -164,6 +165,12 @@ } #endif + if (!FEOpts.IndexStorePath.empty()) { + Act = index::createIndexDataRecordingAction(FEOpts, std::move(Act)); + // Also wrap any GenerateModuleActions created while loading modules + CI.setGenModuleActionWrapper(&index::createIndexDataRecordingAction); + } + // If there are any AST files to merge, create a frontend action // adaptor to perform the merge. if (!FEOpts.ASTMergeFiles.empty()) Index: lib/Index/CMakeLists.txt =================================================================== --- lib/Index/CMakeLists.txt +++ lib/Index/CMakeLists.txt @@ -6,6 +6,7 @@ add_clang_library(clangIndex CodegenNameGenerator.cpp CommentToXML.cpp + FileIndexRecord.cpp IndexBody.cpp IndexDecl.cpp IndexingAction.cpp @@ -23,6 +24,7 @@ clangBasic clangFormat clangFrontend + clangLex clangRewrite clangSerialization clangToolingCore Index: lib/Index/FileIndexRecord.h =================================================================== --- /dev/null +++ lib/Index/FileIndexRecord.h @@ -0,0 +1,74 @@ +//===--- FileIndexRecord.h - Index data per file --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H +#define LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H + +#include "clang/Basic/SourceLocation.h" +#include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include + +namespace clang { +class IdentifierInfo; + +namespace index { + +/// Stores the declaration occurrences seen in a particular source or header +/// file of a translation unit +class FileIndexRecord { +public: + struct DeclOccurrence { + SymbolRoleSet Roles; + unsigned Offset; + const Decl *Dcl; + SmallVector Relations; + + DeclOccurrence(SymbolRoleSet R, unsigned Offset, const Decl *D, + ArrayRef Relations) + : Roles(R), Offset(Offset), Dcl(D), + Relations(Relations.begin(), Relations.end()) {} + + friend bool operator<(const DeclOccurrence &LHS, + const DeclOccurrence &RHS) { + return LHS.Offset < RHS.Offset; + } + }; + +private: + FileID FID; + bool IsSystem; + std::vector Decls; + +public: + FileIndexRecord(FileID FID, bool isSystem) : FID(FID), IsSystem(isSystem) {} + + ArrayRef getDeclOccurrences() const { return Decls; } + + FileID getFileID() const { return FID; } + bool isSystem() const { return IsSystem; } + + /// Adds an occurrence of the canonical declaration \c D at the supplied + /// \c Offset + /// + /// \param Roles the roles the occurrence fulfills in this position. + /// \param Offset the offset in the file of this occurrence. + /// \param D the canonical declaration this is an occurrence of. + /// \param Relations the set of symbols related to this occurrence. + void addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, const Decl *D, + ArrayRef Relations); + void sortOccurrencesByOffset(); + void print(llvm::raw_ostream &OS) const; +}; + +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/FileIndexRecord.cpp =================================================================== --- /dev/null +++ lib/Index/FileIndexRecord.cpp @@ -0,0 +1,49 @@ +//===--- FileIndexRecord.cpp - Index data per file ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FileIndexRecord.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclTemplate.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Path.h" + +using namespace clang; +using namespace clang::index; + +void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, unsigned Offset, + const Decl *D, + ArrayRef Relations) { + assert(D->isCanonicalDecl() && + "Occurrences should be associated with their canonical decl"); + + Decls.emplace_back(Roles, Offset, D, Relations); +} + +void FileIndexRecord::sortOccurrencesByOffset() { + std::sort(Decls.begin(), Decls.end()); +} + +void FileIndexRecord::print(llvm::raw_ostream &OS) const { + OS << "DECLS BEGIN ---\n"; + for (auto &DclInfo : Decls) { + auto D = DclInfo.Dcl; + SourceManager &SM = D->getASTContext().getSourceManager(); + SourceLocation Loc = SM.getFileLoc(D->getLocation()); + PresumedLoc PLoc = SM.getPresumedLoc(Loc); + OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine() + << ':' << PLoc.getColumn(); + + if (auto ND = dyn_cast(D)) { + OS << ' ' << ND->getNameAsString(); + } + + OS << '\n'; + } + OS << "DECLS END ---\n"; +} Index: lib/Index/IndexingAction.cpp =================================================================== --- lib/Index/IndexingAction.cpp +++ lib/Index/IndexingAction.cpp @@ -8,10 +8,16 @@ //===----------------------------------------------------------------------===// #include "clang/Index/IndexingAction.h" -#include "clang/Index/IndexDataConsumer.h" +#include "FileIndexRecord.h" #include "IndexingContext.h" +#include "clang/Basic/FileManager.h" +#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/MultiplexConsumer.h" +#include "clang/Frontend/Utils.h" +#include "clang/Index/IndexDataConsumer.h" +#include "clang/Index/IndexDiagnostic.h" #include "clang/Lex/Preprocessor.h" #include "clang/Serialization/ASTReader.h" @@ -23,19 +29,23 @@ bool IndexDataConsumer::handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, ArrayRef Relations, FileID FID, unsigned Offset, + bool IsInSystemFile, ASTNodeInfo ASTNode) { return true; } bool IndexDataConsumer::handleMacroOccurence(const IdentifierInfo *Name, - const MacroInfo *MI, SymbolRoleSet Roles, - FileID FID, unsigned Offset) { + const MacroInfo *MI, + SymbolRoleSet Roles, FileID FID, + unsigned Offset, + bool IsInSystemFile) { return true; } bool IndexDataConsumer::handleModuleOccurence(const ImportDecl *ImportD, - SymbolRoleSet Roles, - FileID FID, unsigned Offset) { + SymbolRoleSet Roles, FileID FID, + unsigned Offset, + bool IsInSystemFile) { return true; } @@ -74,13 +84,16 @@ protected: std::shared_ptr DataConsumer; IndexingContext IndexCtx; + IsSystemFileCache IsSystemCache; IndexActionBase(std::shared_ptr dataConsumer, IndexingOptions Opts) - : DataConsumer(std::move(dataConsumer)), - IndexCtx(Opts, *DataConsumer) {} + : DataConsumer(std::move(dataConsumer)), + IndexCtx(Opts, *DataConsumer, IsSystemCache) {} - std::unique_ptr createIndexASTConsumer() { + std::unique_ptr + createIndexASTConsumer(CompilerInstance &CI) { + IsSystemCache.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); return llvm::make_unique(IndexCtx); } @@ -98,7 +111,7 @@ protected: std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override { - return createIndexASTConsumer(); + return createIndexASTConsumer(CI); } void EndSourceFileAction() override { @@ -108,7 +121,7 @@ }; class WrappingIndexAction : public WrapperFrontendAction, IndexActionBase { - bool IndexActionFailed = false; + bool CreatedASTConsumer = false; public: WrappingIndexAction(std::unique_ptr WrappedAction, @@ -128,21 +141,20 @@ void WrappingIndexAction::EndSourceFileAction() { // Invoke wrapped action's method. WrapperFrontendAction::EndSourceFileAction(); - if (!IndexActionFailed) + if (CreatedASTConsumer) finish(); } std::unique_ptr WrappingIndexAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); - if (!OtherConsumer) { - IndexActionFailed = true; + if (!OtherConsumer) return nullptr; - } + CreatedASTConsumer = true; std::vector> Consumers; Consumers.push_back(std::move(OtherConsumer)); - Consumers.push_back(createIndexASTConsumer()); + Consumers.push_back(createIndexASTConsumer(CI)); return llvm::make_unique(std::move(Consumers)); } @@ -170,7 +182,8 @@ void index::indexASTUnit(ASTUnit &Unit, std::shared_ptr DataConsumer, IndexingOptions Opts) { - IndexingContext IndexCtx(Opts, *DataConsumer); + IsSystemFileCache IsSystemCache; + IndexingContext IndexCtx(Opts, *DataConsumer, IsSystemCache); IndexCtx.setASTContext(Unit.getASTContext()); DataConsumer->initialize(Unit.getASTContext()); indexTranslationUnit(Unit, IndexCtx); @@ -180,7 +193,8 @@ void index::indexTopLevelDecls(ASTContext &Ctx, ArrayRef Decls, std::shared_ptr DataConsumer, IndexingOptions Opts) { - IndexingContext IndexCtx(Opts, *DataConsumer); + IsSystemFileCache IsSystemCache; + IndexingContext IndexCtx(Opts, *DataConsumer, IsSystemCache); IndexCtx.setASTContext(Ctx); DataConsumer->initialize(Ctx); @@ -194,7 +208,8 @@ std::shared_ptr DataConsumer, IndexingOptions Opts) { ASTContext &Ctx = Reader.getContext(); - IndexingContext IndexCtx(Opts, *DataConsumer); + IsSystemFileCache IsSystemCache; + IndexingContext IndexCtx(Opts, *DataConsumer, IsSystemCache); IndexCtx.setASTContext(Ctx); DataConsumer->initialize(Ctx); @@ -203,3 +218,414 @@ } DataConsumer->finish(); } + +//===----------------------------------------------------------------------===// +// Index Data Recording +//===----------------------------------------------------------------------===// + +namespace { + +class IndexDataRecorder : public IndexDataConsumer { + Preprocessor *PP = nullptr; + typedef llvm::DenseMap> + RecordByFileTy; + RecordByFileTy RecordByFile; + +public: + void init(Preprocessor &PreProc, ASTContext &Ctx) { + PP = &PreProc; + initialize(Ctx); + } + + RecordByFileTy::const_iterator record_begin() const { + return RecordByFile.begin(); + } + + RecordByFileTy::const_iterator record_end() const { + return RecordByFile.end(); + } + + bool record_empty() const { return RecordByFile.empty(); } + + void finish() override { + // Sort occurrences so ordering doesn't impact the hashing + for (auto &Entry : RecordByFile) + Entry.getSecond()->sortOccurrencesByOffset(); + } + +private: + bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, + ArrayRef Relations, FileID FID, + unsigned Offset, bool IsInSystemFile, + ASTNodeInfo ASTNode) override { + // Ignore the predefines buffer. + if (FID == PP->getPredefinesFileID()) + return true; + + FileIndexRecord &Rec = getFileIndexRecord(FID, IsInSystemFile); + Rec.addDeclOccurence(Roles, Offset, D, Relations); + return true; + } + + FileIndexRecord &getFileIndexRecord(FileID FID, bool IsInSystemFile) { + auto &Entry = RecordByFile[FID]; + if (!Entry) { + Entry.reset(new FileIndexRecord(FID, IsInSystemFile)); + } + return *Entry; + } +}; + +struct IncludeLocation { + const FileEntry *Source; + const FileEntry *Target; + unsigned Line; +}; + +class IncludePPCallbacks : public PPCallbacks { + IsSystemFileCache &IsSystemCache; + RecordingOptions::IncludesRecordingKind RecordIncludes; + std::vector &Includes; + SourceManager &SourceMgr; + +public: + IncludePPCallbacks(IsSystemFileCache &IsSystemCache, + RecordingOptions::IncludesRecordingKind RecordIncludes, + std::vector &IncludesForFile, + SourceManager &SourceMgr) + : IsSystemCache(IsSystemCache), RecordIncludes(RecordIncludes), + Includes(IncludesForFile), SourceMgr(SourceMgr) {} + +private: + void addInclude(SourceLocation From, const FileEntry *To) { + assert(To); + if (RecordIncludes == RecordingOptions::IncludesRecordingKind::None) + return; + + std::pair LocInfo = + SourceMgr.getDecomposedExpansionLoc(From); + + if (LocInfo.first.isInvalid()) + return; // Ignore invalid locations + + switch (RecordIncludes) { + case RecordingOptions::IncludesRecordingKind::None: + llvm_unreachable("should have already checked in the beginning"); + case RecordingOptions::IncludesRecordingKind::UserOnly: + if (IsSystemCache.isSystem(LocInfo.first, SourceMgr)) + return; // Ignore includes of system headers. + break; + case RecordingOptions::IncludesRecordingKind::All: + break; + } + + if (auto *FE = SourceMgr.getFileEntryForID(LocInfo.first)) { + auto lineNo = SourceMgr.getLineNumber(LocInfo.first, LocInfo.second); + Includes.push_back({FE, To, lineNo}); + } + } + + virtual void InclusionDirective(SourceLocation HashLoc, + const Token &IncludeTok, StringRef FileName, + bool IsAngled, CharSourceRange FilenameRange, + const FileEntry *File, StringRef SearchPath, + StringRef RelativePath, + const Module *Imported) override { + if (HashLoc.isFileID() && File && File->isValid()) + addInclude(HashLoc, File); + } +}; + +/// Abstract interface for providing the file and module dependencies of a +/// translation unit, as well as the set of file to file inclusions +class IndexDependencyProvider { +public: + virtual ~IndexDependencyProvider() {} + + virtual void visitFileDependencies( + const CompilerInstance &CI, + llvm::function_ref visitor) = 0; + virtual void + visitIncludes(llvm::function_ref + visitor) = 0; + virtual void visitModuleImports( + const CompilerInstance &CI, + llvm::function_ref + visitor) = 0; +}; + +/// Collects and provides the file and module dependency information, including +/// file to file inclusions, for the source files in a translation unit +class SourceFilesIndexDependencyCollector : public DependencyCollector, + public IndexDependencyProvider { + IsSystemFileCache &IsSystemCache; + RecordingOptions RecordOpts; + llvm::SetVector Entries; + llvm::BitVector IsSystemByUID; + std::vector Includes; + SourceManager *SourceMgr = nullptr; + +public: + SourceFilesIndexDependencyCollector(IsSystemFileCache &SysrootPath, + RecordingOptions recordOpts) + : IsSystemCache(SysrootPath), RecordOpts(recordOpts) {} + + void attachToPreprocessor(Preprocessor &PP) override { + DependencyCollector::attachToPreprocessor(PP); + PP.addPPCallbacks(llvm::make_unique( + IsSystemCache, RecordOpts.RecordIncludes, Includes, + PP.getSourceManager())); + } + + void setSourceManager(SourceManager *SourceMgr) { + this->SourceMgr = SourceMgr; + } + + void visitFileDependencies( + const CompilerInstance &CI, + llvm::function_ref visitor) + override { + for (auto *FE : getEntries()) { + visitor(FE, isSystemFile(FE)); + } + } + + void + visitIncludes(llvm::function_ref + visitor) override { + for (auto &Include : Includes) + visitor(Include.Source, Include.Line, Include.Target); + } + + void visitModuleImports( + const CompilerInstance &CI, + llvm::function_ref + visitor) override { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + + if (auto Reader = CI.getModuleManager()) { + Reader->getModuleManager().visit( + [&](serialization::ModuleFile &Mod) -> bool { + bool isSystemMod = false; + if (Mod.isModule()) { + if (auto *M = + HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false)) + isSystemMod = M->IsSystem; + } + if (!isSystemMod || needSystemDependencies()) + visitor(Mod, isSystemMod); + return true; // skip module dependencies. + }); + } + } + +private: + bool isSystemFile(const FileEntry *FE) { + auto UID = FE->getUID(); + return IsSystemByUID.size() > UID && IsSystemByUID[UID]; + } + + ArrayRef getEntries() const { + return Entries.getArrayRef(); + } + + bool needSystemDependencies() override { + return RecordOpts.RecordSystemDependencies; + } + + bool sawDependency(StringRef Filename, bool FromModule, bool IsSystem, + bool IsModuleFile, bool IsMissing) override { + bool sawIt = DependencyCollector::sawDependency( + Filename, FromModule, IsSystem, IsModuleFile, IsMissing); + if (auto *FE = SourceMgr->getFileManager().getFile(Filename)) { + if (sawIt) + Entries.insert(FE); + // Record system-ness for all files that we pass through. + if (IsSystemByUID.size() < FE->getUID() + 1) + IsSystemByUID.resize(FE->getUID() + 1); + IsSystemByUID[FE->getUID()] = IsSystem || isInSysroot(Filename); + } + return sawIt; + } + + bool isInSysroot(StringRef Filename) { + StringRef SysrootPath = IsSystemCache.getSysrootPath(); + return !SysrootPath.empty() && Filename.startswith(SysrootPath); + } +}; + +class IndexRecordActionBase { +protected: + RecordingOptions RecordOpts; + IndexDataRecorder Recorder; + IndexingContext IndexCtx; + IsSystemFileCache IsSystemCache; + SourceFilesIndexDependencyCollector DepCollector; + + IndexRecordActionBase(IndexingOptions IndexOpts, RecordingOptions recordOpts) + : RecordOpts(std::move(recordOpts)), + IndexCtx(IndexOpts, Recorder, IsSystemCache), + DepCollector(IsSystemCache, RecordOpts) {} + + std::unique_ptr + createIndexASTConsumer(CompilerInstance &CI) { + IsSystemCache.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); + + Preprocessor &PP = CI.getPreprocessor(); + Recorder.init(PP, CI.getASTContext()); + + DepCollector.setSourceManager(&CI.getSourceManager()); + DepCollector.attachToPreprocessor(PP); + + return llvm::make_unique(IndexCtx); + } + + void finish(CompilerInstance &CI); +}; + +class IndexRecordAction : public ASTFrontendAction, IndexRecordActionBase { +public: + IndexRecordAction(IndexingOptions IndexOpts, RecordingOptions RecordOpts) + : IndexRecordActionBase(std::move(IndexOpts), std::move(RecordOpts)) {} + +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + return createIndexASTConsumer(CI); + } + + void EndSourceFileAction() override { + FrontendAction::EndSourceFileAction(); + finish(getCompilerInstance()); + } +}; + +class WrappingIndexRecordAction : public WrapperFrontendAction, + IndexRecordActionBase { + bool CreatedASTConsumer = false; + +public: + WrappingIndexRecordAction(std::unique_ptr WrappedAction, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts) + : WrapperFrontendAction(std::move(WrappedAction)), + IndexRecordActionBase(std::move(IndexOpts), std::move(RecordOpts)) {} + +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); + if (!OtherConsumer) + return nullptr; + + CreatedASTConsumer = true; + std::vector> Consumers; + Consumers.push_back(std::move(OtherConsumer)); + Consumers.push_back(createIndexASTConsumer(CI)); + return llvm::make_unique(std::move(Consumers)); + } + + void EndSourceFileAction() override { + // Invoke wrapped action's method. + WrapperFrontendAction::EndSourceFileAction(); + if (CreatedASTConsumer) + finish(getCompilerInstance()); + } +}; + +} // anonymous namespace + +static void writeUnitData(const CompilerInstance &CI, + IndexDataRecorder &Recorder, + IndexDependencyProvider &DepProvider, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, StringRef OutputFile, + const FileEntry *RootFile, Module *UnitModule, + StringRef SysrootPath); + +void IndexRecordActionBase::finish(CompilerInstance &CI) { + // We may emit more diagnostics so do the begin/end source file invocations + // on the diagnostic client. + // FIXME: FrontendAction::EndSourceFile() should probably not call + // CI.getDiagnosticClient().EndSourceFile()' until after it has called + // 'EndSourceFileAction()', so that code executing during + // EndSourceFileAction() can emit diagnostics. If this is fixed, + // DiagClientBeginEndRAII can go away. + struct DiagClientBeginEndRAII { + CompilerInstance &CI; + DiagClientBeginEndRAII(CompilerInstance &CI) : CI(CI) { + CI.getDiagnosticClient().BeginSourceFile(CI.getLangOpts()); + } + ~DiagClientBeginEndRAII() { CI.getDiagnosticClient().EndSourceFile(); } + } diagClientBeginEndRAII(CI); + + SourceManager &SM = CI.getSourceManager(); + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + + std::string OutputFile = CI.getFrontendOpts().OutputFile; + if (OutputFile.empty()) { + OutputFile = CI.getFrontendOpts().Inputs[0].getFile(); + OutputFile += ".o"; + } + + const FileEntry *RootFile = nullptr; + Module *UnitMod = nullptr; + bool isModuleGeneration = CI.getLangOpts().isCompilingModule(); + if (!isModuleGeneration && + CI.getFrontendOpts().ProgramAction != frontend::GeneratePCH) { + RootFile = SM.getFileEntryForID(SM.getMainFileID()); + } + if (isModuleGeneration) { + UnitMod = HS.lookupModule(CI.getLangOpts().CurrentModule, + /*AllowSearch=*/false); + } + Recorder.finish(); + writeUnitData(CI, Recorder, DepCollector, IndexCtx.getIndexOpts(), RecordOpts, + OutputFile, RootFile, UnitMod, IsSystemCache.getSysrootPath()); +} + +static void writeUnitData(const CompilerInstance &CI, + IndexDataRecorder &Recorder, + IndexDependencyProvider &DepProvider, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, StringRef OutputFile, + const FileEntry *RootFile, Module *UnitModule, + StringRef SysrootPath) { + + // TODO persist collected index data +} + +static std::unique_ptr +createIndexDataRecordingAction(IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + std::unique_ptr WrappedAction) { + if (WrappedAction) + return llvm::make_unique( + std::move(WrappedAction), std::move(IndexOpts), std::move(RecordOpts)); + return llvm::make_unique(std::move(IndexOpts), + std::move(RecordOpts)); +} + +static std::pair +getIndexOptionsFromFrontendOptions(const FrontendOptions &FEOpts) { + index::IndexingOptions IndexOpts; + index::RecordingOptions RecordOpts; + RecordOpts.DataDirPath = FEOpts.IndexStorePath; + if (FEOpts.IndexIgnoreSystemSymbols) { + IndexOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::None; + } + RecordOpts.RecordSymbolCodeGenName = FEOpts.IndexRecordCodegenName; + return {IndexOpts, RecordOpts}; +} + +std::unique_ptr index::createIndexDataRecordingAction( + const FrontendOptions &FEOpts, + std::unique_ptr WrappedAction) { + auto IndexAndRecordOpts = getIndexOptionsFromFrontendOptions(FEOpts); + return ::createIndexDataRecordingAction(IndexAndRecordOpts.first, + IndexAndRecordOpts.second, + std::move(WrappedAction)); +} Index: lib/Index/IndexingContext.h =================================================================== --- lib/Index/IndexingContext.h +++ lib/Index/IndexingContext.h @@ -11,9 +11,11 @@ #define LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H #include "clang/Basic/LLVM.h" +#include "clang/Basic/SourceLocation.h" #include "clang/Index/IndexSymbol.h" #include "clang/Index/IndexingAction.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" namespace clang { class ASTContext; @@ -29,19 +31,43 @@ class Stmt; class Expr; class TypeLoc; - class SourceLocation; + class DirectoryEntry; namespace index { class IndexDataConsumer; +/// Tracks the current system root path and computes and caches whether a +/// file is considered a system file or not +class IsSystemFileCache { + std::string SysrootPath; + // Records whether a directory entry is system or not. + llvm::DenseMap DirEntries; + // Keeps track of the last check for whether a FileID is system or + // not. This is used to speed up isSystemFile() call. + std::pair LastFileCheck; + +public: + IsSystemFileCache() = default; + IsSystemFileCache(std::string SysrootPath); + + void setSysrootPath(StringRef path); + StringRef getSysrootPath() const { return SysrootPath; } + bool isSystem(FileID FID, SourceManager &SM); +}; + +/// Generates and reports indexing data to the provided \c IndexDataConsumer +/// for any AST nodes passed to its various \c index* methods. class IndexingContext { IndexingOptions IndexOpts; IndexDataConsumer &DataConsumer; + IsSystemFileCache &IsSystemCache; ASTContext *Ctx = nullptr; public: - IndexingContext(IndexingOptions IndexOpts, IndexDataConsumer &DataConsumer) - : IndexOpts(IndexOpts), DataConsumer(DataConsumer) {} + IndexingContext(IndexingOptions IndexOpts, IndexDataConsumer &DataConsumer, + IsSystemFileCache &IsSystemCache) + : IndexOpts(IndexOpts), DataConsumer(DataConsumer), + IsSystemCache(IsSystemCache) {} const IndexingOptions &getIndexOpts() const { return IndexOpts; } IndexDataConsumer &getDataConsumer() { return DataConsumer; } Index: lib/Index/IndexingContext.cpp =================================================================== --- lib/Index/IndexingContext.cpp +++ lib/Index/IndexingContext.cpp @@ -8,14 +8,16 @@ //===----------------------------------------------------------------------===// #include "IndexingContext.h" -#include "clang/Index/IndexDataConsumer.h" #include "clang/AST/ASTContext.h" -#include "clang/AST/DeclTemplate.h" #include "clang/AST/DeclObjC.h" +#include "clang/AST/DeclTemplate.h" #include "clang/Basic/SourceManager.h" +#include "clang/Index/IndexDataConsumer.h" +#include "llvm/Support/Path.h" using namespace clang; using namespace index; +using namespace llvm; static bool isGeneratedDecl(const Decl *D) { if (auto *attr = D->getAttr()) { @@ -24,6 +26,57 @@ return false; } +void IsSystemFileCache::setSysrootPath(llvm::StringRef path) { + // Ignore sysroot path if it points to root, otherwise every header will be + // treated as system one. + if (sys::path::root_path(path) == path) + path = StringRef(); + SysrootPath = path; +} + +IsSystemFileCache::IsSystemFileCache(std::string path) { setSysrootPath(path); } + +bool IsSystemFileCache::isSystem(clang::FileID FID, clang::SourceManager &SM) { + if (LastFileCheck.first == FID) + return LastFileCheck.second; + + auto result = [&](bool res) -> bool { + LastFileCheck = {FID, res}; + return res; + }; + + bool Invalid = false; + const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); + if (Invalid || !SEntry.isFile()) + return result(false); + + const SrcMgr::FileInfo &FI = SEntry.getFile(); + if (FI.getFileCharacteristic() != SrcMgr::C_User) + return result(true); + + auto *CC = FI.getContentCache(); + if (!CC) + return result(false); + auto *FE = CC->OrigEntry; + if (!FE) + return result(false); + + if (SysrootPath.empty()) + return result(false); + + // Check if directory is in sysroot so that we can consider system headers + // even the headers found via a user framework search path, pointing inside + // sysroot. + auto dirEntry = FE->getDir(); + auto pair = DirEntries.insert(std::make_pair(dirEntry, false)); + bool &isSystemDir = pair.first->second; + bool wasInserted = pair.second; + if (wasInserted) { + isSystemDir = StringRef(dirEntry->getName()).startswith(SysrootPath); + } + return result(isSystemDir); +} + bool IndexingContext::shouldIndex(const Decl *D) { return !isGeneratedDecl(D); } @@ -93,12 +146,8 @@ if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + bool IsInSystemFile = IsSystemCache.isSystem(FID, SM); + if (IsInSystemFile) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; @@ -112,7 +161,8 @@ if (ImportD->isImplicit()) Roles |= (unsigned)SymbolRole::Implicit; - return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset); + return DataConsumer.handleModuleOccurence(ImportD, Roles, FID, Offset, + IsInSystemFile); } bool IndexingContext::isTemplateImplicitInstantiation(const Decl *D) { @@ -205,7 +255,8 @@ return nullptr; } -static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, ASTContext &Ctx) { +static bool isDeclADefinition(const Decl *D, const DeclContext *ContainerDC, + ASTContext &Ctx) { if (auto VD = dyn_cast(D)) return VD->isThisDeclarationADefinition(Ctx); @@ -321,7 +372,7 @@ const Expr *OrigE, const Decl *OrigD, const DeclContext *ContainerDC) { - if (D->isImplicit() && !isa(D)) + if (D->isImplicit() && !(isa(D) || isa(D))) return true; if (!isa(D) || shouldSkipNamelessDecl(cast(D))) return true; @@ -337,12 +388,8 @@ if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + bool IsInSystemFile = IsSystemCache.isSystem(FID, SM); + if (IsInSystemFile) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; @@ -416,5 +463,5 @@ IndexDataConsumer::ASTNodeInfo Node{ OrigE, OrigD, Parent, ContainerDC }; return DataConsumer.handleDeclOccurence(D, Roles, FinalRelations, FID, Offset, - Node); + IsInSystemFile, Node); } Index: test/Index/Store/assembly-invocation.c =================================================================== --- /dev/null +++ test/Index/Store/assembly-invocation.c @@ -0,0 +1,3 @@ +// Make sure it doesn't crash. +// RUN: %clang -target x86_64-apple-macosx10.7 -S %s -o %t.s +// RUN: %clang -target x86_64-apple-macosx10.7 -c %t.s -o %t.o -index-store-path %t.idx Index: tools/c-index-test/core_main.cpp =================================================================== --- tools/c-index-test/core_main.cpp +++ tools/c-index-test/core_main.cpp @@ -87,8 +87,8 @@ } bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, - ArrayRef Relations, - FileID FID, unsigned Offset, + ArrayRef Relations, FileID FID, + unsigned Offset, bool IsInSystemFile, ASTNodeInfo ASTNode) override { ASTContext &Ctx = D->getASTContext(); SourceManager &SM = Ctx.getSourceManager(); @@ -124,7 +124,8 @@ } bool handleModuleOccurence(const ImportDecl *ImportD, SymbolRoleSet Roles, - FileID FID, unsigned Offset) override { + FileID FID, unsigned Offset, + bool IsInSystemFile) override { ASTContext &Ctx = ImportD->getASTContext(); SourceManager &SM = Ctx.getSourceManager(); Index: tools/diagtool/DiagnosticNames.cpp =================================================================== --- tools/diagtool/DiagnosticNames.cpp +++ tools/diagtool/DiagnosticNames.cpp @@ -43,6 +43,7 @@ #include "clang/Basic/DiagnosticSemaKinds.inc" #include "clang/Basic/DiagnosticAnalysisKinds.inc" #include "clang/Basic/DiagnosticRefactoringKinds.inc" +#include "clang/Basic/DiagnosticIndexKinds.inc" #undef DIAG }; Index: tools/libclang/CXIndexDataConsumer.h =================================================================== --- tools/libclang/CXIndexDataConsumer.h +++ tools/libclang/CXIndexDataConsumer.h @@ -463,12 +463,12 @@ private: bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, ArrayRef Relations, - FileID FID, unsigned Offset, + FileID FID, unsigned Offset, bool IsInSystemFile, ASTNodeInfo ASTNode) override; bool handleModuleOccurence(const ImportDecl *ImportD, - index::SymbolRoleSet Roles, - FileID FID, unsigned Offset) override; + index::SymbolRoleSet Roles, FileID FID, + unsigned Offset, bool IsInSystemFile) override; void finish() override; Index: tools/libclang/CXIndexDataConsumer.cpp =================================================================== --- tools/libclang/CXIndexDataConsumer.cpp +++ tools/libclang/CXIndexDataConsumer.cpp @@ -150,11 +150,9 @@ }; } -bool CXIndexDataConsumer::handleDeclOccurence(const Decl *D, - SymbolRoleSet Roles, - ArrayRef Relations, - FileID FID, unsigned Offset, - ASTNodeInfo ASTNode) { +bool CXIndexDataConsumer::handleDeclOccurence( + const Decl *D, SymbolRoleSet Roles, ArrayRef Relations, + FileID FID, unsigned Offset, bool IsInSystemFile, ASTNodeInfo ASTNode) { SourceLocation Loc = getASTContext().getSourceManager() .getLocForStartOfFile(FID).getLocWithOffset(Offset); @@ -219,9 +217,9 @@ } bool CXIndexDataConsumer::handleModuleOccurence(const ImportDecl *ImportD, - SymbolRoleSet Roles, - FileID FID, - unsigned Offset) { + SymbolRoleSet Roles, FileID FID, + unsigned Offset, + bool IsInSystemFile) { IndexingDeclVisitor(*this, SourceLocation(), nullptr).Visit(ImportD); return !shouldAbort(); }