Index: include/clang/Basic/DiagnosticFrontendKinds.td =================================================================== --- include/clang/Basic/DiagnosticFrontendKinds.td +++ include/clang/Basic/DiagnosticFrontendKinds.td @@ -216,6 +216,10 @@ Error<"file '%0' specified by '-fmodules-embed-file=' not found">, DefaultFatal; +def remark_index_producing_module_file_data : Remark<"producing index data for " + "module file '%0'">, + InGroup; + def err_test_module_file_extension_version : Error< "test module file extension '%0' has different version (%1.%2) than expected " "(%3.%4)">; Index: include/clang/Basic/DiagnosticGroups.td =================================================================== --- include/clang/Basic/DiagnosticGroups.td +++ include/clang/Basic/DiagnosticGroups.td @@ -322,6 +322,7 @@ def ModuleBuild : DiagGroup<"module-build">; def ModuleConflict : DiagGroup<"module-conflict">; def ModuleFileExtension : DiagGroup<"module-file-extension">; +def IndexStore : DiagGroup<"index-store">; def NewlineEOF : DiagGroup<"newline-eof">; def Nullability : DiagGroup<"nullability">; def NullabilityDeclSpec : DiagGroup<"nullability-declspec">; Index: include/clang/DirectoryWatcher/DirectoryWatcher.h =================================================================== --- /dev/null +++ include/clang/DirectoryWatcher/DirectoryWatcher.h @@ -0,0 +1,47 @@ +//===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// \brief Utility class for listening for file system changes in a directory. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H +#define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H + +#include "clang/Basic/LLVM.h" +#include "clang/Index/IndexDataStore.h" +#include +#include +#include + +namespace clang { + +/// Provides notifications for file system changes in a directory. +/// +/// Guarantees that the first time the directory is processed, the receiver will +/// be invoked even if the directory is empty. +class DirectoryWatcher : public index::AbstractDirectoryWatcher { + struct Implementation; + Implementation &Impl; + + DirectoryWatcher(); + + DirectoryWatcher(const DirectoryWatcher&) = delete; + DirectoryWatcher &operator =(const DirectoryWatcher&) = delete; + +public: + ~DirectoryWatcher(); + + static std::unique_ptr + create(StringRef Path, EventReceiver Receiver, bool waitInitialSync, + std::string &Error); +}; + +} // namespace clang + +#endif Index: include/clang/Driver/Job.h =================================================================== --- include/clang/Driver/Job.h +++ include/clang/Driver/Job.h @@ -34,9 +34,11 @@ struct CrashReportInfo { StringRef Filename; StringRef VFSPath; + StringRef IndexStorePath; - CrashReportInfo(StringRef Filename, StringRef VFSPath) - : Filename(Filename), VFSPath(VFSPath) {} + CrashReportInfo(StringRef Filename, StringRef VFSPath, + StringRef IndexStorePath) + : Filename(Filename), VFSPath(VFSPath), IndexStorePath(IndexStorePath) {} }; /// Command - An executable path/name and argument vector to Index: include/clang/Driver/Options.td =================================================================== --- include/clang/Driver/Options.td +++ include/clang/Driver/Options.td @@ -320,6 +320,13 @@ def : Joined<["-"], "objcmt-white-list-dir-path=">, Flags<[CC1Option]>, Alias; +def index_store_path : Separate<["-"], "index-store-path">, Flags<[CC1Option]>, + HelpText<"Enable indexing with the specified data store path">; +def index_ignore_system_symbols : Flag<["-"], "index-ignore-system-symbols">, Flags<[CC1Option]>, + HelpText<"Ignore symbols from system headers">; +def index_record_codegen_name : Flag<["-"], "index-record-codegen-name">, Flags<[CC1Option]>, + HelpText<"Record the codegen name for symbols">; + // Make sure all other -ccc- options are rejected. def ccc_ : Joined<["-"], "ccc-">, Group, Flags<[Unsupported]>; Index: include/clang/Frontend/CompilerInstance.h =================================================================== --- include/clang/Frontend/CompilerInstance.h +++ include/clang/Frontend/CompilerInstance.h @@ -183,6 +183,12 @@ /// The list of active output files. std::list OutputFiles; + /// \brief An optional callback function used to wrap all FrontendActions + /// produced to generate imported modules before they are executed. + std::function + (const FrontendOptions &opts, std::unique_ptr action)> + GenModuleActionWrapper; + CompilerInstance(const CompilerInstance &) = delete; void operator=(const CompilerInstance &) = delete; public: @@ -796,6 +802,15 @@ bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override; + void setGenModuleActionWrapper(std::function + (const FrontendOptions &Opts, std::unique_ptr Action)> Wrapper) { + GenModuleActionWrapper = Wrapper; + }; + + std::function + (const FrontendOptions &Opts, std::unique_ptr Action)> + getGenModuleActionWrapper() const { return GenModuleActionWrapper; } + void addDependencyCollector(std::shared_ptr Listener) { DependencyCollectors.push_back(std::move(Listener)); } Index: include/clang/Frontend/FrontendOptions.h =================================================================== --- include/clang/Frontend/FrontendOptions.h +++ include/clang/Frontend/FrontendOptions.h @@ -259,6 +259,10 @@ std::string MTMigrateDir; std::string ARCMTMigrateReportOut; + std::string IndexStorePath; + unsigned IndexIgnoreSystemSymbols : 1; + unsigned IndexRecordCodegenName : 1; + /// The input files and their types. std::vector Inputs; @@ -336,8 +340,9 @@ SkipFunctionBodies(false), UseGlobalModuleIndex(true), GenerateGlobalModuleIndex(true), ASTDumpDecls(false), ASTDumpLookups(false), BuildingImplicitModule(false), ModulesEmbedAllFiles(false), - IncludeTimestamps(true), ARCMTAction(ARCMT_None), - ObjCMTAction(ObjCMT_None), ProgramAction(frontend::ParseSyntaxOnly) + IncludeTimestamps(true), ARCMTAction(ARCMT_None), ObjCMTAction(ObjCMT_None), + IndexIgnoreSystemSymbols(false), IndexRecordCodegenName(false), + ProgramAction(frontend::ParseSyntaxOnly) {} /// getInputKindForExtension - Return the appropriate input kind for a file Index: include/clang/Index/IndexDataStore.h =================================================================== --- /dev/null +++ include/clang/Index/IndexDataStore.h @@ -0,0 +1,102 @@ +//===--- IndexDataStore.h - Index data store info -------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXDATASTORE_H +#define LLVM_CLANG_INDEX_INDEXDATASTORE_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLExtras.h" +#include +#include +#include +#include + +namespace clang { +namespace index { + +class AbstractDirectoryWatcher { +public: + enum class EventKind { + /// A file was added. + Added, + /// A file was removed. + Removed, + /// A file was modified. + Modified, + /// The watched directory got deleted. No more events will follow. + DirectoryDeleted, + }; + + struct Event { + EventKind Kind; + std::string Filename; + timespec ModTime; + }; + + typedef std::function Events, bool isInitial)> EventReceiver; + typedef std::unique_ptr(CreateFnTy) + (StringRef Path, EventReceiver Receiver, bool waitInitialSync, std::string &Error); + + virtual ~AbstractDirectoryWatcher() {} +}; + +class IndexDataStore { +public: + ~IndexDataStore(); + + static std::unique_ptr + create(StringRef IndexStorePath, std::string &Error); + + StringRef getFilePath() const; + bool foreachUnitName(bool sorted, + llvm::function_ref receiver); + + static unsigned getFormatVersion(); + + enum class UnitEventKind { + Added, + Removed, + Modified, + /// The directory got deleted. No more events will follow. + DirectoryDeleted, + }; + struct UnitEvent { + UnitEventKind Kind; + StringRef UnitName; + timespec ModTime; + }; + struct UnitEventNotification { + bool IsInitial; + ArrayRef Events; + }; + typedef std::function UnitEventHandler; + + void setUnitEventHandler(UnitEventHandler Handler); + /// \returns true if an error occurred. + bool startEventListening(llvm::function_ref createFn, + bool waitInitialSync, std::string &Error); + void stopEventListening(); + + void discardUnit(StringRef UnitName); + void discardRecord(StringRef RecordName); + + void purgeStaleData(); + +private: + IndexDataStore(void *Impl) : Impl(Impl) {} + + void *Impl; // An IndexDataStoreImpl. +}; + +} // namespace index +} // namespace clang + +#endif Index: include/clang/Index/IndexDataStoreSymbolUtils.h =================================================================== --- /dev/null +++ include/clang/Index/IndexDataStoreSymbolUtils.h @@ -0,0 +1,53 @@ +//===--- IndexDataStoreSymbolUtils.h - Utilities for indexstore symbols ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXDATASTORESYMBOLUTILS_H +#define LLVM_CLANG_INDEX_INDEXDATASTORESYMBOLUTILS_H + +#include "indexstore/indexstore.h" +#include "clang/Index/IndexSymbol.h" + +namespace clang { +namespace index { + +/// Map an indexstore_symbol_kind_t to a SymbolKind, handling unknown values. +SymbolKind getSymbolKind(indexstore_symbol_kind_t K); + +SymbolSubKind getSymbolSubKind(indexstore_symbol_subkind_t K); + +/// Map an indexstore_symbol_language_t to a SymbolLanguage, handling unknown +/// values. +SymbolLanguage getSymbolLanguage(indexstore_symbol_language_t L); + +/// Map an indexstore representation to a SymbolPropertySet, handling +/// unknown values. +SymbolPropertySet getSymbolProperties(uint64_t Props); + +/// Map an indexstore representation to a SymbolRoleSet, handling unknown +/// values. +SymbolRoleSet getSymbolRoles(uint64_t Roles); + +/// Map a SymbolLanguage to a indexstore_symbol_language_t. +indexstore_symbol_kind_t getIndexStoreKind(SymbolKind K); + +indexstore_symbol_subkind_t getIndexStoreSubKind(SymbolSubKind K); + +/// Map a SymbolLanguage to a indexstore_symbol_language_t. +indexstore_symbol_language_t getIndexStoreLang(SymbolLanguage L); + +/// Map a SymbolPropertySet to its indexstore representation. +uint64_t getIndexStoreProperties(SymbolPropertySet Props); + +/// Map a SymbolRoleSet to its indexstore representation. +uint64_t getIndexStoreRoles(SymbolRoleSet Roles); + +} // end namespace index +} // end namespace clang + +#endif // LLVM_CLANG_INDEX_INDEXDATASTORESYMBOLUTILS_H Index: include/clang/Index/IndexRecordReader.h =================================================================== --- /dev/null +++ include/clang/Index/IndexRecordReader.h @@ -0,0 +1,109 @@ +//===--- IndexRecordReader.h - Index record deserialization ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXRECORDREADER_H +#define LLVM_CLANG_INDEX_INDEXRECORDREADER_H + +#include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include + +namespace llvm { + class MemoryBuffer; +} + +namespace clang { +namespace index { + +struct IndexRecordDecl { + unsigned DeclID; + SymbolInfo SymInfo; + SymbolRoleSet Roles; + SymbolRoleSet RelatedRoles; + StringRef Name; + StringRef USR; + StringRef CodeGenName; +}; + +struct IndexRecordRelation { + SymbolRoleSet Roles; + const IndexRecordDecl *Dcl = nullptr; + + IndexRecordRelation() = default; + IndexRecordRelation(SymbolRoleSet Roles, const IndexRecordDecl *Dcl) + : Roles(Roles), Dcl(Dcl) {} +}; + +struct IndexRecordOccurrence { + const IndexRecordDecl *Dcl; + SmallVector Relations; + SymbolRoleSet Roles; + unsigned Line; + unsigned Column; +}; + +class IndexRecordReader { + IndexRecordReader(); + +public: + static std::unique_ptr + createWithRecordFilename(StringRef RecordFilename, StringRef StorePath, + std::string &Error); + static std::unique_ptr + createWithFilePath(StringRef FilePath, std::string &Error); + static std::unique_ptr + createWithBuffer(std::unique_ptr Buffer, + std::string &Error); + + ~IndexRecordReader(); + + struct DeclSearchReturn { + bool AcceptDecl; + bool ContinueSearch; + }; + typedef DeclSearchReturn(DeclSearchCheck)(const IndexRecordDecl &); + + /// Goes through and passes record decls, after filtering using a \c Checker + /// function. + /// + /// Resulting decls can be used as filter for \c foreachOccurrence. This + /// allows allocating memory only for the record decls that the caller is + /// interested in. + bool searchDecls(llvm::function_ref Checker, + llvm::function_ref Receiver); + + /// \param NoCache if true, avoids allocating memory for the decls. + /// Useful when the caller does not intend to keep \c IndexRecordReader + /// for more queries. + bool foreachDecl(bool NoCache, + llvm::function_ref Receiver); + + /// \param DeclsFilter if non-empty indicates the list of decls that we want + /// to get occurrences for. An empty array indicates that we want occurrences + /// for all decls. + /// \param RelatedDeclsFilter Same as \c DeclsFilter but for related decls. + bool foreachOccurrence(ArrayRef DeclsFilter, + ArrayRef RelatedDeclsFilter, + llvm::function_ref Receiver); + bool foreachOccurrence( + llvm::function_ref Receiver); + + bool foreachOccurrenceInLineRange(unsigned lineStart, unsigned lineCount, + llvm::function_ref Receiver); + + struct Implementation; +private: + Implementation &Impl; +}; + +} // namespace index +} // namespace clang + +#endif Index: include/clang/Index/IndexRecordWriter.h =================================================================== --- /dev/null +++ include/clang/Index/IndexRecordWriter.h @@ -0,0 +1,102 @@ +//===--- IndexRecordWriter.h - Index record serialization -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXRECORDWRITER_H +#define LLVM_CLANG_INDEX_INDEXRECORDWRITER_H + +#include "clang/Index/IndexSymbol.h" +#include "llvm/ADT/SmallString.h" + +namespace clang { +namespace index { + +namespace writer { +/// An opaque pointer to a declaration or other symbol used by the +/// IndexRecordWriter to identify when two occurrences refer to the same symbol, +/// and as a token for getting information about a symbol from the caller. +typedef const void *OpaqueDecl; + +/// An indexer symbol suitable for serialization. +/// +/// This includes all the information about the symbol that will be serialized +/// except for roles, which are synthesized by looking at all the occurrences. +/// +/// \seealso IndexRecordDecl +/// \note this struct is generally accompanied by a buffer that owns the string +/// storage. It should not be stored permanently. +struct Symbol { + SymbolInfo SymInfo; + StringRef Name; + StringRef USR; + StringRef CodeGenName; +}; + +/// An relation to an opaque symbol. +/// \seealso IndexRecordRelation +struct SymbolRelation { + OpaqueDecl RelatedSymbol; + SymbolRoleSet Roles; +}; + +typedef llvm::function_ref &Scratch)> + SymbolWriterCallback; +} // end namespace writer + +/// A language-independent utility for serializing index record files. +/// +/// Internally, this class is a small state machine. Users should first call +/// beginRecord, and if the file does not already exist, then proceed to add +/// all symbol occurrences (addOccurrence) and finally finish with endRecord. +class IndexRecordWriter { + SmallString<64> RecordsPath; ///< The records directory path. + void *Record = nullptr; ///< The state of the current record. +public: + IndexRecordWriter(StringRef IndexPath); + + enum class Result { + Success, + Failure, + AlreadyExists, + }; + + /// Begin writing a record for the file \p Filename with contents uniquely + /// identified by \p RecordHash. + /// + /// \param Filename the name of the file this is a record for. + /// \param RecordHash the unique hash of the record contents. + /// \param Error on failure, set to the error message. + /// \param RecordFile if non-null, this is set to the name of the record file. + /// + /// \returns Success if we should continue writing this record, AlreadyExists + /// if the record file has already been written, or Failure if there was an + /// error, in which case \p Error will be set. + Result beginRecord(StringRef Filename, llvm::hash_code RecordHash, + std::string &Error, std::string *RecordFile = nullptr); + + /// Finish writing the record file. + /// + /// \param Error on failure, set to the error message. + /// \param GetSymbolForDecl a callback mapping an writer::OpaqueDecl to its + /// writer::Symbol. This is how the language-specific symbol information is + /// provided to the IndexRecordWriter. The scratch parameter can be used for + /// any necessary storage. + /// + /// \return Success, or Failure and sets \p Error. + Result endRecord(std::string &Error, + writer::SymbolWriterCallback GetSymbolForDecl); + + /// Add an occurrence of the symbol \p D with the given \p Roles and location. + void addOccurrence(writer::OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, + unsigned Column, ArrayRef Related); +}; + +} // end namespace index +} // end namespace clang + +#endif // LLVM_CLANG_INDEX_INDEXRECORDWRITER_H Index: include/clang/Index/IndexUnitReader.h =================================================================== --- /dev/null +++ include/clang/Index/IndexUnitReader.h @@ -0,0 +1,85 @@ +//===--- IndexUnitReader.h - Index unit deserialization -------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXUNITREADER_H +#define LLVM_CLANG_INDEX_INDEXUNITREADER_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/Chrono.h" + +namespace clang { +namespace index { + +class IndexUnitReader { +public: + enum class DependencyKind { + Unit, + Record, + File, + }; + + ~IndexUnitReader(); + + static std::unique_ptr + createWithUnitFilename(StringRef UnitFilename, StringRef StorePath, + std::string &Error); + static std::unique_ptr + createWithFilePath(StringRef FilePath, std::string &Error); + + static Optional> + getModificationTimeForUnit(StringRef UnitFilename, StringRef StorePath, + std::string &Error); + + StringRef getProviderIdentifier() const; + StringRef getProviderVersion() const; + + llvm::sys::TimePoint<> getModificationTime() const; + StringRef getWorkingDirectory() const; + StringRef getOutputFile() const; + StringRef getSysrootPath() const; + StringRef getMainFilePath() const; + StringRef getModuleName() const; + StringRef getTarget() const; + bool hasMainFile() const; + bool isSystemUnit() const; + bool isModuleUnit() const; + bool isDebugCompilation() const; + + struct DependencyInfo { + DependencyKind Kind; + bool IsSystem; + StringRef UnitOrRecordName; + StringRef FilePath; + StringRef ModuleName; + size_t FileSize; + time_t ModTime; + }; + struct IncludeInfo { + StringRef SourcePath; + unsigned SourceLine; + StringRef TargetPath; + }; + /// Unit dependencies are provided ahead of record ones, record ones + /// ahead of the file ones. + bool foreachDependency(llvm::function_ref Receiver); + + bool foreachInclude(llvm::function_ref Receiver); + +private: + IndexUnitReader(void *Impl) : Impl(Impl) {} + + void *Impl; // An IndexUnitReaderImpl. +}; + +} // namespace index +} // namespace clang + +#endif Index: include/clang/Index/IndexUnitWriter.h =================================================================== --- /dev/null +++ include/clang/Index/IndexUnitWriter.h @@ -0,0 +1,140 @@ +//===--- IndexUnitWriter.h - Index unit serialization ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEX_INDEXUNITWRITER_H +#define LLVM_CLANG_INDEX_INDEXUNITWRITER_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallString.h" +#include +#include + +namespace llvm { + class BitstreamWriter; +} + +namespace clang { + class FileEntry; + class FileManager; + +namespace index { + +namespace writer { +/// An opaque pointer to a module used by the IndexUnitWriter to associate +/// record and file dependencies with a module, and as a token for getting +/// information about the module from the caller. +typedef const void *OpaqueModule; + +/// Module info suitable for serialization. +/// +/// This is used for top-level modules and sub-modules. +struct ModuleInfo { + /// Full, dot-separate, module name. + StringRef Name; +}; + +typedef llvm::function_ref &Scratch)> + ModuleInfoWriterCallback; +} // end namespace writer + +class IndexUnitWriter { + FileManager &FileMgr; + SmallString<64> UnitsPath; + std::string ProviderIdentifier; + std::string ProviderVersion; + std::string OutputFile; + std::string ModuleName; + const FileEntry *MainFile; + bool IsSystemUnit; + bool IsModuleUnit; + bool IsDebugCompilation; + std::string TargetTriple; + std::string WorkDir; + std::string SysrootPath; + std::function &Scratch)> GetInfoForModuleFn; + struct FileInclude { + int Index; + unsigned Line; + }; + struct FileEntryData { + const FileEntry *File; + bool IsSystem; + int ModuleIndex; + std::vector Includes; + }; + std::vector Files; + std::vector Modules; + llvm::DenseMap IndexByFile; + llvm::DenseMap IndexByModule; + llvm::DenseSet SeenASTFiles; + struct RecordOrUnitData { + std::string Name; + int FileIndex; + int ModuleIndex; + bool IsSystem; + }; + std::vector Records; + std::vector ASTFileUnits; + +public: + /// \param MainFile the main file for a compiled source file. This should be + /// null for PCH and module units. + /// \param IsSystem true for system module units, false otherwise. + IndexUnitWriter(FileManager &FileMgr, + StringRef StorePath, + StringRef ProviderIdentifier, StringRef ProviderVersion, + StringRef OutputFile, + StringRef ModuleName, + const FileEntry *MainFile, + bool IsSystem, + bool IsModuleUnit, + bool IsDebugCompilation, + StringRef TargetTriple, + StringRef SysrootPath, + writer::ModuleInfoWriterCallback GetInfoForModule); + ~IndexUnitWriter(); + + int addFileDependency(const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod); + void addRecordFile(StringRef RecordFile, const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod); + void addASTFileDependency(const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod, bool withoutUnitName = false); + void addUnitDependency(StringRef UnitFile, const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod); + bool addInclude(const FileEntry *Source, unsigned Line, const FileEntry *Target); + + bool write(std::string &Error); + + void getUnitNameForOutputFile(StringRef FilePath, SmallVectorImpl &Str); + void getUnitPathForOutputFile(StringRef FilePath, SmallVectorImpl &Str); + /// If the unit file exists and \p timeCompareFilePath is provided, it will + /// return true if \p timeCompareFilePath is older than the unit file. + Optional isUnitUpToDateForOutputFile(StringRef FilePath, + Optional TimeCompareFilePath, + std::string &Error); + static void getUnitNameForAbsoluteOutputFile(StringRef FilePath, SmallVectorImpl &Str); + static bool initIndexDirectory(StringRef StorePath, std::string &Error); + +private: + class PathStorage; + int addModule(writer::OpaqueModule Mod); + void writeUnitInfo(llvm::BitstreamWriter &Stream, PathStorage &PathStore); + void writeDependencies(llvm::BitstreamWriter &Stream, PathStorage &PathStore); + void writeIncludes(llvm::BitstreamWriter &Stream, PathStorage &PathStore); + void writePaths(llvm::BitstreamWriter &Stream, PathStorage &PathStore); + void writeModules(llvm::BitstreamWriter &Stream); +}; + +} // end namespace index +} // end namespace clang + +#endif Index: include/clang/Index/IndexingAction.h =================================================================== --- include/clang/Index/IndexingAction.h +++ include/clang/Index/IndexingAction.h @@ -13,13 +13,17 @@ #include "clang/Basic/LLVM.h" #include "llvm/ADT/ArrayRef.h" #include +#include namespace clang { class ASTContext; class ASTReader; class ASTUnit; + class CompilerInstance; class Decl; class FrontendAction; + class FrontendOptions; + class Module; namespace serialization { class ModuleFile; @@ -27,6 +31,7 @@ namespace index { class IndexDataConsumer; + class IndexUnitWriter; struct IndexingOptions { enum class SystemSymbolFilterKind { @@ -40,6 +45,19 @@ bool IndexFunctionLocals = false; }; +struct RecordingOptions { + enum class IncludesRecordingKind { + None, + UserOnly, // only record includes inside non-system files. + All, + }; + + std::string DataDirPath; + bool RecordSymbolCodeGenName = false; + bool RecordSystemDependencies = true; + IncludesRecordingKind RecordIncludes = IncludesRecordingKind::UserOnly; +}; + /// \param WrappedAction another frontend action to wrap over or null. std::unique_ptr createIndexingAction(std::shared_ptr DataConsumer, @@ -58,6 +76,18 @@ std::shared_ptr DataConsumer, IndexingOptions Opts); +/// \param WrappedAction another frontend action to wrap over or null. +std::unique_ptr +createIndexDataRecordingAction(const FrontendOptions &FEOpts, + std::unique_ptr WrappedAction); + +/// Checks if the unit file exists for the module file, if it doesn't it +/// generates index data for it. +/// +/// \returns true if the index data were generated, false otherwise. +bool emitIndexDataForModuleFile(const Module *Mod, const CompilerInstance &CI, + IndexUnitWriter &ParentUnitWriter); + } // namespace index } // namespace clang Index: include/indexstore/IndexStoreCXX.h =================================================================== --- /dev/null +++ include/indexstore/IndexStoreCXX.h @@ -0,0 +1,502 @@ +//===--- IndexStoreCXX.h - C++ wrapper for the Index Store C API. ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Header-only C++ wrapper for the Index Store C API. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INDEXSTORE_INDEXSTORECXX_H +#define LLVM_CLANG_INDEXSTORE_INDEXSTORECXX_H + +#include "indexstore/indexstore.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include + +namespace indexstore { + using llvm::ArrayRef; + using llvm::Optional; + using llvm::StringRef; + +static inline StringRef stringFromIndexStoreStringRef(indexstore_string_ref_t str) { + return StringRef(str.data, str.length); +} + +class IndexRecordSymbol { + indexstore_symbol_t obj; + friend class IndexRecordReader; + +public: + IndexRecordSymbol(indexstore_symbol_t obj) : obj(obj) {} + + indexstore_symbol_language_t getLanguage() { + return indexstore_symbol_get_language(obj); + } + indexstore_symbol_kind_t getKind() { return indexstore_symbol_get_kind(obj); } + indexstore_symbol_subkind_t getSubKind() { return indexstore_symbol_get_subkind(obj); } + uint64_t getProperties() { + return indexstore_symbol_get_properties(obj); + } + uint64_t getRoles() { return indexstore_symbol_get_roles(obj); } + uint64_t getRelatedRoles() { return indexstore_symbol_get_related_roles(obj); } + StringRef getName() { return stringFromIndexStoreStringRef(indexstore_symbol_get_name(obj)); } + StringRef getUSR() { return stringFromIndexStoreStringRef(indexstore_symbol_get_usr(obj)); } + StringRef getCodegenName() { return stringFromIndexStoreStringRef(indexstore_symbol_get_codegen_name(obj)); } +}; + +class IndexSymbolRelation { + indexstore_symbol_relation_t obj; + +public: + IndexSymbolRelation(indexstore_symbol_relation_t obj) : obj(obj) {} + + uint64_t getRoles() { return indexstore_symbol_relation_get_roles(obj); } + IndexRecordSymbol getSymbol() { return indexstore_symbol_relation_get_symbol(obj); } +}; + +class IndexRecordOccurrence { + indexstore_occurrence_t obj; + +public: + IndexRecordOccurrence(indexstore_occurrence_t obj) : obj(obj) {} + + IndexRecordSymbol getSymbol() { return indexstore_occurrence_get_symbol(obj); } + uint64_t getRoles() { return indexstore_occurrence_get_roles(obj); } + + bool foreachRelation(llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_occurrence_relations_apply(obj, ^bool(indexstore_symbol_relation_t sym_rel) { + return receiver(sym_rel); + }); +#else + return false; +#endif + } + + std::pair getLineCol() { + unsigned line, col; + indexstore_occurrence_get_line_col(obj, &line, &col); + return std::make_pair(line, col); + } +}; + +class IndexStore; +typedef std::shared_ptr IndexStoreRef; + +class IndexStore { + indexstore_t obj; + friend class IndexRecordReader; + friend class IndexUnitReader; + +public: + IndexStore(StringRef path, std::string &error) { + llvm::SmallString<64> buf = path; + indexstore_error_t c_err = nullptr; + obj = indexstore_store_create(buf.c_str(), &c_err); + if (c_err) { + error = indexstore_error_get_description(c_err); + indexstore_error_dispose(c_err); + } + } + + IndexStore(IndexStore &&other) : obj(other.obj) { + other.obj = nullptr; + } + + ~IndexStore() { + indexstore_store_dispose(obj); + } + + static IndexStoreRef create(StringRef path, std::string &error) { + auto storeRef = std::make_shared(path, error); + if (storeRef->isInvalid()) + return nullptr; + return storeRef; + } + + static unsigned formatVersion() { + return indexstore_format_version(); + } + + bool isValid() const { return obj; } + bool isInvalid() const { return !isValid(); } + explicit operator bool() const { return isValid(); } + + bool foreachUnit(bool sorted, llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_store_units_apply(obj, sorted, ^bool(indexstore_string_ref_t unit_name) { + return receiver(stringFromIndexStoreStringRef(unit_name)); + }); +#else + return false; +#endif + } + + class UnitEvent { + indexstore_unit_event_t obj; + public: + UnitEvent(indexstore_unit_event_t obj) : obj(obj) {} + + enum class Kind { + Added, + Removed, + Modified, + DirectoryDeleted, + }; + Kind getKind() const { + indexstore_unit_event_kind_t c_k = indexstore_unit_event_get_kind(obj); + Kind K; + switch (c_k) { + case INDEXSTORE_UNIT_EVENT_ADDED: K = Kind::Added; break; + case INDEXSTORE_UNIT_EVENT_REMOVED: K = Kind::Removed; break; + case INDEXSTORE_UNIT_EVENT_MODIFIED: K = Kind::Modified; break; + case INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED: K = Kind::DirectoryDeleted; break; + } + return K; + } + + StringRef getUnitName() const { + return stringFromIndexStoreStringRef(indexstore_unit_event_get_unit_name(obj)); + } + + timespec getModificationTime() const { return indexstore_unit_event_get_modification_time(obj); } + }; + + class UnitEventNotification { + indexstore_unit_event_notification_t obj; + public: + UnitEventNotification(indexstore_unit_event_notification_t obj) : obj(obj) {} + + bool isInitial() const { return indexstore_unit_event_notification_is_initial(obj); } + size_t getEventsCount() const { return indexstore_unit_event_notification_get_events_count(obj); } + UnitEvent getEvent(size_t index) const { return indexstore_unit_event_notification_get_event(obj, index); } + }; + + typedef std::function UnitEventHandler; + + void setUnitEventHandler(UnitEventHandler handler) { +#if INDEXSTORE_HAS_BLOCKS + if (!handler) { + indexstore_store_set_unit_event_handler(obj, nullptr); + return; + } + + indexstore_store_set_unit_event_handler(obj, ^(indexstore_unit_event_notification_t evt_note) { + handler(UnitEventNotification(evt_note)); + }); +#endif + } + + bool startEventListening(bool waitInitialSync, std::string &error) { + indexstore_unit_event_listen_options_t opts; + opts.wait_initial_sync = waitInitialSync; + indexstore_error_t c_err = nullptr; + bool ret = indexstore_store_start_unit_event_listening(obj, &opts, sizeof(opts), &c_err); + if (c_err) { + error = indexstore_error_get_description(c_err); + indexstore_error_dispose(c_err); + } + return ret; + } + + void stopEventListening() { + return indexstore_store_stop_unit_event_listening(obj); + } + + void discardUnit(StringRef UnitName) { + llvm::SmallString<64> buf = UnitName; + indexstore_store_discard_unit(obj, buf.c_str()); + } + + void discardRecord(StringRef RecordName) { + llvm::SmallString<64> buf = RecordName; + indexstore_store_discard_record(obj, buf.c_str()); + } + + void getUnitNameFromOutputPath(StringRef outputPath, llvm::SmallVectorImpl &nameBuf) { + llvm::SmallString<256> buf = outputPath; + size_t nameLen = indexstore_store_get_unit_name_from_output_path(obj, buf.c_str(), nameBuf.data(), nameBuf.size()); + if (nameLen+1 > nameBuf.size()) { + nameBuf.resize(nameLen+1); + indexstore_store_get_unit_name_from_output_path(obj, buf.c_str(), nameBuf.data(), nameBuf.size()); + } + } + + llvm::Optional + getUnitModificationTime(StringRef unitName, std::string &error) { + llvm::SmallString<64> buf = unitName; + int64_t seconds, nanoseconds; + indexstore_error_t c_err = nullptr; + bool err = indexstore_store_get_unit_modification_time(obj, buf.c_str(), + &seconds, &nanoseconds, &c_err); + if (err && c_err) { + error = indexstore_error_get_description(c_err); + indexstore_error_dispose(c_err); + return llvm::None; + } + timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = nanoseconds; + return ts; + } + + void purgeStaleData() { + indexstore_store_purge_stale_data(obj); + } +}; + +class IndexRecordReader { + indexstore_record_reader_t obj; + +public: + IndexRecordReader(IndexStore &store, StringRef recordName, std::string &error) { + llvm::SmallString<64> buf = recordName; + indexstore_error_t c_err = nullptr; + obj = indexstore_record_reader_create(store.obj, buf.c_str(), &c_err); + if (c_err) { + error = indexstore_error_get_description(c_err); + indexstore_error_dispose(c_err); + } + } + + IndexRecordReader(IndexRecordReader &&other) : obj(other.obj) { + other.obj = nullptr; + } + + ~IndexRecordReader() { + indexstore_record_reader_dispose(obj); + } + + bool isValid() const { return obj; } + bool isInvalid() const { return !isValid(); } + explicit operator bool() const { return isValid(); } + + /// Goes through and passes record decls, after filtering using a \c Checker + /// function. + /// + /// Resulting decls can be used as filter for \c foreachOccurrence. This + /// allows allocating memory only for the record decls that the caller is + /// interested in. + bool searchSymbols(llvm::function_ref filter, + llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_record_reader_search_symbols(obj, ^bool(indexstore_symbol_t symbol, bool *stop) { + return filter(symbol, *stop); + }, ^(indexstore_symbol_t symbol) { + receiver(symbol); + }); +#else + return false; +#endif + } + + bool foreachSymbol(bool noCache, llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_record_reader_symbols_apply(obj, noCache, ^bool(indexstore_symbol_t sym) { + return receiver(sym); + }); +#else + return false; +#endif + } + + /// \param DeclsFilter if non-empty indicates the list of decls that we want + /// to get occurrences for. An empty array indicates that we want occurrences + /// for all decls. + /// \param RelatedDeclsFilter Same as \c DeclsFilter but for related decls. + bool foreachOccurrence(ArrayRef symbolsFilter, + ArrayRef relatedSymbolsFilter, + llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + llvm::SmallVector c_symbolsFilter; + c_symbolsFilter.reserve(symbolsFilter.size()); + for (IndexRecordSymbol sym : symbolsFilter) { + c_symbolsFilter.push_back(sym.obj); + } + llvm::SmallVector c_relatedSymbolsFilter; + c_relatedSymbolsFilter.reserve(relatedSymbolsFilter.size()); + for (IndexRecordSymbol sym : relatedSymbolsFilter) { + c_relatedSymbolsFilter.push_back(sym.obj); + } + return indexstore_record_reader_occurrences_of_symbols_apply(obj, + c_symbolsFilter.data(), c_symbolsFilter.size(), + c_relatedSymbolsFilter.data(), + c_relatedSymbolsFilter.size(), + ^bool(indexstore_occurrence_t occur) { + return receiver(occur); + }); +#else + return false; +#endif + } + + bool foreachOccurrence( + llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_record_reader_occurrences_apply(obj, ^bool(indexstore_occurrence_t occur) { + return receiver(occur); + }); +#else + return false; +#endif + } + + bool foreachOccurrenceInLineRange(unsigned lineStart, unsigned lineEnd, + llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_record_reader_occurrences_in_line_range_apply(obj, + lineStart, + lineEnd, + ^bool(indexstore_occurrence_t occur) { + return receiver(occur); + }); +#else + return false; +#endif + } +}; + +class IndexUnitDependency { + indexstore_unit_dependency_t obj; + friend class IndexUnitReader; + +public: + IndexUnitDependency(indexstore_unit_dependency_t obj) : obj(obj) {} + + enum class DependencyKind { + Unit, + Record, + File, + }; + DependencyKind getKind() { + switch (indexstore_unit_dependency_get_kind(obj)) { + case INDEXSTORE_UNIT_DEPENDENCY_UNIT: return DependencyKind::Unit; + case INDEXSTORE_UNIT_DEPENDENCY_RECORD: return DependencyKind::Record; + case INDEXSTORE_UNIT_DEPENDENCY_FILE: return DependencyKind::File; + } + } + bool isSystem() { return indexstore_unit_dependency_is_system(obj); } + StringRef getName() { return stringFromIndexStoreStringRef(indexstore_unit_dependency_get_name(obj)); } + StringRef getFilePath() { return stringFromIndexStoreStringRef(indexstore_unit_dependency_get_filepath(obj)); } + StringRef getModuleName() { return stringFromIndexStoreStringRef(indexstore_unit_dependency_get_modulename(obj)); } + time_t getModificationTime() { return indexstore_unit_dependency_get_modification_time(obj); } + size_t getFileSize() { return indexstore_unit_dependency_get_file_size(obj); } + +}; + +class IndexUnitInclude { + indexstore_unit_include_t obj; + friend class IndexUnitReader; + +public: + IndexUnitInclude(indexstore_unit_include_t obj) : obj(obj) {} + + StringRef getSourcePath() { + return stringFromIndexStoreStringRef(indexstore_unit_include_get_source_path(obj)); + } + StringRef getTargetPath() { + return stringFromIndexStoreStringRef(indexstore_unit_include_get_target_path(obj)); + } + unsigned getSourceLine() { + return indexstore_unit_include_get_source_line(obj); + } +}; + +class IndexUnitReader { + indexstore_unit_reader_t obj; + +public: + IndexUnitReader(IndexStore &store, StringRef unitName, std::string &error) { + llvm::SmallString<64> buf = unitName; + indexstore_error_t c_err = nullptr; + obj = indexstore_unit_reader_create(store.obj, buf.c_str(), &c_err); + if (c_err) { + error = indexstore_error_get_description(c_err); + indexstore_error_dispose(c_err); + } + } + + IndexUnitReader(IndexUnitReader &&other) : obj(other.obj) { + other.obj = nullptr; + } + + ~IndexUnitReader() { + indexstore_unit_reader_dispose(obj); + } + + bool isValid() const { return obj; } + bool isInvalid() const { return !isValid(); } + explicit operator bool() const { return isValid(); } + + StringRef getProviderIdentifier() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_provider_identifier(obj)); + } + StringRef getProviderVersion() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_provider_version(obj)); + } + + timespec getModificationTime() { + int64_t seconds, nanoseconds; + indexstore_unit_reader_get_modification_time(obj, &seconds, &nanoseconds); + timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = nanoseconds; + return ts; + } + + bool isSystemUnit() { return indexstore_unit_reader_is_system_unit(obj); } + bool isModuleUnit() { return indexstore_unit_reader_is_module_unit(obj); } + bool isDebugCompilation() { return indexstore_unit_reader_is_debug_compilation(obj); } + bool hasMainFile() { return indexstore_unit_reader_has_main_file(obj); } + + StringRef getMainFilePath() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_main_file(obj)); + } + StringRef getModuleName() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_module_name(obj)); + } + StringRef getWorkingDirectory() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_working_dir(obj)); + } + StringRef getOutputFile() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_output_file(obj)); + } + StringRef getSysrootPath() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_sysroot_path(obj)); + } + StringRef getTarget() { + return stringFromIndexStoreStringRef(indexstore_unit_reader_get_target(obj)); + } + + bool foreachDependency(llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_unit_reader_dependencies_apply(obj, ^bool(indexstore_unit_dependency_t dep) { + return receiver(dep); + }); +#else + return false; +#endif + } + + bool foreachInclude(llvm::function_ref receiver) { +#if INDEXSTORE_HAS_BLOCKS + return indexstore_unit_reader_includes_apply(obj, ^bool(indexstore_unit_include_t inc) { + return receiver(inc); + }); +#else + return false; +#endif + } +}; + +} // namespace indexstore + +#endif Index: include/indexstore/indexstore.h =================================================================== --- /dev/null +++ include/indexstore/indexstore.h @@ -0,0 +1,475 @@ +/*===-- indexstore/indexstore.h - Index Store C API ----------------- C -*-===*\ +|* *| +|* The LLVM Compiler Infrastructure *| +|* *| +|* This file is distributed under the University of Illinois Open Source *| +|* License. See LICENSE.TXT for details. *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a C API for the index store. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_CLANG_C_INDEXSTORE_INDEXSTORE_H +#define LLVM_CLANG_C_INDEXSTORE_INDEXSTORE_H + +#include +#include +#include + +/** + * \brief The version constants for the Index Store C API. + * INDEXSTORE_VERSION_MINOR should increase when there are API additions. + * INDEXSTORE_VERSION_MAJOR is intended for "major" source/ABI breaking changes. + */ +#define INDEXSTORE_VERSION_MAJOR 0 +#define INDEXSTORE_VERSION_MINOR 9 + +#define INDEXSTORE_VERSION_ENCODE(major, minor) ( \ + ((major) * 10000) \ + + ((minor) * 1)) + +#define INDEXSTORE_VERSION INDEXSTORE_VERSION_ENCODE( \ + INDEXSTORE_VERSION_MAJOR, \ + INDEXSTORE_VERSION_MINOR ) + +#define INDEXSTORE_VERSION_STRINGIZE_(major, minor) \ + #major"."#minor +#define INDEXSTORE_VERSION_STRINGIZE(major, minor) \ + INDEXSTORE_VERSION_STRINGIZE_(major, minor) + +#define INDEXSTORE_VERSION_STRING INDEXSTORE_VERSION_STRINGIZE( \ + INDEXSTORE_VERSION_MAJOR, \ + INDEXSTORE_VERSION_MINOR) + +#ifdef __cplusplus +# define INDEXSTORE_BEGIN_DECLS extern "C" { +# define INDEXSTORE_END_DECLS } +#else +# define INDEXSTORE_BEGIN_DECLS +# define INDEXSTORE_END_DECLS +#endif + +#ifndef INDEXSTORE_PUBLIC +# if defined (_MSC_VER) +# define INDEXSTORE_PUBLIC __declspec(dllimport) +# else +# define INDEXSTORE_PUBLIC +# endif +#endif + +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +#if __has_feature(blocks) +# define INDEXSTORE_HAS_BLOCKS 1 +#else +# define INDEXSTORE_HAS_BLOCKS 0 +#endif + +INDEXSTORE_BEGIN_DECLS + +typedef void *indexstore_error_t; + +INDEXSTORE_PUBLIC const char * +indexstore_error_get_description(indexstore_error_t); + +INDEXSTORE_PUBLIC void +indexstore_error_dispose(indexstore_error_t); + +typedef struct { + const char *data; + size_t length; +} indexstore_string_ref_t; + +INDEXSTORE_PUBLIC unsigned +indexstore_format_version(void); + +typedef void *indexstore_t; + +INDEXSTORE_PUBLIC indexstore_t +indexstore_store_create(const char *store_path, indexstore_error_t *error); + +INDEXSTORE_PUBLIC void +indexstore_store_dispose(indexstore_t); + +#if INDEXSTORE_HAS_BLOCKS +INDEXSTORE_PUBLIC bool +indexstore_store_units_apply(indexstore_t, unsigned sorted, + bool(^applier)(indexstore_string_ref_t unit_name)); +#endif + +typedef void *indexstore_unit_event_notification_t; +typedef void *indexstore_unit_event_t; + +INDEXSTORE_PUBLIC size_t +indexstore_unit_event_notification_get_events_count(indexstore_unit_event_notification_t); + +INDEXSTORE_PUBLIC indexstore_unit_event_t +indexstore_unit_event_notification_get_event(indexstore_unit_event_notification_t, size_t index); + +INDEXSTORE_PUBLIC bool +indexstore_unit_event_notification_is_initial(indexstore_unit_event_notification_t); + +typedef enum { + INDEXSTORE_UNIT_EVENT_ADDED = 1, + INDEXSTORE_UNIT_EVENT_REMOVED = 2, + INDEXSTORE_UNIT_EVENT_MODIFIED = 3, + INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED = 4, +} indexstore_unit_event_kind_t; + +INDEXSTORE_PUBLIC indexstore_unit_event_kind_t +indexstore_unit_event_get_kind(indexstore_unit_event_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_event_get_unit_name(indexstore_unit_event_t); + +INDEXSTORE_PUBLIC timespec +indexstore_unit_event_get_modification_time(indexstore_unit_event_t); + +#if INDEXSTORE_HAS_BLOCKS +typedef void (^indexstore_unit_event_handler_t)(indexstore_unit_event_notification_t); + +INDEXSTORE_PUBLIC void +indexstore_store_set_unit_event_handler(indexstore_t, + indexstore_unit_event_handler_t handler); +#endif + +typedef struct { + /// If true, \c indexstore_store_start_unit_event_listening will block until + /// the initial set of units is passed to the unit event handler, otherwise + /// the function will return and the initial set will be passed asynchronously. + bool wait_initial_sync; +} indexstore_unit_event_listen_options_t; + +INDEXSTORE_PUBLIC bool +indexstore_store_start_unit_event_listening(indexstore_t, + indexstore_unit_event_listen_options_t *, + size_t listen_options_struct_size, + indexstore_error_t *error); + +INDEXSTORE_PUBLIC void +indexstore_store_stop_unit_event_listening(indexstore_t); + +INDEXSTORE_PUBLIC void +indexstore_store_discard_unit(indexstore_t, const char *unit_name); + +INDEXSTORE_PUBLIC void +indexstore_store_discard_record(indexstore_t, const char *record_name); + +INDEXSTORE_PUBLIC void +indexstore_store_purge_stale_data(indexstore_t); + +/// Determines the unit name from the \c output_path and writes it out in the +/// \c name_buf buffer. It doesn't write more than \c buf_size. +/// \returns the length of the name. If this is larger than \c buf_size, the +/// caller should call the function again with a buffer of the appropriate size. +INDEXSTORE_PUBLIC size_t +indexstore_store_get_unit_name_from_output_path(indexstore_t store, + const char *output_path, + char *name_buf, + size_t buf_size); + +/// \returns true if an error occurred, false otherwise. +INDEXSTORE_PUBLIC bool +indexstore_store_get_unit_modification_time(indexstore_t store, + const char *unit_name, + int64_t *seconds, + int64_t *nanoseconds, + indexstore_error_t *error); + +typedef void *indexstore_symbol_t; + +typedef enum { + INDEXSTORE_SYMBOL_KIND_UNKNOWN = 0, + INDEXSTORE_SYMBOL_KIND_MODULE = 1, + INDEXSTORE_SYMBOL_KIND_NAMESPACE = 2, + INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS = 3, + INDEXSTORE_SYMBOL_KIND_MACRO = 4, + INDEXSTORE_SYMBOL_KIND_ENUM = 5, + INDEXSTORE_SYMBOL_KIND_STRUCT = 6, + INDEXSTORE_SYMBOL_KIND_CLASS = 7, + INDEXSTORE_SYMBOL_KIND_PROTOCOL = 8, + INDEXSTORE_SYMBOL_KIND_EXTENSION = 9, + INDEXSTORE_SYMBOL_KIND_UNION = 10, + INDEXSTORE_SYMBOL_KIND_TYPEALIAS = 11, + INDEXSTORE_SYMBOL_KIND_FUNCTION = 12, + INDEXSTORE_SYMBOL_KIND_VARIABLE = 13, + INDEXSTORE_SYMBOL_KIND_FIELD = 14, + INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT = 15, + INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD = 16, + INDEXSTORE_SYMBOL_KIND_CLASSMETHOD = 17, + INDEXSTORE_SYMBOL_KIND_STATICMETHOD = 18, + INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY = 19, + INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY = 20, + INDEXSTORE_SYMBOL_KIND_STATICPROPERTY = 21, + INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR = 22, + INDEXSTORE_SYMBOL_KIND_DESTRUCTOR = 23, + INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION = 24, + INDEXSTORE_SYMBOL_KIND_PARAMETER = 25, + INDEXSTORE_SYMBOL_KIND_USING = 26, + + INDEXSTORE_SYMBOL_KIND_COMMENTTAG = 1000, +} indexstore_symbol_kind_t; + +typedef enum { + INDEXSTORE_SYMBOL_SUBKIND_NONE = 0, + INDEXSTORE_SYMBOL_SUBKIND_CXXCOPYCONSTRUCTOR = 1, + INDEXSTORE_SYMBOL_SUBKIND_CXXMOVECONSTRUCTOR = 2, + INDEXSTORE_SYMBOL_SUBKIND_ACCESSORGETTER = 3, + INDEXSTORE_SYMBOL_SUBKIND_ACCESSORSETTER = 4, + INDEXSTORE_SYMBOL_SUBKIND_USINGTYPENAME = 5, + INDEXSTORE_SYMBOL_SUBKIND_USINGVALUE = 6, +} indexstore_symbol_subkind_t; + +typedef enum { + INDEXSTORE_SYMBOL_PROPERTY_GENERIC = 1 << 0, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_PARTIAL_SPECIALIZATION = 1 << 1, + INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_SPECIALIZATION = 1 << 2, + INDEXSTORE_SYMBOL_PROPERTY_UNITTEST = 1 << 3, + INDEXSTORE_SYMBOL_PROPERTY_IBANNOTATED = 1 << 4, + INDEXSTORE_SYMBOL_PROPERTY_IBOUTLETCOLLECTION = 1 << 5, + INDEXSTORE_SYMBOL_PROPERTY_GKINSPECTABLE = 1 << 6, + INDEXSTORE_SYMBOL_PROPERTY_LOCAL = 1 << 7, +} indexstore_symbol_property_t; + +typedef enum { + INDEXSTORE_SYMBOL_LANG_C = 0, + INDEXSTORE_SYMBOL_LANG_OBJC = 1, + INDEXSTORE_SYMBOL_LANG_CXX = 2, + + INDEXSTORE_SYMBOL_LANG_SWIFT = 100, +} indexstore_symbol_language_t; + +typedef enum { + INDEXSTORE_SYMBOL_ROLE_DECLARATION = 1 << 0, + INDEXSTORE_SYMBOL_ROLE_DEFINITION = 1 << 1, + INDEXSTORE_SYMBOL_ROLE_REFERENCE = 1 << 2, + INDEXSTORE_SYMBOL_ROLE_READ = 1 << 3, + INDEXSTORE_SYMBOL_ROLE_WRITE = 1 << 4, + INDEXSTORE_SYMBOL_ROLE_CALL = 1 << 5, + INDEXSTORE_SYMBOL_ROLE_DYNAMIC = 1 << 6, + INDEXSTORE_SYMBOL_ROLE_ADDRESSOF = 1 << 7, + INDEXSTORE_SYMBOL_ROLE_IMPLICIT = 1 << 8, + + // Relation roles. + INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF = 1 << 9, + INDEXSTORE_SYMBOL_ROLE_REL_BASEOF = 1 << 10, + INDEXSTORE_SYMBOL_ROLE_REL_OVERRIDEOF = 1 << 11, + INDEXSTORE_SYMBOL_ROLE_REL_RECEIVEDBY = 1 << 12, + INDEXSTORE_SYMBOL_ROLE_REL_CALLEDBY = 1 << 13, + INDEXSTORE_SYMBOL_ROLE_REL_EXTENDEDBY = 1 << 14, + INDEXSTORE_SYMBOL_ROLE_REL_ACCESSOROF = 1 << 15, + INDEXSTORE_SYMBOL_ROLE_REL_CONTAINEDBY = 1 << 16, + INDEXSTORE_SYMBOL_ROLE_REL_IBTYPEOF = 1 << 17, + INDEXSTORE_SYMBOL_ROLE_REL_SPECIALIZATIONOF = 1 << 18, +} indexstore_symbol_role_t; + +INDEXSTORE_PUBLIC indexstore_symbol_language_t +indexstore_symbol_get_language(indexstore_symbol_t); + +INDEXSTORE_PUBLIC indexstore_symbol_kind_t +indexstore_symbol_get_kind(indexstore_symbol_t); + +INDEXSTORE_PUBLIC indexstore_symbol_subkind_t +indexstore_symbol_get_subkind(indexstore_symbol_t); + +INDEXSTORE_PUBLIC uint64_t +indexstore_symbol_get_properties(indexstore_symbol_t); + +INDEXSTORE_PUBLIC uint64_t +indexstore_symbol_get_roles(indexstore_symbol_t); + +INDEXSTORE_PUBLIC uint64_t +indexstore_symbol_get_related_roles(indexstore_symbol_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_symbol_get_name(indexstore_symbol_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_symbol_get_usr(indexstore_symbol_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_symbol_get_codegen_name(indexstore_symbol_t); + +typedef void *indexstore_symbol_relation_t; + +INDEXSTORE_PUBLIC uint64_t +indexstore_symbol_relation_get_roles(indexstore_symbol_relation_t); + +INDEXSTORE_PUBLIC indexstore_symbol_t +indexstore_symbol_relation_get_symbol(indexstore_symbol_relation_t); + +typedef void *indexstore_occurrence_t; + +INDEXSTORE_PUBLIC indexstore_symbol_t +indexstore_occurrence_get_symbol(indexstore_occurrence_t); + +#if INDEXSTORE_HAS_BLOCKS +INDEXSTORE_PUBLIC bool +indexstore_occurrence_relations_apply(indexstore_occurrence_t, + bool(^applier)(indexstore_symbol_relation_t symbol_rel)); +#endif + +INDEXSTORE_PUBLIC uint64_t +indexstore_occurrence_get_roles(indexstore_occurrence_t); + +INDEXSTORE_PUBLIC void +indexstore_occurrence_get_line_col(indexstore_occurrence_t, + unsigned *line, unsigned *column); + +typedef void *indexstore_record_reader_t; + +INDEXSTORE_PUBLIC indexstore_record_reader_t +indexstore_record_reader_create(indexstore_t store, const char *record_name, + indexstore_error_t *error); + +INDEXSTORE_PUBLIC void +indexstore_record_reader_dispose(indexstore_record_reader_t); + +#if INDEXSTORE_HAS_BLOCKS +/// Goes through the symbol data and passes symbols to \c receiver, for the +/// symbol data that \c filter returns true on. +/// +/// This allows allocating memory only for the record symbols that the caller is +/// interested in. +INDEXSTORE_PUBLIC bool +indexstore_record_reader_search_symbols(indexstore_record_reader_t, + bool(^filter)(indexstore_symbol_t symbol, bool *stop), + void(^receiver)(indexstore_symbol_t symbol)); + +/// \param nocache if true, avoids allocating memory for the symbols. +/// Useful when the caller does not intend to keep \c indexstore_record_reader_t +/// for more queries. +INDEXSTORE_PUBLIC bool +indexstore_record_reader_symbols_apply(indexstore_record_reader_t, + bool nocache, + bool(^applier)(indexstore_symbol_t symbol)); + +INDEXSTORE_PUBLIC bool +indexstore_record_reader_occurrences_apply(indexstore_record_reader_t, + bool(^applier)(indexstore_occurrence_t occur)); + +INDEXSTORE_PUBLIC bool +indexstore_record_reader_occurrences_in_line_range_apply(indexstore_record_reader_t, + unsigned line_start, + unsigned line_count, + bool(^applier)(indexstore_occurrence_t occur)); + +/// \param symbols if non-zero \c symbols_count, indicates the list of symbols +/// that we want to get occurrences for. An empty array indicates that we want +/// occurrences for all symbols. +/// \param related_symbols Same as \c symbols but for related symbols. +INDEXSTORE_PUBLIC bool +indexstore_record_reader_occurrences_of_symbols_apply(indexstore_record_reader_t, + indexstore_symbol_t *symbols, size_t symbols_count, + indexstore_symbol_t *related_symbols, size_t related_symbols_count, + bool(^applier)(indexstore_occurrence_t occur)); +#endif + + +typedef void *indexstore_unit_reader_t; + +INDEXSTORE_PUBLIC indexstore_unit_reader_t +indexstore_unit_reader_create(indexstore_t store, const char *unit_name, + indexstore_error_t *error); + +INDEXSTORE_PUBLIC void +indexstore_unit_reader_dispose(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_provider_identifier(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_provider_version(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC void +indexstore_unit_reader_get_modification_time(indexstore_unit_reader_t, + int64_t *seconds, + int64_t *nanoseconds); + +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_is_system_unit(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_is_module_unit(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_is_debug_compilation(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_has_main_file(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_main_file(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_module_name(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_working_dir(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_output_file(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_sysroot_path(indexstore_unit_reader_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_reader_get_target(indexstore_unit_reader_t); + +typedef void *indexstore_unit_dependency_t; +typedef void *indexstore_unit_include_t; + +typedef enum { + INDEXSTORE_UNIT_DEPENDENCY_UNIT = 1, + INDEXSTORE_UNIT_DEPENDENCY_RECORD = 2, + INDEXSTORE_UNIT_DEPENDENCY_FILE = 3, +} indexstore_unit_dependency_kind_t; + +INDEXSTORE_PUBLIC indexstore_unit_dependency_kind_t +indexstore_unit_dependency_get_kind(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC bool +indexstore_unit_dependency_is_system(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_dependency_get_filepath(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_dependency_get_modulename(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_dependency_get_name(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC time_t +indexstore_unit_dependency_get_modification_time(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC size_t +indexstore_unit_dependency_get_file_size(indexstore_unit_dependency_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_include_get_source_path(indexstore_unit_include_t); + +INDEXSTORE_PUBLIC indexstore_string_ref_t +indexstore_unit_include_get_target_path(indexstore_unit_include_t); + +INDEXSTORE_PUBLIC unsigned +indexstore_unit_include_get_source_line(indexstore_unit_include_t); + +#if INDEXSTORE_HAS_BLOCKS +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_dependencies_apply(indexstore_unit_reader_t, + bool(^applier)(indexstore_unit_dependency_t)); + +INDEXSTORE_PUBLIC bool +indexstore_unit_reader_includes_apply(indexstore_unit_reader_t, + bool(^applier)(indexstore_unit_include_t)); + +#endif + +INDEXSTORE_END_DECLS + +#endif Index: lib/CMakeLists.txt =================================================================== --- lib/CMakeLists.txt +++ lib/CMakeLists.txt @@ -19,6 +19,7 @@ add_subdirectory(FrontendTool) add_subdirectory(Tooling) add_subdirectory(Index) +add_subdirectory(DirectoryWatcher) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(StaticAnalyzer) endif() Index: lib/DirectoryWatcher/CMakeLists.txt =================================================================== --- /dev/null +++ lib/DirectoryWatcher/CMakeLists.txt @@ -0,0 +1,8 @@ +set(LLVM_LINK_COMPONENTS support) + +add_clang_library(clangDirectoryWatcher + DirectoryWatcher.cpp + + LINK_LIBS + clangBasic + ) Index: lib/DirectoryWatcher/DirectoryWatcher.cpp =================================================================== --- /dev/null +++ lib/DirectoryWatcher/DirectoryWatcher.cpp @@ -0,0 +1,275 @@ +//===- DirectoryWatcher.cpp - Listens for directory file changes ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// \brief Utility class for listening for file system changes in a directory. +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +#define HAVE_CORESERVICES 0 + +#if defined(__has_include) +#if __has_include() + +#include +#undef HAVE_CORESERVICES +#define HAVE_CORESERVICES 1 + +#endif +#endif + +using namespace clang; +using namespace llvm; + +static timespec toTimeSpec(sys::TimePoint<> tp) { + std::chrono::seconds sec = std::chrono::time_point_cast( + tp).time_since_epoch(); + std::chrono::nanoseconds nsec = + std::chrono::time_point_cast(tp - sec) + .time_since_epoch(); + timespec ts; + ts.tv_sec = sec.count(); + ts.tv_nsec = nsec.count(); + return ts; +} + +static Optional getModTime(StringRef path) { + sys::fs::file_status Status; + std::error_code EC = status(path, Status); + if (EC) + return None; + return toTimeSpec(Status.getLastModificationTime()); +} + +struct DirectoryWatcher::Implementation { +#if HAVE_CORESERVICES + FSEventStreamRef EventStream = nullptr; + + bool setupFSEventStream(StringRef path, EventReceiver receiver, + dispatch_queue_t queue); + void stopFSEventStream(); + + ~Implementation() { + stopFSEventStream(); + }; +#endif +}; + +#if HAVE_CORESERVICES +namespace { +struct EventStreamContextData { + std::string WatchedPath; + DirectoryWatcher::EventReceiver Receiver; + + EventStreamContextData(std::string watchedPath, DirectoryWatcher::EventReceiver receiver) + : WatchedPath(std::move(watchedPath)), Receiver(std::move(receiver)) { + } + + static void dispose(const void *ctx) { + delete static_cast(ctx); + } +}; +} + +static void eventStreamCallback( + ConstFSEventStreamRef stream, + void *clientCallBackInfo, + size_t numEvents, + void *eventPaths, + const FSEventStreamEventFlags eventFlags[], + const FSEventStreamEventId eventIds[]) { + auto *ctx = static_cast(clientCallBackInfo); + + std::vector Events; + for (size_t i = 0; i < numEvents; ++i) { + StringRef path = ((const char **)eventPaths)[i]; + const FSEventStreamEventFlags flags = eventFlags[i]; + if (!(flags & kFSEventStreamEventFlagItemIsFile)) { + if ((flags & kFSEventStreamEventFlagItemRemoved) && path == ctx->WatchedPath) { + DirectoryWatcher::Event Evt{DirectoryWatcher::EventKind::DirectoryDeleted, path, timespec{}}; + Events.push_back(Evt); + break; + } + continue; + } + DirectoryWatcher::EventKind K = DirectoryWatcher::EventKind::Modified; + if ((flags & kFSEventStreamEventFlagItemCreated) || + (flags & kFSEventStreamEventFlagItemRenamed)) + K = DirectoryWatcher::EventKind::Added; + if (flags & kFSEventStreamEventFlagItemRemoved) + K = DirectoryWatcher::EventKind::Removed; + timespec modTime{}; + if (K != DirectoryWatcher::EventKind::Removed) { + auto modTimeOpt = getModTime(path); + if (!modTimeOpt.hasValue()) + continue; + modTime = modTimeOpt.getValue(); + } + DirectoryWatcher::Event Evt{K, path, modTime}; + Events.push_back(Evt); + } + + ctx->Receiver(Events, /*isInitial=*/false); +} + +bool DirectoryWatcher::Implementation::setupFSEventStream(StringRef path, + EventReceiver receiver, + dispatch_queue_t queue) { + if (path.empty()) + return true; + + CFMutableArrayRef pathsToWatch = CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); + CFStringRef cfPathStr = CFStringCreateWithBytes(nullptr, (const UInt8 *)path.data(), path.size(), kCFStringEncodingUTF8, false); + CFArrayAppendValue(pathsToWatch, cfPathStr); + CFRelease(cfPathStr); + CFAbsoluteTime latency = 0.2; // Latency in seconds. + + std::string realPath; + { + SmallString<128> Storage; + StringRef P = llvm::Twine(path).toNullTerminatedStringRef(Storage); + char Buffer[PATH_MAX]; + // Use ::realpath to get the real path name + if (::realpath(P.begin(), Buffer) != nullptr) + realPath = Buffer; + else + realPath = path; + } + + EventStreamContextData *ctxData = new EventStreamContextData(std::move(realPath), std::move(receiver)); + FSEventStreamContext context; + context.version = 0; + context.info = ctxData; + context.retain = nullptr; + context.release = EventStreamContextData::dispose; + context.copyDescription = nullptr; + + EventStream = FSEventStreamCreate(nullptr, + eventStreamCallback, + &context, + pathsToWatch, + kFSEventStreamEventIdSinceNow, + latency, + kFSEventStreamCreateFlagFileEvents | + kFSEventStreamCreateFlagNoDefer); + CFRelease(pathsToWatch); + if (!EventStream) { + return true; + } + FSEventStreamSetDispatchQueue(EventStream, queue); + FSEventStreamStart(EventStream); + return false; +} + +void DirectoryWatcher::Implementation::stopFSEventStream() { + if (!EventStream) + return; + FSEventStreamStop(EventStream); + FSEventStreamInvalidate(EventStream); + FSEventStreamRelease(EventStream); + EventStream = nullptr; +} +#endif + +DirectoryWatcher::DirectoryWatcher() + : Impl(*new Implementation()) {} + +DirectoryWatcher::~DirectoryWatcher() { + delete &Impl; +} + +#if HAVE_CORESERVICES +static std::vector scanDirectory(StringRef Path) { + using namespace llvm::sys; + + std::vector Events; + std::error_code EC; + for (auto It = fs::directory_iterator(Path, EC), End = fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + auto modTime = getModTime(It->path()); + if (!modTime.hasValue()) + continue; + DirectoryWatcher::Event Event{DirectoryWatcher::EventKind::Added, It->path(), modTime.getValue()}; + Events.push_back(std::move(Event)); + } + return Events; +} +#endif + +std::unique_ptr DirectoryWatcher::create(StringRef Path, + EventReceiver Receiver, bool waitInitialSync, std::string &Error) { +#if HAVE_CORESERVICES + + using namespace llvm::sys; + + if (!fs::exists(Path)) { + std::error_code EC = fs::create_directories(Path); + if (EC) { + Error = EC.message(); + return nullptr; + } + } + + bool IsDir; + std::error_code EC = fs::is_directory(Path, IsDir); + if (EC) { + Error = EC.message(); + return nullptr; + } + if (!IsDir) { + Error = "path is not a directory: "; + Error += Path; + return nullptr; + } + + std::unique_ptr DirWatch; + DirWatch.reset(new DirectoryWatcher()); + auto &Impl = DirWatch->Impl; + + dispatch_queue_t queue = dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); + dispatch_semaphore_t initScanSema = dispatch_semaphore_create(0); + dispatch_semaphore_t setupFSEventsSema = dispatch_semaphore_create(0); + + std::string copiedPath = Path; + dispatch_retain(initScanSema); + dispatch_retain(setupFSEventsSema); + dispatch_async(queue, ^{ + // Wait for the event stream to be setup before doing the initial scan, + // to make sure we won't miss any events. + dispatch_semaphore_wait(setupFSEventsSema, DISPATCH_TIME_FOREVER); + auto events = scanDirectory(copiedPath); + Receiver(events, /*isInitial=*/true); + dispatch_semaphore_signal(initScanSema); + dispatch_release(setupFSEventsSema); + dispatch_release(initScanSema); + }); + bool fsErr = Impl.setupFSEventStream(Path, Receiver, queue); + dispatch_semaphore_signal(setupFSEventsSema); + + if (waitInitialSync) { + dispatch_semaphore_wait(initScanSema, DISPATCH_TIME_FOREVER); + } + dispatch_release(setupFSEventsSema); + dispatch_release(initScanSema); + dispatch_release(queue); + + if (fsErr) { + raw_string_ostream(Error) << "failed to setup FSEvents stream for path: " << Path; + return nullptr; + } + + return DirWatch; +#else + return nullptr; +#endif +} Index: lib/Driver/Driver.cpp =================================================================== --- lib/Driver/Driver.cpp +++ lib/Driver/Driver.cpp @@ -993,7 +993,9 @@ } // Assume associated files are based off of the first temporary file. - CrashReportInfo CrashInfo(TempFiles[0], VFS); + CrashReportInfo CrashInfo( + TempFiles[0], VFS, + C.getArgs().getLastArgValue(options::OPT_index_store_path)); std::string Script = CrashInfo.Filename.rsplit('.').first.str() + ".sh"; std::error_code EC; Index: lib/Driver/Job.cpp =================================================================== --- lib/Driver/Job.cpp +++ lib/Driver/Job.cpp @@ -68,6 +68,8 @@ .Default(false); if (IsInclude) return HaveCrashVFS ? false : true; + if (StringRef(Flag).startswith("-index-store-path")) + return true; // The remaining flags are treated as a single argument. @@ -220,6 +222,7 @@ } bool HaveCrashVFS = CrashInfo && !CrashInfo->VFSPath.empty(); + bool HaveIndexStorePath = CrashInfo && !CrashInfo->IndexStorePath.empty(); for (size_t i = 0, e = Args.size(); i < e; ++i) { const char *const Arg = Args[i]; @@ -283,6 +286,24 @@ printArg(OS, ModCachePath, Quote); } + if (CrashInfo && HaveIndexStorePath) { + SmallString<128> IndexStoreDir; + + if (HaveCrashVFS) { + IndexStoreDir = llvm::sys::path::parent_path( + llvm::sys::path::parent_path(CrashInfo->VFSPath)); + llvm::sys::path::append(IndexStoreDir, "index-store"); + } else { + IndexStoreDir = "index-store"; + } + + OS << ' '; + printArg(OS, "-index-store-path", Quote); + OS << ' '; + printArg(OS, IndexStoreDir.c_str(), Quote); + } + + if (ResponseFile != nullptr) { OS << "\n Arguments passed via response file:\n"; writeResponseFile(OS); Index: lib/Driver/ToolChains/Clang.cpp =================================================================== --- lib/Driver/ToolChains/Clang.cpp +++ lib/Driver/ToolChains/Clang.cpp @@ -3556,6 +3556,26 @@ RenderARCMigrateToolOptions(D, Args, CmdArgs); + if (Args.hasArg(options::OPT_index_store_path)) { + Args.AddLastArg(CmdArgs, options::OPT_index_store_path); + Args.AddLastArg(CmdArgs, options::OPT_index_ignore_system_symbols); + Args.AddLastArg(CmdArgs, options::OPT_index_record_codegen_name); + + // If '-o' is passed along with '-fsyntax-only' pass it along the cc1 + // invocation so that the index action knows what the out file is. + if (isa(JA) && JA.getType() == types::TY_Nothing) { + Args.AddLastArg(CmdArgs, options::OPT_o); + } + } + + if (const char *IdxStorePath = ::getenv("CLANG_PROJECT_INDEX_PATH")) { + CmdArgs.push_back("-index-store-path"); + CmdArgs.push_back(IdxStorePath); + CmdArgs.push_back("-index-ignore-system-symbols"); + CmdArgs.push_back("-index-record-codegen-name"); + } + + // Add preprocessing options like -I, -D, etc. if we are using the // preprocessor. // Index: lib/Driver/ToolChains/Darwin.cpp =================================================================== --- lib/Driver/ToolChains/Darwin.cpp +++ lib/Driver/ToolChains/Darwin.cpp @@ -436,6 +436,10 @@ // more information. ArgStringList CmdArgs; + Args.ClaimAllArgs(options::OPT_index_store_path); + Args.ClaimAllArgs(options::OPT_index_ignore_system_symbols); + Args.ClaimAllArgs(options::OPT_index_record_codegen_name); + /// Hack(tm) to ignore linking errors when we are doing ARC migration. if (Args.hasArg(options::OPT_ccc_arcmt_check, options::OPT_ccc_arcmt_migrate)) { Index: lib/Frontend/CompilerInstance.cpp =================================================================== --- lib/Frontend/CompilerInstance.cpp +++ lib/Frontend/CompilerInstance.cpp @@ -28,6 +28,7 @@ #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" #include "clang/Frontend/VerifyDiagnosticConsumer.h" +#include "clang/Index/IndexingAction.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/PTHManager.h" #include "clang/Lex/Preprocessor.h" @@ -1147,6 +1148,10 @@ SourceMgr.pushModuleBuildStack(ModuleName, FullSourceLoc(ImportLoc, ImportingInstance.getSourceManager())); + // Pass along the GenModuleActionWrapper callback + auto wrapGenModuleAction = ImportingInstance.getGenModuleActionWrapper(); + Instance.setGenModuleActionWrapper(wrapGenModuleAction); + // If we're collecting module dependencies, we need to share a collector // between all of the module CompilerInstances. Other than that, we don't // want to produce any dependency output from the module build. @@ -1165,8 +1170,12 @@ llvm::CrashRecoveryContext CRC; CRC.RunSafelyOnThread( [&]() { - GenerateModuleFromModuleMapAction Action; - Instance.ExecuteAction(Action); + std::unique_ptr Action( + new GenerateModuleFromModuleMapAction); + if (wrapGenModuleAction) { + Action = wrapGenModuleAction(FrontendOpts, std::move(Action)); + } + Instance.ExecuteAction(*Action); }, ThreadStackSize); Index: lib/Frontend/CompilerInvocation.cpp =================================================================== --- lib/Frontend/CompilerInvocation.cpp +++ lib/Frontend/CompilerInvocation.cpp @@ -1434,6 +1434,10 @@ << "ARC migration" << "ObjC migration"; } + Opts.IndexStorePath = Args.getLastArgValue(OPT_index_store_path); + Opts.IndexIgnoreSystemSymbols = Args.hasArg(OPT_index_ignore_system_symbols); + Opts.IndexRecordCodegenName = Args.hasArg(OPT_index_record_codegen_name); + InputKind DashX(InputKind::Unknown); if (const Arg *A = Args.getLastArg(OPT_x)) { StringRef XValue = A->getValue(); Index: lib/FrontendTool/CMakeLists.txt =================================================================== --- lib/FrontendTool/CMakeLists.txt +++ lib/FrontendTool/CMakeLists.txt @@ -8,6 +8,7 @@ clangCodeGen clangDriver clangFrontend + clangIndex clangRewriteFrontend ) Index: lib/FrontendTool/ExecuteCompilerInvocation.cpp =================================================================== --- lib/FrontendTool/ExecuteCompilerInvocation.cpp +++ lib/FrontendTool/ExecuteCompilerInvocation.cpp @@ -23,6 +23,7 @@ #include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/FrontendPluginRegistry.h" #include "clang/Frontend/Utils.h" +#include "clang/Index/IndexingAction.h" #include "clang/Rewrite/Frontend/FrontendActions.h" #include "clang/StaticAnalyzer/Frontend/FrontendActions.h" #include "llvm/Option/OptTable.h" @@ -164,6 +165,11 @@ } #endif + if (!FEOpts.IndexStorePath.empty()) { + Act = index::createIndexDataRecordingAction(FEOpts, std::move(Act)); + CI.setGenModuleActionWrapper(&index::createIndexDataRecordingAction); + } + // If there are any AST files to merge, create a frontend action // adaptor to perform the merge. if (!FEOpts.ASTMergeFiles.empty()) Index: lib/Index/BitstreamVisitor.h =================================================================== --- /dev/null +++ lib/Index/BitstreamVisitor.h @@ -0,0 +1,163 @@ +//===--- BitstreamVisitor.h - Helper for reading a bitstream --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_BITSTREAMVISITOR_H +#define LLVM_CLANG_LIB_INDEX_BITSTREAMVISITOR_H + +#include "llvm/Bitcode/BitstreamReader.h" +#include "clang/Basic/LLVM.h" +#include "clang/Serialization/ASTReader.h" +#include + +namespace clang { +namespace index { +namespace store { + +/// Helper class that saves the current stream position and +/// then restores it when destroyed. +struct SavedStreamPosition { + explicit SavedStreamPosition(llvm::BitstreamCursor &Cursor) + : Cursor(Cursor), Offset(Cursor.GetCurrentBitNo()) { } + + ~SavedStreamPosition() { + Cursor.JumpToBit(Offset); + } + +private: + llvm::BitstreamCursor &Cursor; + uint64_t Offset; +}; + +enum class StreamVisit { + Continue, + Skip, + Abort +}; + +template +class BitstreamVisitor { + SmallVector BlockStack; + +protected: + llvm::BitstreamCursor &Stream; + Optional BlockInfo; + std::string *Error; + +public: + BitstreamVisitor(llvm::BitstreamCursor &Stream) + : Stream(Stream) {} + + StreamVisit visitBlock(unsigned ID) { + return StreamVisit::Continue; + } + + bool visit(std::string &Error) { + this->Error = &Error; + + ASTReader::RecordData Record; + while (1) { + llvm::BitstreamEntry Entry = Stream.advance(llvm::BitstreamCursor::AF_DontPopBlockAtEnd); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + Error = "malformed serialization"; + return false; + + case llvm::BitstreamEntry::EndBlock: + if (BlockStack.empty()) + return true; + BlockStack.pop_back(); + if (Stream.ReadBlockEnd()) { + Error = "malformed serialization"; + return false; + } + if (Stream.AtEndOfStream()) + return true; + break; + + case llvm::BitstreamEntry::SubBlock: { + if (Entry.ID == llvm::bitc::BLOCKINFO_BLOCK_ID) { + BlockInfo = Stream.ReadBlockInfoBlock(); + if (!BlockInfo) { + Error = "malformed BlockInfoBlock"; + return false; + } + Stream.setBlockInfo(&*BlockInfo); + break; + } + + StreamVisit Ret = static_cast(this)->visitBlock(Entry.ID); + switch (Ret) { + case StreamVisit::Continue: + if (Stream.EnterSubBlock(Entry.ID)) { + Error = "malformed block record"; + return false; + } + readBlockAbbrevs(Stream); + BlockStack.push_back(Entry.ID); + break; + + case StreamVisit::Skip: + if (Stream.SkipBlock()) { + Error = "malformed serialization"; + return false; + } + if (Stream.AtEndOfStream()) + return true; + break; + + case StreamVisit::Abort: + return false; + } + break; + } + + case llvm::BitstreamEntry::Record: { + Record.clear(); + StringRef Blob; + unsigned RecID = Stream.readRecord(Entry.ID, Record, &Blob); + unsigned BlockID = BlockStack.empty() ? 0 : BlockStack.back(); + StreamVisit Ret = static_cast(this)->visitRecord(BlockID, RecID, Record, Blob); + switch (Ret) { + case StreamVisit::Continue: + break; + + case StreamVisit::Skip: + Stream.skipRecord(Entry.ID); + break; + + case StreamVisit::Abort: + return false; + } + break; + } + } + } + } + + static void readBlockAbbrevs(llvm::BitstreamCursor &Cursor) { + while (true) { + uint64_t Offset = Cursor.GetCurrentBitNo(); + unsigned Code = Cursor.ReadCode(); + + // We expect all abbrevs to be at the start of the block. + if (Code != llvm::bitc::DEFINE_ABBREV) { + Cursor.JumpToBit(Offset); + return; + } + Cursor.ReadAbbrevRecord(); + } + } +}; + +} // end namespace store +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/CMakeLists.txt =================================================================== --- lib/Index/CMakeLists.txt +++ lib/Index/CMakeLists.txt @@ -1,17 +1,27 @@ set(LLVM_LINK_COMPONENTS + BitReader Core Support ) add_clang_library(clangIndex + ClangIndexRecordWriter.cpp CodegenNameGenerator.cpp CommentToXML.cpp + FileIndexRecord.cpp IndexBody.cpp + IndexDataStore.cpp + IndexDataStoreUtils.cpp IndexDecl.cpp IndexingAction.cpp IndexingContext.cpp + IndexRecordHasher.cpp + IndexRecordReader.cpp + IndexRecordWriter.cpp IndexSymbol.cpp IndexTypeSourceInfo.cpp + IndexUnitReader.cpp + IndexUnitWriter.cpp USRGeneration.cpp ADDITIONAL_HEADERS @@ -23,6 +33,7 @@ clangBasic clangFormat clangFrontend + clangLex clangRewrite clangSerialization clangToolingCore Index: lib/Index/ClangIndexRecordWriter.h =================================================================== --- /dev/null +++ lib/Index/ClangIndexRecordWriter.h @@ -0,0 +1,55 @@ +//===--- ClangIndexRecordWriter.h - Index record serialization ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_CLANGINDEXRECORDWRITER_H +#define LLVM_CLANG_LIB_INDEX_CLANGINDEXRECORDWRITER_H + +#include "IndexRecordHasher.h" +#include "clang/Index/IndexRecordWriter.h" +#include "clang/Index/IndexingAction.h" +#include "clang/Index/CodegenNameGenerator.h" +#include "llvm/ADT/SmallString.h" + +namespace clang { + class ASTContext; + class Decl; + +namespace index { + class FileIndexRecord; + +class ClangIndexRecordWriter { + IndexRecordWriter Impl; + + ASTContext &Ctx; + RecordingOptions RecordOpts; + + std::unique_ptr CGNameGen; + llvm::BumpPtrAllocator Allocator; + llvm::DenseMap USRByDecl; + IndexRecordHasher Hasher; + +public: + ClangIndexRecordWriter(ASTContext &Ctx, RecordingOptions Opts); + ~ClangIndexRecordWriter(); + + ASTContext &getASTContext() { return Ctx; } + CodegenNameGenerator *getCGNameGen() { return CGNameGen.get(); } + + bool writeRecord(StringRef Filename, const FileIndexRecord &Record, + std::string &Error, std::string *RecordFile = nullptr); + StringRef getUSR(const Decl *D); + +private: + StringRef getUSRNonCached(const Decl *D); +}; + +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/ClangIndexRecordWriter.cpp =================================================================== --- /dev/null +++ lib/Index/ClangIndexRecordWriter.cpp @@ -0,0 +1,128 @@ +//===--- ClangIndexRecordWriter.cpp - Index record serialization ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ClangIndexRecordWriter.h" +#include "FileIndexRecord.h" +#include "clang/Index/IndexSymbol.h" +#include "clang/Index/IndexRecordReader.h" +#include "clang/Index/USRGeneration.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclCXX.h" +#include "clang/AST/DeclObjC.h" + +using namespace clang; +using namespace clang::index; + +StringRef ClangIndexRecordWriter::getUSR(const Decl *D) { + assert(D->isCanonicalDecl()); + auto Insert = USRByDecl.insert(std::make_pair(D, StringRef())); + if (Insert.second) { + Insert.first->second = getUSRNonCached(D); + } + return Insert.first->second; +} + +StringRef ClangIndexRecordWriter::getUSRNonCached(const Decl *D) { + SmallString<256> Buf; + bool Ignore = generateUSRForDecl(D, Buf); + if (Ignore) + return StringRef(); + StringRef USR = Buf.str(); + char *Ptr = Allocator.Allocate(USR.size()); + std::copy(USR.begin(), USR.end(), Ptr); + return StringRef(Ptr, USR.size()); +} + +ClangIndexRecordWriter::ClangIndexRecordWriter(ASTContext &Ctx, + RecordingOptions Opts) + : Impl(Opts.DataDirPath), Ctx(Ctx), RecordOpts(std::move(Opts)), + Hasher(Ctx) { + if (Opts.RecordSymbolCodeGenName) + CGNameGen.reset(new CodegenNameGenerator(Ctx)); +} + +ClangIndexRecordWriter::~ClangIndexRecordWriter() {} + +bool ClangIndexRecordWriter::writeRecord(StringRef Filename, + const FileIndexRecord &IdxRecord, + std::string &Error, + std::string *OutRecordFile) { + + auto RecordHash = Hasher.hashRecord(IdxRecord); + + switch (Impl.beginRecord(Filename, RecordHash, Error, OutRecordFile)) { + case IndexRecordWriter::Result::Success: + break; // Continue writing. + case IndexRecordWriter::Result::Failure: + return true; + case IndexRecordWriter::Result::AlreadyExists: + return false; + } + + ASTContext &Ctx = getASTContext(); + SourceManager &SM = Ctx.getSourceManager(); + FileID FID = IdxRecord.getFileID(); + auto getLineCol = [&](unsigned Offset) -> std::pair { + unsigned LineNo = SM.getLineNumber(FID, Offset); + unsigned ColNo = SM.getColumnNumber(FID, Offset); + return std::make_pair(LineNo, ColNo); + }; + + for (auto &Occur : IdxRecord.getDeclOccurrences()) { + unsigned Line, Col; + std::tie(Line, Col) = getLineCol(Occur.Offset); + SmallVector Related; + Related.reserve(Occur.Relations.size()); + for (auto &Rel : Occur.Relations) + Related.push_back(writer::SymbolRelation{Rel.RelatedSymbol, Rel.Roles}); + + Impl.addOccurrence(Occur.Dcl, Occur.Roles, Line, Col, Related); + } + + PrintingPolicy Policy(Ctx.getLangOpts()); + Policy.SuppressTemplateArgsInCXXConstructors = true; + + auto Result = Impl.endRecord(Error, + [&](writer::OpaqueDecl OD, SmallVectorImpl &Scratch) { + const Decl *D = static_cast(OD); + auto Info = getSymbolInfo(D); + + writer::Symbol Sym; + Sym.SymInfo = Info; + + auto *ND = dyn_cast(D); + if (ND) { + llvm::raw_svector_ostream OS(Scratch); + DeclarationName DeclName = ND->getDeclName(); + if (!DeclName.isEmpty()) + DeclName.print(OS, Policy); + } + unsigned NameLen = Scratch.size(); + Sym.Name = StringRef(Scratch.data(), NameLen); + + Sym.USR = getUSR(D); + assert(!Sym.USR.empty() && "Recorded decl without USR!"); + + if (CGNameGen && ND) { + llvm::raw_svector_ostream OS(Scratch); + CGNameGen->writeName(ND, OS); + } + unsigned CGNameLen = Scratch.size() - NameLen; + Sym.CodeGenName = StringRef(Scratch.data() + NameLen, CGNameLen); + return Sym; + }); + + switch (Result) { + case IndexRecordWriter::Result::Success: + case IndexRecordWriter::Result::AlreadyExists: + return false; + case IndexRecordWriter::Result::Failure: + return true; + } +} Index: lib/Index/FileIndexRecord.h =================================================================== --- /dev/null +++ lib/Index/FileIndexRecord.h @@ -0,0 +1,69 @@ +//===--- FileIndexRecord.h - Index data per file --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H +#define LLVM_CLANG_LIB_INDEX_FILEINDEXRECORD_H + +#include "clang/Index/IndexSymbol.h" +#include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include + +namespace clang { + class IdentifierInfo; + +namespace index { + +class FileIndexRecord { +public: + struct DeclOccurrence { + SymbolRoleSet Roles; + unsigned Offset; + const Decl *Dcl; + SmallVector Relations; + + DeclOccurrence(SymbolRoleSet R, + unsigned Offset, + const Decl *D, + ArrayRef Relations) + : Roles(R), + Offset(Offset), + Dcl(D), + Relations(Relations.begin(), Relations.end()) {} + + friend bool operator <(const DeclOccurrence &LHS, const DeclOccurrence &RHS) { + return LHS.Offset < RHS.Offset; + } + }; + +private: + FileID FID; + bool IsSystem; + std::vector Decls; + +public: + FileIndexRecord(FileID FID, bool isSystem) : FID(FID), IsSystem(isSystem) {} + + ArrayRef getDeclOccurrences() const { return Decls; } + + FileID getFileID() const { return FID; } + bool isSystem() const { return IsSystem; } + + void addDeclOccurence(SymbolRoleSet Roles, + unsigned Offset, + const Decl *D, + ArrayRef Relations); + void print(llvm::raw_ostream &OS); +}; + +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/FileIndexRecord.cpp =================================================================== --- /dev/null +++ lib/Index/FileIndexRecord.cpp @@ -0,0 +1,59 @@ +//===--- FileIndexRecord.cpp - Index data per file ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "FileIndexRecord.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/DeclTemplate.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/Path.h" + +using namespace clang; +using namespace clang::index; + +void FileIndexRecord::addDeclOccurence(SymbolRoleSet Roles, + unsigned Offset, + const Decl *D, + ArrayRef Relations) { + assert(D->isCanonicalDecl()); + + auto IsNextOccurence = [&]()->bool { + if (Decls.empty()) + return true; + auto &Last = Decls.back(); + return Last.Offset < Offset; + }; + + if (IsNextOccurence()) { + Decls.emplace_back(Roles, Offset, D, Relations); + return; + } + + DeclOccurrence NewInfo(Roles, Offset, D, Relations); + auto It = std::upper_bound(Decls.begin(), Decls.end(), NewInfo); + Decls.insert(It, std::move(NewInfo)); +} + +void FileIndexRecord::print(llvm::raw_ostream &OS) { + OS << "DECLS BEGIN ---\n"; + for (auto &DclInfo : Decls) { + auto D = DclInfo.Dcl; + SourceManager &SM = D->getASTContext().getSourceManager(); + SourceLocation Loc = SM.getFileLoc(D->getLocation()); + PresumedLoc PLoc = SM.getPresumedLoc(Loc); + OS << llvm::sys::path::filename(PLoc.getFilename()) << ':' << PLoc.getLine() + << ':' << PLoc.getColumn(); + + if (auto ND = dyn_cast(D)) { + OS << ' ' << ND->getNameAsString(); + } + + OS << '\n'; + } + OS << "DECLS END ---\n"; +} Index: lib/Index/IndexDataStore.cpp =================================================================== --- /dev/null +++ lib/Index/IndexDataStore.cpp @@ -0,0 +1,259 @@ +//===--- IndexDataStore.cpp - Index data store info -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexDataStore.h" +#include "IndexDataStoreUtils.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + +static void appendSubDir(StringRef subdir, SmallVectorImpl &StorePathBuf) { + SmallString<10> VersionPath; + raw_svector_ostream(VersionPath) << 'v' << STORE_FORMAT_VERSION; + + sys::path::append(StorePathBuf, VersionPath); + sys::path::append(StorePathBuf, subdir); +} + +void store::appendInteriorUnitPath(StringRef UnitName, + SmallVectorImpl &PathBuf) { + sys::path::append(PathBuf, UnitName); +} + +void store::appendUnitSubDir(SmallVectorImpl &StorePathBuf) { + return appendSubDir("units", StorePathBuf); +} + +void store::appendRecordSubDir(SmallVectorImpl &StorePathBuf) { + return appendSubDir("records", StorePathBuf); +} + +void store::appendInteriorRecordPath(StringRef RecordName, + SmallVectorImpl &PathBuf) { + // To avoid putting a huge number of files into the records directory, create + // subdirectories based on the last 2 characters from the hash. + StringRef hash2chars = RecordName.substr(RecordName.size()-2); + sys::path::append(PathBuf, hash2chars); + sys::path::append(PathBuf, RecordName); +} + +//===----------------------------------------------------------------------===// +// IndexDataStore +//===----------------------------------------------------------------------===// + +namespace { + +class UnitEventHandlerData { + mutable sys::Mutex Mtx; + IndexDataStore::UnitEventHandler Handler; + +public: + void setHandler(IndexDataStore::UnitEventHandler handler) { + sys::ScopedLock L(Mtx); + Handler = std::move(handler); + } + IndexDataStore::UnitEventHandler getHandler() const { + sys::ScopedLock L(Mtx); + return Handler; + } +}; + +class IndexDataStoreImpl { + std::string FilePath; + std::shared_ptr TheUnitEventHandlerData; + std::unique_ptr DirWatcher; + +public: + explicit IndexDataStoreImpl(StringRef indexStorePath) + : FilePath(indexStorePath) { + TheUnitEventHandlerData = std::make_shared(); + } + + StringRef getFilePath() const { return FilePath; } + bool foreachUnitName(bool sorted, + llvm::function_ref receiver); + void setUnitEventHandler(IndexDataStore::UnitEventHandler Handler); + bool startEventListening(llvm::function_ref createFn, + bool waitInitialSync, std::string &Error); + void stopEventListening(); + void discardUnit(StringRef UnitName); + void discardRecord(StringRef RecordName); + void purgeStaleData(); +}; + +} // anonymous namespace + +bool IndexDataStoreImpl::foreachUnitName(bool sorted, + llvm::function_ref receiver) { + SmallString<128> UnitPath; + UnitPath = FilePath; + appendUnitSubDir(UnitPath); + + std::vector filenames; + + std::error_code EC; + for (auto It = sys::fs::directory_iterator(UnitPath, EC), + End = sys::fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + StringRef unitName = sys::path::filename(It->path()); + if (!sorted) { + if (!receiver(unitName)) + return false; + } else { + filenames.push_back(unitName); + } + } + + if (sorted) { + llvm::array_pod_sort(filenames.begin(), filenames.end()); + for (auto &fname : filenames) + if (!receiver(fname)) + return false; + } + return true; +} + +void IndexDataStoreImpl::setUnitEventHandler(IndexDataStore::UnitEventHandler handler) { + TheUnitEventHandlerData->setHandler(std::move(handler)); +} + +bool IndexDataStoreImpl::startEventListening(llvm::function_ref createFn, + bool waitInitialSync, std::string &Error) { + if (DirWatcher) { + Error = "event listener already active"; + return true; + } + + SmallString<128> UnitPath; + UnitPath = FilePath; + appendUnitSubDir(UnitPath); + + auto localUnitEventHandlerData = TheUnitEventHandlerData; + auto OnUnitsChange = [localUnitEventHandlerData](ArrayRef Events, bool isInitial) { + SmallVector UnitEvents; + UnitEvents.reserve(Events.size()); + for (const AbstractDirectoryWatcher::Event &evt : Events) { + IndexDataStore::UnitEventKind K; + StringRef UnitName = sys::path::filename(evt.Filename); + switch (evt.Kind) { + case AbstractDirectoryWatcher::EventKind::Added: + K = IndexDataStore::UnitEventKind::Added; break; + case AbstractDirectoryWatcher::EventKind::Removed: + K = IndexDataStore::UnitEventKind::Removed; break; + case AbstractDirectoryWatcher::EventKind::Modified: + K = IndexDataStore::UnitEventKind::Modified; break; + case AbstractDirectoryWatcher::EventKind::DirectoryDeleted: + K = IndexDataStore::UnitEventKind::DirectoryDeleted; + UnitName = StringRef(); + break; + } + UnitEvents.push_back(IndexDataStore::UnitEvent{K, UnitName, evt.ModTime}); + } + + if (auto handler = localUnitEventHandlerData->getHandler()) { + IndexDataStore::UnitEventNotification EventNote{isInitial, UnitEvents}; + handler(EventNote); + } + }; + + DirWatcher = createFn(UnitPath.str(), OnUnitsChange, waitInitialSync, Error); + if (!DirWatcher) + return true; + + return false; +} + +void IndexDataStoreImpl::stopEventListening() { + DirWatcher.reset(); +} + +void IndexDataStoreImpl::discardUnit(StringRef UnitName) { + SmallString<128> UnitPath; + UnitPath = FilePath; + appendUnitSubDir(UnitPath); + appendInteriorUnitPath(UnitName, UnitPath); + sys::fs::remove(UnitPath); +} + +void IndexDataStoreImpl::discardRecord(StringRef RecordName) { + SmallString<128> RecordPath; + RecordPath = FilePath; + appendRecordSubDir(RecordPath); + appendInteriorRecordPath(RecordName, RecordPath); + sys::fs::remove(RecordPath); +} + +void IndexDataStoreImpl::purgeStaleData() { + // FIXME: Implement. +} + + +std::unique_ptr +IndexDataStore::create(StringRef IndexStorePath, std::string &Error) { + if (!sys::fs::exists(IndexStorePath)) { + raw_string_ostream OS(Error); + OS << "index store path does not exist: " << IndexStorePath; + return nullptr; + } + + return std::unique_ptr( + new IndexDataStore(new IndexDataStoreImpl(IndexStorePath))); +} + +#define IMPL static_cast(Impl) + +IndexDataStore::~IndexDataStore() { + delete IMPL; +} + +StringRef IndexDataStore::getFilePath() const { + return IMPL->getFilePath(); +} + +bool IndexDataStore::foreachUnitName(bool sorted, + llvm::function_ref receiver) { + return IMPL->foreachUnitName(sorted, std::move(receiver)); +} + +unsigned IndexDataStore::getFormatVersion() { + return STORE_FORMAT_VERSION; +} + +void IndexDataStore::setUnitEventHandler(UnitEventHandler Handler) { + return IMPL->setUnitEventHandler(std::move(Handler)); +} + +bool IndexDataStore::startEventListening(llvm::function_ref createFn, + bool waitInitialSync, std::string &Error) { + return IMPL->startEventListening(std::move(createFn), waitInitialSync, Error); +} + +void IndexDataStore::stopEventListening() { + return IMPL->stopEventListening(); +} + +void IndexDataStore::discardUnit(StringRef UnitName) { + IMPL->discardUnit(UnitName); +} + +void IndexDataStore::discardRecord(StringRef RecordName) { + IMPL->discardRecord(RecordName); +} + +void IndexDataStore::purgeStaleData() { + IMPL->purgeStaleData(); +} Index: lib/Index/IndexDataStoreUtils.h =================================================================== --- /dev/null +++ lib/Index/IndexDataStoreUtils.h @@ -0,0 +1,116 @@ +//===--- IndexDataStoreUtils.h - Functions/constants for the data store ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_INDEXDATASTOREUTILS_H +#define LLVM_CLANG_LIB_INDEX_INDEXDATASTOREUTILS_H + +#include "llvm/Bitcode/BitCodes.h" +#include "clang/Basic/LLVM.h" + +namespace llvm { + class BitstreamWriter; +} + +namespace clang { +namespace index { +namespace store { + +static const unsigned STORE_FORMAT_VERSION = 5; + +void appendUnitSubDir(SmallVectorImpl &StorePathBuf); +void appendInteriorUnitPath(StringRef UnitName, + SmallVectorImpl &PathBuf); +void appendRecordSubDir(SmallVectorImpl &StorePathBuf); +void appendInteriorRecordPath(StringRef RecordName, + SmallVectorImpl &PathBuf); + +enum RecordBitRecord { + REC_VERSION = 0, + REC_DECLINFO = 1, + REC_DECLOFFSETS = 2, + REC_DECLOCCURRENCE = 3, +}; + +enum RecordBitBlock { + REC_VERSION_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID, + REC_DECLS_BLOCK_ID, + REC_DECLOFFSETS_BLOCK_ID, + REC_DECLOCCURRENCES_BLOCK_ID, +}; + +enum UnitBitRecord { + UNIT_VERSION = 0, + UNIT_INFO = 1, + UNIT_DEPENDENCY = 2, + UNIT_INCLUDE = 3, + UNIT_PATH = 4, + UNIT_PATH_BUFFER = 5, + UNIT_MODULE = 6, + UNIT_MODULE_BUFFER = 7, +}; + +enum UnitBitBlock { + UNIT_VERSION_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID, + UNIT_INFO_BLOCK_ID, + UNIT_DEPENDENCIES_BLOCK_ID, + UNIT_INCLUDES_BLOCK_ID, + UNIT_PATHS_BLOCK_ID, + UNIT_MODULES_BLOCK_ID, +}; + +enum UnitDependencyKind { + UNIT_DEPEND_KIND_FILE = 0, + UNIT_DEPEND_KIND_RECORD = 1, + UNIT_DEPEND_KIND_UNIT = 2, +}; +static const unsigned UnitDependencyKindBitNum = 2; + +enum UnitFilePathPrefixKind { + UNIT_PATH_PREFIX_NONE = 0, + UNIT_PATH_PREFIX_WORKDIR = 1, + UNIT_PATH_PREFIX_SYSROOT = 2, +}; +static const unsigned UnitFilePathPrefixKindBitNum = 2; + +typedef SmallVector RecordData; +typedef SmallVectorImpl RecordDataImpl; + +struct BitPathComponent { + size_t Offset = 0; + size_t Size = 0; + BitPathComponent(size_t Offset, size_t Size) : Offset(Offset), Size(Size) {} + BitPathComponent() = default; +}; + +struct DirBitPath { + UnitFilePathPrefixKind PrefixKind = UNIT_PATH_PREFIX_NONE; + BitPathComponent Dir; + DirBitPath(UnitFilePathPrefixKind Kind, + BitPathComponent Dir) : PrefixKind(Kind), Dir(Dir) {} + DirBitPath() = default; +}; + +struct FileBitPath : DirBitPath { + BitPathComponent Filename; + FileBitPath(UnitFilePathPrefixKind Kind, BitPathComponent Dir, + BitPathComponent Filename) : DirBitPath(Kind, Dir), Filename(Filename) {} + FileBitPath() = default; +}; + +void emitBlockID(unsigned ID, const char *Name, + llvm::BitstreamWriter &Stream, RecordDataImpl &Record); + +void emitRecordID(unsigned ID, const char *Name, + llvm::BitstreamWriter &Stream, RecordDataImpl &Record); + +} // end namespace store +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/IndexDataStoreUtils.cpp =================================================================== --- /dev/null +++ lib/Index/IndexDataStoreUtils.cpp @@ -0,0 +1,350 @@ +//===--- IndexDataStoreUtils.cpp - Functions/constants for the data store -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexDataStoreSymbolUtils.h" +#include "IndexDataStoreUtils.h" +#include "llvm/Bitcode/BitstreamWriter.h" + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + +void store::emitBlockID(unsigned ID, const char *Name, + BitstreamWriter &Stream, RecordDataImpl &Record) { + Record.clear(); + Record.push_back(ID); + Stream.EmitRecord(bitc::BLOCKINFO_CODE_SETBID, Record); + + // Emit the block name if present. + if (!Name || Name[0] == 0) + return; + Record.clear(); + while (*Name) + Record.push_back(*Name++); + Stream.EmitRecord(bitc::BLOCKINFO_CODE_BLOCKNAME, Record); +} + +void store::emitRecordID(unsigned ID, const char *Name, + BitstreamWriter &Stream, + RecordDataImpl &Record) { + Record.clear(); + Record.push_back(ID); + while (*Name) + Record.push_back(*Name++); + Stream.EmitRecord(bitc::BLOCKINFO_CODE_SETRECORDNAME, Record); +} + +/// Map an indexstore_symbol_kind_t to a SymbolKind, handling unknown values. +SymbolKind index::getSymbolKind(indexstore_symbol_kind_t K) { + switch ((uint64_t)K) { + default: + case INDEXSTORE_SYMBOL_KIND_UNKNOWN: + return SymbolKind::Unknown; + case INDEXSTORE_SYMBOL_KIND_MODULE: + return SymbolKind::Module; + case INDEXSTORE_SYMBOL_KIND_NAMESPACE: + return SymbolKind::Namespace; + case INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS: + return SymbolKind::NamespaceAlias; + case INDEXSTORE_SYMBOL_KIND_MACRO: + return SymbolKind::Macro; + case INDEXSTORE_SYMBOL_KIND_ENUM: + return SymbolKind::Enum; + case INDEXSTORE_SYMBOL_KIND_STRUCT: + return SymbolKind::Struct; + case INDEXSTORE_SYMBOL_KIND_CLASS: + return SymbolKind::Class; + case INDEXSTORE_SYMBOL_KIND_PROTOCOL: + return SymbolKind::Protocol; + case INDEXSTORE_SYMBOL_KIND_EXTENSION: + return SymbolKind::Extension; + case INDEXSTORE_SYMBOL_KIND_UNION: + return SymbolKind::Union; + case INDEXSTORE_SYMBOL_KIND_TYPEALIAS: + return SymbolKind::TypeAlias; + case INDEXSTORE_SYMBOL_KIND_FUNCTION: + return SymbolKind::Function; + case INDEXSTORE_SYMBOL_KIND_VARIABLE: + return SymbolKind::Variable; + case INDEXSTORE_SYMBOL_KIND_FIELD: + return SymbolKind::Field; + case INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT: + return SymbolKind::EnumConstant; + case INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD: + return SymbolKind::InstanceMethod; + case INDEXSTORE_SYMBOL_KIND_CLASSMETHOD: + return SymbolKind::ClassMethod; + case INDEXSTORE_SYMBOL_KIND_STATICMETHOD: + return SymbolKind::StaticMethod; + case INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY: + return SymbolKind::InstanceProperty; + case INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY: + return SymbolKind::ClassProperty; + case INDEXSTORE_SYMBOL_KIND_STATICPROPERTY: + return SymbolKind::StaticProperty; + case INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR: + return SymbolKind::Constructor; + case INDEXSTORE_SYMBOL_KIND_DESTRUCTOR: + return SymbolKind::Destructor; + case INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION: + return SymbolKind::ConversionFunction; + case INDEXSTORE_SYMBOL_KIND_PARAMETER: + return SymbolKind::Parameter; + case INDEXSTORE_SYMBOL_KIND_USING: + return SymbolKind::Using; + } +} + +SymbolSubKind index::getSymbolSubKind(indexstore_symbol_subkind_t K) { + switch ((uint64_t)K) { + default: + case INDEXSTORE_SYMBOL_SUBKIND_NONE: + return SymbolSubKind::None; + case INDEXSTORE_SYMBOL_SUBKIND_CXXCOPYCONSTRUCTOR: + return SymbolSubKind::CXXCopyConstructor; + case INDEXSTORE_SYMBOL_SUBKIND_CXXMOVECONSTRUCTOR: + return SymbolSubKind::CXXMoveConstructor; + case INDEXSTORE_SYMBOL_SUBKIND_ACCESSORGETTER: + return SymbolSubKind::AccessorGetter; + case INDEXSTORE_SYMBOL_SUBKIND_ACCESSORSETTER: + return SymbolSubKind::AccessorSetter; + case INDEXSTORE_SYMBOL_SUBKIND_USINGTYPENAME: + return SymbolSubKind::UsingTypename; + case INDEXSTORE_SYMBOL_SUBKIND_USINGVALUE: + return SymbolSubKind::UsingValue; + } +} + +/// Map an indexstore_symbol_language_t to a SymbolLanguage, handling unknown +/// values. +SymbolLanguage index::getSymbolLanguage(indexstore_symbol_language_t L) { + switch ((uint64_t)L) { + default: // FIXME: add an unknown language? + case INDEXSTORE_SYMBOL_LANG_C: + return SymbolLanguage::C; + case INDEXSTORE_SYMBOL_LANG_OBJC: + return SymbolLanguage::ObjC; + case INDEXSTORE_SYMBOL_LANG_CXX: + return SymbolLanguage::CXX; + case INDEXSTORE_SYMBOL_LANG_SWIFT: + return SymbolLanguage::Swift; + } +} + +/// Map an indexstore representation to a SymbolPropertySet, handling +/// unknown values. +SymbolPropertySet index::getSymbolProperties(uint64_t Props) { + // FIXME: currently these enums must be kept in sync. + return (uint64_t)Props; +} + +/// Map an indexstore representation to a SymbolRoleSet, handling unknown +/// values. +SymbolRoleSet index::getSymbolRoles(uint64_t Roles) { + // FIXME: currently these enums must be kept in sync. + return (uint64_t)Roles; +} + +/// Map a SymbolLanguage to a indexstore_symbol_language_t. +indexstore_symbol_kind_t index::getIndexStoreKind(SymbolKind K) { + switch (K) { + case SymbolKind::Unknown: + return INDEXSTORE_SYMBOL_KIND_UNKNOWN; + case SymbolKind::Module: + return INDEXSTORE_SYMBOL_KIND_MODULE; + case SymbolKind::Namespace: + return INDEXSTORE_SYMBOL_KIND_NAMESPACE; + case SymbolKind::NamespaceAlias: + return INDEXSTORE_SYMBOL_KIND_NAMESPACEALIAS; + case SymbolKind::Macro: + return INDEXSTORE_SYMBOL_KIND_MACRO; + case SymbolKind::Enum: + return INDEXSTORE_SYMBOL_KIND_ENUM; + case SymbolKind::Struct: + return INDEXSTORE_SYMBOL_KIND_STRUCT; + case SymbolKind::Class: + return INDEXSTORE_SYMBOL_KIND_CLASS; + case SymbolKind::Protocol: + return INDEXSTORE_SYMBOL_KIND_PROTOCOL; + case SymbolKind::Extension: + return INDEXSTORE_SYMBOL_KIND_EXTENSION; + case SymbolKind::Union: + return INDEXSTORE_SYMBOL_KIND_UNION; + case SymbolKind::TypeAlias: + return INDEXSTORE_SYMBOL_KIND_TYPEALIAS; + case SymbolKind::Function: + return INDEXSTORE_SYMBOL_KIND_FUNCTION; + case SymbolKind::Variable: + return INDEXSTORE_SYMBOL_KIND_VARIABLE; + case SymbolKind::Field: + return INDEXSTORE_SYMBOL_KIND_FIELD; + case SymbolKind::EnumConstant: + return INDEXSTORE_SYMBOL_KIND_ENUMCONSTANT; + case SymbolKind::InstanceMethod: + return INDEXSTORE_SYMBOL_KIND_INSTANCEMETHOD; + case SymbolKind::ClassMethod: + return INDEXSTORE_SYMBOL_KIND_CLASSMETHOD; + case SymbolKind::StaticMethod: + return INDEXSTORE_SYMBOL_KIND_STATICMETHOD; + case SymbolKind::InstanceProperty: + return INDEXSTORE_SYMBOL_KIND_INSTANCEPROPERTY; + case SymbolKind::ClassProperty: + return INDEXSTORE_SYMBOL_KIND_CLASSPROPERTY; + case SymbolKind::StaticProperty: + return INDEXSTORE_SYMBOL_KIND_STATICPROPERTY; + case SymbolKind::Constructor: + return INDEXSTORE_SYMBOL_KIND_CONSTRUCTOR; + case SymbolKind::Destructor: + return INDEXSTORE_SYMBOL_KIND_DESTRUCTOR; + case SymbolKind::ConversionFunction: + return INDEXSTORE_SYMBOL_KIND_CONVERSIONFUNCTION; + case SymbolKind::Parameter: + return INDEXSTORE_SYMBOL_KIND_PARAMETER; + case SymbolKind::Using: + return INDEXSTORE_SYMBOL_KIND_USING; + } + llvm_unreachable("unexpected symbol kind"); +} + +indexstore_symbol_subkind_t index::getIndexStoreSubKind(SymbolSubKind K) { + switch (K) { + case SymbolSubKind::None: + return INDEXSTORE_SYMBOL_SUBKIND_NONE; + case SymbolSubKind::CXXCopyConstructor: + return INDEXSTORE_SYMBOL_SUBKIND_CXXCOPYCONSTRUCTOR; + case SymbolSubKind::CXXMoveConstructor: + return INDEXSTORE_SYMBOL_SUBKIND_CXXMOVECONSTRUCTOR; + case SymbolSubKind::AccessorGetter: + return INDEXSTORE_SYMBOL_SUBKIND_ACCESSORGETTER; + case SymbolSubKind::AccessorSetter: + return INDEXSTORE_SYMBOL_SUBKIND_ACCESSORSETTER; + case SymbolSubKind::UsingTypename: + return INDEXSTORE_SYMBOL_SUBKIND_USINGTYPENAME; + case SymbolSubKind::UsingValue: + return INDEXSTORE_SYMBOL_SUBKIND_USINGVALUE; + } + llvm_unreachable("unexpected symbol subkind"); +} + +/// Map a SymbolLanguage to a indexstore_symbol_language_t. +indexstore_symbol_language_t index::getIndexStoreLang(SymbolLanguage L) { + switch (L) { + case SymbolLanguage::C: + return INDEXSTORE_SYMBOL_LANG_C; + case SymbolLanguage::ObjC: + return INDEXSTORE_SYMBOL_LANG_OBJC; + case SymbolLanguage::CXX: + return INDEXSTORE_SYMBOL_LANG_CXX; + case SymbolLanguage::Swift: + return INDEXSTORE_SYMBOL_LANG_SWIFT; + } + llvm_unreachable("unexpected symbol language"); +} + +/// Map a SymbolPropertySet to its indexstore representation. +uint64_t index::getIndexStoreProperties(SymbolPropertySet Props) { + uint64_t storeProp = 0; + applyForEachSymbolProperty(Props, [&](SymbolProperty prop) { + switch (prop) { + case SymbolProperty::Generic: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_GENERIC; + break; + case SymbolProperty::TemplatePartialSpecialization: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_PARTIAL_SPECIALIZATION; + break; + case SymbolProperty::TemplateSpecialization: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_TEMPLATE_SPECIALIZATION; + break; + case SymbolProperty::UnitTest: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_UNITTEST; + break; + case SymbolProperty::IBAnnotated: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_IBANNOTATED; + break; + case SymbolProperty::IBOutletCollection: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_IBOUTLETCOLLECTION; + break; + case SymbolProperty::GKInspectable: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_GKINSPECTABLE; + break; + case SymbolProperty::Local: + storeProp |= INDEXSTORE_SYMBOL_PROPERTY_LOCAL; + break; + } + }); + return storeProp; +} + +/// Map a SymbolRoleSet to its indexstore representation. +uint64_t index::getIndexStoreRoles(SymbolRoleSet Roles) { + uint64_t storeRoles = 0; + applyForEachSymbolRole(Roles, [&](SymbolRole role) { + switch (role) { + case SymbolRole::Declaration: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_DECLARATION; + break; + case SymbolRole::Definition: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_DEFINITION; + break; + case SymbolRole::Reference: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REFERENCE; + break; + case SymbolRole::Read: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_READ; + break; + case SymbolRole::Write: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_WRITE; + break; + case SymbolRole::Call: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_CALL; + break; + case SymbolRole::Dynamic: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_DYNAMIC; + break; + case SymbolRole::AddressOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_ADDRESSOF; + break; + case SymbolRole::Implicit: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_IMPLICIT; + break; + case SymbolRole::RelationChildOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_CHILDOF; + break; + case SymbolRole::RelationBaseOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_BASEOF; + break; + case SymbolRole::RelationOverrideOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_OVERRIDEOF; + break; + case SymbolRole::RelationReceivedBy: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_RECEIVEDBY; + break; + case SymbolRole::RelationCalledBy: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_CALLEDBY; + break; + case SymbolRole::RelationExtendedBy: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_EXTENDEDBY; + break; + case SymbolRole::RelationAccessorOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_ACCESSOROF; + break; + case SymbolRole::RelationContainedBy: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_CONTAINEDBY; + break; + case SymbolRole::RelationIBTypeOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_IBTYPEOF; + break; + case SymbolRole::RelationSpecializationOf: + storeRoles |= INDEXSTORE_SYMBOL_ROLE_REL_SPECIALIZATIONOF; + break; + } + }); + return storeRoles; +} Index: lib/Index/IndexRecordHasher.h =================================================================== --- /dev/null +++ lib/Index/IndexRecordHasher.h @@ -0,0 +1,58 @@ +//===--- IndexRecordHasher.h - Index record hashing -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INDEX_INDEXRECORDHASHER_H +#define LLVM_CLANG_LIB_INDEX_INDEXRECORDHASHER_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" + +namespace clang { + class ASTContext; + class Decl; + class DeclarationName; + class NestedNameSpecifier; + class QualType; + class Type; + template class CanQual; + typedef CanQual CanQualType; + +namespace index { + class FileIndexRecord; + +class IndexRecordHasher { + ASTContext &Ctx; + llvm::DenseMap HashByPtr; + +public: + explicit IndexRecordHasher(ASTContext &Ctx) : Ctx(Ctx) {} + ASTContext &getASTContext() { return Ctx; } + + llvm::hash_code hashRecord(const FileIndexRecord &Record); + llvm::hash_code hash(const Decl *D); + llvm::hash_code hash(QualType Ty); + llvm::hash_code hash(CanQualType Ty); + llvm::hash_code hash(DeclarationName Name); + llvm::hash_code hash(const NestedNameSpecifier *NNS); + +private: + template + llvm::hash_code tryCache(const void *Ptr, T Obj); + + llvm::hash_code hashImpl(const Decl *D); + llvm::hash_code hashImpl(CanQualType Ty); + llvm::hash_code hashImpl(DeclarationName Name); + llvm::hash_code hashImpl(const NestedNameSpecifier *NNS); +}; + +} // end namespace index +} // end namespace clang + +#endif Index: lib/Index/IndexRecordHasher.cpp =================================================================== --- /dev/null +++ lib/Index/IndexRecordHasher.cpp @@ -0,0 +1,477 @@ +//===--- IndexRecordHasher.cpp - Index record hashing ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "IndexRecordHasher.h" +#include "FileIndexRecord.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Decl.h" +#include "clang/AST/DeclVisitor.h" +#include "llvm/Support/Path.h" + +#define INITIAL_HASH 5381 +#define COMBINE_HASH(...) (Hash = hash_combine(Hash, __VA_ARGS__)) + +using namespace clang; +using namespace clang::index; +using namespace llvm; + +static hash_code computeHash(const TemplateArgument &Arg, + IndexRecordHasher &Hasher); + +namespace { +class DeclHashVisitor : public ConstDeclVisitor { + IndexRecordHasher &Hasher; + +public: + DeclHashVisitor(IndexRecordHasher &Hasher) : Hasher(Hasher) {} + + hash_code VisitDecl(const Decl *D) { + return VisitDeclContext(D->getDeclContext()); + } + + hash_code VisitNamedDecl(const NamedDecl *D) { + hash_code Hash = VisitDecl(D); + if (auto *attr = D->getExternalSourceSymbolAttr()) { + COMBINE_HASH(hash_value(attr->getDefinedIn())); + } + return COMBINE_HASH(Hasher.hash(D->getDeclName())); + } + + hash_code VisitTagDecl(const TagDecl *D) { + if (D->getDeclName().isEmpty()) { + if (const TypedefNameDecl *TD = D->getTypedefNameForAnonDecl()) + return Visit(TD); + + hash_code Hash = VisitDeclContext(D->getDeclContext()); + if (D->isEmbeddedInDeclarator() && !D->isFreeStanding()) { + COMBINE_HASH(hashLoc(D->getLocation(), /*IncludeOffset=*/true)); + } else + COMBINE_HASH('a'); + return Hash; + } + + hash_code Hash = VisitTypeDecl(D); + return COMBINE_HASH('T'); + } + + hash_code VisitClassTemplateSpecializationDecl(const ClassTemplateSpecializationDecl *D) { + hash_code Hash = VisitCXXRecordDecl(D); + const TemplateArgumentList &Args = D->getTemplateArgs(); + COMBINE_HASH('>'); + for (unsigned I = 0, N = Args.size(); I != N; ++I) { + COMBINE_HASH(computeHash(Args.get(I), Hasher)); + } + return Hash; + } + + hash_code VisitObjCContainerDecl(const ObjCContainerDecl *D) { + hash_code Hash = VisitNamedDecl(D); + return COMBINE_HASH('I'); + } + + hash_code VisitObjCImplDecl(const ObjCImplDecl *D) { + if (auto *ID = D->getClassInterface()) + return VisitObjCInterfaceDecl(ID); + else + return 0; + } + + hash_code VisitObjCCategoryDecl(const ObjCCategoryDecl *D) { + // FIXME: Differentiate between category and the interface ? + if (auto *ID = D->getClassInterface()) + return VisitObjCInterfaceDecl(ID); + else + return 0; + } + + hash_code VisitFunctionDecl(const FunctionDecl *D) { + hash_code Hash = VisitNamedDecl(D); + ASTContext &Ctx = Hasher.getASTContext(); + if ((!Ctx.getLangOpts().CPlusPlus && !D->hasAttr()) + || D->isExternC()) + return Hash; + + for (auto param : D->parameters()) { + COMBINE_HASH(Hasher.hash(param->getType())); + } + return Hash; + } + + hash_code VisitUnresolvedUsingTypenameDecl(const UnresolvedUsingTypenameDecl *D) { + hash_code Hash = VisitNamedDecl(D); + COMBINE_HASH(Hasher.hash(D->getQualifier())); + return Hash; + } + + hash_code VisitUnresolvedUsingValueDecl(const UnresolvedUsingValueDecl *D) { + hash_code Hash = VisitNamedDecl(D); + COMBINE_HASH(Hasher.hash(D->getQualifier())); + return Hash; + } + + hash_code VisitDeclContext(const DeclContext *DC) { + // FIXME: Add location if this is anonymous namespace ? + DC = DC->getRedeclContext(); + const Decl *D = cast(DC)->getCanonicalDecl(); + if (auto *ND = dyn_cast(D)) + return Hasher.hash(ND); + else + return 0; + } + + hash_code hashLoc(SourceLocation Loc, bool IncludeOffset) { + if (Loc.isInvalid()) { + return 0; + } + hash_code Hash = INITIAL_HASH; + const SourceManager &SM = Hasher.getASTContext().getSourceManager(); + Loc = SM.getFileLoc(Loc); + const std::pair &Decomposed = SM.getDecomposedLoc(Loc); + const FileEntry *FE = SM.getFileEntryForID(Decomposed.first); + if (FE) { + COMBINE_HASH(llvm::sys::path::filename(FE->getName())); + } else { + // This case really isn't interesting. + return 0; + } + if (IncludeOffset) { + // Use the offest into the FileID to represent the location. Using + // a line/column can cause us to look back at the original source file, + // which is expensive. + COMBINE_HASH(Decomposed.second); + } + return Hash; + } +}; +} + +hash_code IndexRecordHasher::hashRecord(const FileIndexRecord &Record) { + hash_code Hash = INITIAL_HASH; + for (auto &Info : Record.getDeclOccurrences()) { + COMBINE_HASH(Info.Roles, Info.Offset, hash(Info.Dcl)); + for (auto &Rel : Info.Relations) { + COMBINE_HASH(hash(Rel.RelatedSymbol)); + } + } + return Hash; +} + +hash_code IndexRecordHasher::hash(const Decl *D) { + assert(D->isCanonicalDecl()); + + if (isa(D) || isa(D)) { + return tryCache(D, D); + } else if (auto *NS = dyn_cast(D)) { + if (NS->isAnonymousNamespace()) + return hash_value(StringRef("@aN")); + return tryCache(D, D); + } else { + // There's a balance between caching results and not growing the cache too + // much. Measurements showed that avoiding caching all decls is beneficial + // particularly when including all of Cocoa. + return hashImpl(D); + } +} + +hash_code IndexRecordHasher::hash(QualType NonCanTy) { + CanQualType CanTy = Ctx.getCanonicalType(NonCanTy); + return hash(CanTy); +} + +hash_code IndexRecordHasher::hash(CanQualType CT) { + // Do some hashing without going to the cache, for example we can avoid + // storing the hash for both the type and its const-qualified version. + hash_code Hash = INITIAL_HASH; + + auto asCanon = [](QualType Ty) -> CanQualType { + return CanQualType::CreateUnsafe(Ty); + }; + + while (true) { + Qualifiers Q = CT.getQualifiers(); + CT = CT.getUnqualifiedType(); + const Type *T = CT.getTypePtr(); + unsigned qVal = 0; + if (Q.hasConst()) + qVal |= 0x1; + if (Q.hasVolatile()) + qVal |= 0x2; + if (Q.hasRestrict()) + qVal |= 0x4; + if(qVal) + COMBINE_HASH(qVal); + + // Hash in ObjC GC qualifiers? + + if (const BuiltinType *BT = dyn_cast(T)) { + return COMBINE_HASH(BT->getKind()); + } + if (const PointerType *PT = dyn_cast(T)) { + COMBINE_HASH('*'); + CT = asCanon(PT->getPointeeType()); + continue; + } + if (const ReferenceType *RT = dyn_cast(T)) { + COMBINE_HASH('&'); + CT = asCanon(RT->getPointeeType()); + continue; + } + if (const BlockPointerType *BT = dyn_cast(T)) { + COMBINE_HASH('B'); + CT = asCanon(BT->getPointeeType()); + continue; + } + if (const ObjCObjectPointerType *OPT = dyn_cast(T)) { + COMBINE_HASH('*'); + CT = asCanon(OPT->getPointeeType()); + continue; + } + if (const TagType *TT = dyn_cast(T)) { + return COMBINE_HASH('$', hash(TT->getDecl()->getCanonicalDecl())); + } + if (const ObjCInterfaceType *OIT = dyn_cast(T)) { + return COMBINE_HASH('$', hash(OIT->getDecl()->getCanonicalDecl())); + } + if (const ObjCObjectType *OIT = dyn_cast(T)) { + for (auto *Prot : OIT->getProtocols()) + COMBINE_HASH(hash(Prot)); + CT = asCanon(OIT->getBaseType()); + continue; + } + if (const TemplateTypeParmType *TTP = dyn_cast(T)) { + return COMBINE_HASH('t', TTP->getDepth(), TTP->getIndex()); + } + if (const InjectedClassNameType *InjT = dyn_cast(T)) { + CT = asCanon(InjT->getInjectedSpecializationType().getCanonicalType()); + continue; + } + + break; + } + + return COMBINE_HASH(tryCache(CT.getAsOpaquePtr(), CT)); +} + +hash_code IndexRecordHasher::hash(DeclarationName Name) { + assert(!Name.isEmpty()); + // Measurements for using cache or not here, showed significant slowdown when + // using the cache for all DeclarationNames when parsing Cocoa, and minor + // improvement or no difference for a couple of C++ single translation unit + // files. So we avoid caching DeclarationNames. + return hashImpl(Name); +} + +hash_code IndexRecordHasher::hash(const NestedNameSpecifier *NNS) { + assert(NNS); + // Measurements for the C++ single translation unit files did not show much + // difference here; choosing to cache them currently. + return tryCache(NNS, NNS); +} + +template +hash_code IndexRecordHasher::tryCache(const void *Ptr, T Obj) { + auto It = HashByPtr.find(Ptr); + if (It != HashByPtr.end()) + return It->second; + + hash_code Hash = hashImpl(Obj); + // hashImpl() may call into tryCache recursively and mutate + // HashByPtr, so we use find() earlier and insert the hash with another + // lookup here instead of calling insert() earlier and utilizing the iterator + // that insert() returns. + HashByPtr[Ptr] = Hash; + return Hash; +} + +hash_code IndexRecordHasher::hashImpl(const Decl *D) { + return DeclHashVisitor(*this).Visit(D); +} + +static hash_code computeHash(const IdentifierInfo *II) { + return hash_value(II->getName()); +} + +static hash_code computeHash(Selector Sel) { + unsigned N = Sel.getNumArgs(); + if (N == 0) + ++N; + hash_code Hash = INITIAL_HASH; + for (unsigned I = 0; I != N; ++I) + if (IdentifierInfo *II = Sel.getIdentifierInfoForSlot(I)) + COMBINE_HASH(computeHash(II)); + return Hash; +} + +static hash_code computeHash(TemplateName Name, IndexRecordHasher &Hasher) { + hash_code Hash = INITIAL_HASH; + if (TemplateDecl *Template = Name.getAsTemplateDecl()) { + if (TemplateTemplateParmDecl *TTP + = dyn_cast(Template)) { + return COMBINE_HASH('t', TTP->getDepth(), TTP->getIndex()); + } + + return COMBINE_HASH(Hasher.hash(Template->getCanonicalDecl())); + } + + // FIXME: Hash dependent template names. + return Hash; +} + +static hash_code computeHash(const TemplateArgument &Arg, + IndexRecordHasher &Hasher) { + hash_code Hash = INITIAL_HASH; + + switch (Arg.getKind()) { + case TemplateArgument::Null: + break; + + case TemplateArgument::Declaration: + COMBINE_HASH(Hasher.hash(Arg.getAsDecl())); + break; + + case TemplateArgument::NullPtr: + break; + + case TemplateArgument::TemplateExpansion: + COMBINE_HASH('P'); // pack expansion of... + // Fall through + case TemplateArgument::Template: + COMBINE_HASH(computeHash(Arg.getAsTemplateOrTemplatePattern(), Hasher)); + break; + + case TemplateArgument::Expression: + // FIXME: Hash expressions. + break; + + case TemplateArgument::Pack: + COMBINE_HASH('p'); + for (const auto &P : Arg.pack_elements()) + COMBINE_HASH(computeHash(P, Hasher)); + break; + + case TemplateArgument::Type: + COMBINE_HASH(Hasher.hash(Arg.getAsType())); + break; + + case TemplateArgument::Integral: + COMBINE_HASH('V', Hasher.hash(Arg.getIntegralType()), Arg.getAsIntegral()); + break; + } + + return Hash; +} + +hash_code IndexRecordHasher::hashImpl(CanQualType CQT) { + hash_code Hash = INITIAL_HASH; + + auto asCanon = [](QualType Ty) -> CanQualType { + return CanQualType::CreateUnsafe(Ty); + }; + + const Type *T = CQT.getTypePtr(); + + if (const PackExpansionType *Expansion = dyn_cast(T)) { + return COMBINE_HASH('P', hash(asCanon(Expansion->getPattern()))); + } + if (const RValueReferenceType *RT = dyn_cast(T)) { + return COMBINE_HASH('%', hash(asCanon(RT->getPointeeType()))); + } + if (const FunctionProtoType *FT = dyn_cast(T)) { + COMBINE_HASH('F', hash(asCanon(FT->getReturnType()))); + for (const auto &I : FT->param_types()) + COMBINE_HASH(hash(asCanon(I))); + return COMBINE_HASH(FT->isVariadic()); + } + if (const ComplexType *CT = dyn_cast(T)) { + return COMBINE_HASH('<', hash(asCanon(CT->getElementType()))); + } + if (const TemplateSpecializationType *Spec + = dyn_cast(T)) { + COMBINE_HASH('>', computeHash(Spec->getTemplateName(), *this)); + for (unsigned I = 0, N = Spec->getNumArgs(); I != N; ++I) + COMBINE_HASH(computeHash(Spec->getArg(I), *this)); + return Hash; + } + if (const DependentNameType *DNT = dyn_cast(T)) { + COMBINE_HASH('^'); + if (const NestedNameSpecifier *NNS = DNT->getQualifier()) + COMBINE_HASH(hash(NNS)); + return COMBINE_HASH(computeHash(DNT->getIdentifier())); + } + + // Unhandled type. + return Hash; +} + +hash_code IndexRecordHasher::hashImpl(DeclarationName Name) { + hash_code Hash = INITIAL_HASH; + COMBINE_HASH(Name.getNameKind()); + + switch (Name.getNameKind()) { + case DeclarationName::Identifier: + COMBINE_HASH(computeHash(Name.getAsIdentifierInfo())); + break; + case DeclarationName::ObjCZeroArgSelector: + case DeclarationName::ObjCOneArgSelector: + case DeclarationName::ObjCMultiArgSelector: + COMBINE_HASH(computeHash(Name.getObjCSelector())); + break; + case DeclarationName::CXXConstructorName: + case DeclarationName::CXXDestructorName: + case DeclarationName::CXXConversionFunctionName: + break; + case DeclarationName::CXXOperatorName: + COMBINE_HASH(Name.getCXXOverloadedOperator()); + break; + case DeclarationName::CXXLiteralOperatorName: + COMBINE_HASH(computeHash(Name.getCXXLiteralIdentifier())); + case DeclarationName::CXXUsingDirective: + case DeclarationName::CXXDeductionGuideName: + break; + } + + return Hash; +} + +hash_code IndexRecordHasher::hashImpl(const NestedNameSpecifier *NNS) { + hash_code Hash = INITIAL_HASH; + if (auto *Pre = NNS->getPrefix()) + COMBINE_HASH(hash(Pre)); + + COMBINE_HASH(NNS->getKind()); + + switch (NNS->getKind()) { + case NestedNameSpecifier::Identifier: + COMBINE_HASH(computeHash(NNS->getAsIdentifier())); + break; + + case NestedNameSpecifier::Namespace: + COMBINE_HASH(hash(NNS->getAsNamespace()->getCanonicalDecl())); + break; + + case NestedNameSpecifier::NamespaceAlias: + COMBINE_HASH(hash(NNS->getAsNamespaceAlias()->getCanonicalDecl())); + break; + + case NestedNameSpecifier::Global: + break; + + case NestedNameSpecifier::Super: + break; + + case NestedNameSpecifier::TypeSpecWithTemplate: + // Fall through to hash the type. + + case NestedNameSpecifier::TypeSpec: + COMBINE_HASH(hash(QualType(NNS->getAsType(), 0))); + break; + } + + return Hash; +} Index: lib/Index/IndexRecordReader.cpp =================================================================== --- /dev/null +++ lib/Index/IndexRecordReader.cpp @@ -0,0 +1,407 @@ +//===--- IndexRecordReader.cpp - Index record deserialization -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexRecordReader.h" +#include "IndexDataStoreUtils.h" +#include "BitstreamVisitor.h" +#include "clang/Index/IndexDataStoreSymbolUtils.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + +struct IndexRecordReader::Implementation { + BumpPtrAllocator Allocator; + std::unique_ptr Buffer; + llvm::BitstreamCursor DeclCursor; + llvm::BitstreamCursor OccurCursor; + ArrayRef DeclOffsets; + const IndexRecordDecl **Decls; + + void setDeclOffsets(ArrayRef Offs) { + DeclOffsets = Offs; + Decls = Allocator.Allocate(Offs.size()); + memset(Decls, 0, sizeof(IndexRecordDecl*)*Offs.size()); + } + + unsigned getNumDecls() const { return DeclOffsets.size(); } + + const IndexRecordDecl *getDeclByID(unsigned DeclID) { + if (DeclID == 0) + return nullptr; + return getDecl(DeclID-1); + } + + const IndexRecordDecl *getDecl(unsigned Index) { + assert(Index < getNumDecls()); + if (const IndexRecordDecl *D = Decls[Index]) + return D; + + IndexRecordDecl *D = Allocator.Allocate(); + readDecl(Index, *D); + Decls[Index] = D; + return D; + } + + /// Goes through the decls and populates a vector of record decls, based on + /// what the given function returns. + /// + /// The advantage of this function is to allocate memory only for the record + /// decls that the caller is interested in. + bool searchDecls(llvm::function_ref Checker, + llvm::function_ref Receiver) { + for (unsigned I = 0, E = getNumDecls(); I != E; ++I) { + if (const IndexRecordDecl *D = Decls[I]) { + DeclSearchReturn Ret = Checker(*D); + if (Ret.AcceptDecl) + Receiver(D); + if (!Ret.ContinueSearch) + return false; + continue; + } + + IndexRecordDecl LocalD; + readDecl(I, LocalD); + DeclSearchReturn Ret = Checker(LocalD); + if (Ret.AcceptDecl) { + IndexRecordDecl *D = Allocator.Allocate(); + *D = LocalD; + Decls[I] = D; + Receiver(D); + } + if (!Ret.ContinueSearch) + return false; + } + return true; + } + + void readDecl(unsigned Index, IndexRecordDecl &RecD) { + RecordData Record; + StringRef Blob; + DeclCursor.JumpToBit(DeclOffsets[Index]); + unsigned Code = DeclCursor.ReadCode(); + unsigned RecID = DeclCursor.readRecord(Code, Record, &Blob); + assert(RecID == REC_DECLINFO); + (void)RecID; + + unsigned I = 0; + RecD.DeclID = Index+1; + RecD.SymInfo.Kind = getSymbolKind((indexstore_symbol_kind_t)read(Record, I)); + RecD.SymInfo.SubKind = getSymbolSubKind((indexstore_symbol_subkind_t)read(Record, I)); + RecD.SymInfo.Lang = + getSymbolLanguage((indexstore_symbol_language_t)read(Record, I)); + RecD.SymInfo.Properties = getSymbolProperties(read(Record, I)); + RecD.Roles = getSymbolRoles(read(Record, I)); + RecD.RelatedRoles = getSymbolRoles(read(Record, I)); + size_t NameLen = read(Record, I); + size_t USRLen = read(Record, I); + RecD.Name = Blob.substr(0, NameLen); + RecD.USR = Blob.substr(NameLen, USRLen); + RecD.CodeGenName = Blob.substr(NameLen+USRLen); + } + + /// Reads occurrence data. + /// \param DeclsFilter if non-empty indicates the list of decls that we want + /// to get occurrences for. If empty then indicates that we want occurrences + /// for all decls. + /// \param RelatedDeclsFilter Same as \c DeclsFilter but for related decls. + /// \returns true if the occurrence info was filled out, false if occurrence + /// was ignored. + bool readOccurrence(RecordDataImpl &Record, StringRef Blob, + ArrayRef DeclsFilter, + ArrayRef RelatedDeclsFilter, + IndexRecordOccurrence &RecOccur) { + + auto isDeclIDContained = [](unsigned DeclID, + ArrayRef Ds) -> bool { + if (Ds.empty()) + return true; // empty means accept all. + auto pred = [DeclID](const IndexRecordDecl *D) { return D->DeclID == DeclID; }; + return std::find_if(Ds.begin(), Ds.end(), pred) != Ds.end(); + }; + + unsigned I = 0; + unsigned DeclID = read(Record, I); + if (!isDeclIDContained(DeclID, DeclsFilter)) + return false; + + if (!RelatedDeclsFilter.empty()) { + unsigned RelI = I+3; + unsigned NumRelated = read(Record, RelI); + bool FoundRelated = false; + while (NumRelated--) { + ++RelI; // roles; + unsigned RelDID = read(Record, RelI); + if (isDeclIDContained(RelDID, RelatedDeclsFilter)) { + FoundRelated = true; + break; + } + } + if (!FoundRelated) + return false; + } + + RecOccur.Dcl = getDeclByID(DeclID); + RecOccur.Roles = getSymbolRoles(read(Record, I)); + RecOccur.Line = read(Record, I); + RecOccur.Column = read(Record, I); + + unsigned NumRelated = read(Record, I); + while (NumRelated--) { + SymbolRoleSet RelRoles = getSymbolRoles(read(Record, I)); + const IndexRecordDecl *RelD = getDeclByID(read(Record, I)); + RecOccur.Relations.emplace_back(RelRoles, RelD); + } + + return true; + } + + bool foreachDecl(bool NoCache, + function_ref Receiver) { + for (unsigned I = 0, E = getNumDecls(); I != E; ++I) { + if (const IndexRecordDecl *D = Decls[I]) { + if (!Receiver(D)) + return false; + continue; + } + + if (NoCache) { + IndexRecordDecl LocalD; + readDecl(I, LocalD); + if (!Receiver(&LocalD)) + return false; + } else { + if (!Receiver(getDecl(I))) + return false; + } + } + return true; + } + + bool foreachOccurrence(ArrayRef DeclsFilter, + ArrayRef RelatedDeclsFilter, + function_ref Receiver) { + class OccurBitVisitor : public BitstreamVisitor { + IndexRecordReader::Implementation &Reader; + ArrayRef DeclsFilter; + ArrayRef RelatedDeclsFilter; + function_ref Receiver; + + public: + OccurBitVisitor(llvm::BitstreamCursor &Stream, + IndexRecordReader::Implementation &Reader, + ArrayRef DeclsFilter, + ArrayRef RelatedDeclsFilter, + function_ref Receiver) + : BitstreamVisitor(Stream), + Reader(Reader), + DeclsFilter(DeclsFilter), + RelatedDeclsFilter(RelatedDeclsFilter), + Receiver(std::move(Receiver)) {} + + StreamVisit visitRecord(unsigned BlockID, unsigned RecID, + RecordDataImpl &Record, StringRef Blob) { + assert(RecID == REC_DECLOCCURRENCE); + IndexRecordOccurrence RecOccur; + if (Reader.readOccurrence(Record, Blob, DeclsFilter, RelatedDeclsFilter, + RecOccur)) + if (!Receiver(RecOccur)) + return StreamVisit::Abort; + return StreamVisit::Continue; + } + }; + + SavedStreamPosition SavedPosition(OccurCursor); + OccurBitVisitor Visitor(OccurCursor, *this, DeclsFilter, RelatedDeclsFilter, + Receiver); + std::string Error; + return Visitor.visit(Error); + } + + bool foreachOccurrenceInLineRange(unsigned lineStart, unsigned lineCount, + llvm::function_ref receiver) { + // FIXME: Use binary search and make this more efficient. + unsigned lineEnd = lineStart+lineCount; + return foreachOccurrence(None, None, [&](const IndexRecordOccurrence &occur) -> bool { + if (occur.Line > lineEnd) + return false; // we're done. + if (occur.Line >= lineStart) { + if (!receiver(occur)) + return false; + } + return true; + }); + } + + static uint64_t read(RecordDataImpl &Record, unsigned &I) { + return Record[I++]; + } +}; + +namespace { + +class IndexBitstreamVisitor : public BitstreamVisitor { + IndexRecordReader::Implementation &Reader; + +public: + IndexBitstreamVisitor(llvm::BitstreamCursor &Stream, + IndexRecordReader::Implementation &Reader) + : BitstreamVisitor(Stream), Reader(Reader) {} + + StreamVisit visitBlock(unsigned ID) { + switch ((RecordBitBlock)ID) { + case REC_VERSION_BLOCK_ID: + case REC_DECLOFFSETS_BLOCK_ID: + return StreamVisit::Continue; + + case REC_DECLS_BLOCK_ID: + Reader.DeclCursor = Stream; + if (Reader.DeclCursor.EnterSubBlock(ID)) { + *Error = "malformed block record"; + return StreamVisit::Abort; + } + readBlockAbbrevs(Reader.DeclCursor); + return StreamVisit::Skip; + + case REC_DECLOCCURRENCES_BLOCK_ID: + Reader.OccurCursor = Stream; + if (Reader.OccurCursor.EnterSubBlock(ID)) { + *Error = "malformed block record"; + return StreamVisit::Abort; + } + readBlockAbbrevs(Reader.OccurCursor); + return StreamVisit::Skip; + } + + // Some newly introduced block in a minor version update that we cannot + // handle. + return StreamVisit::Skip; + } + + StreamVisit visitRecord(unsigned BlockID, unsigned RecID, + RecordDataImpl &Record, StringRef Blob) { + switch (BlockID) { + case REC_VERSION_BLOCK_ID: { + unsigned StoreFormatVersion = Record[0]; + if (StoreFormatVersion != STORE_FORMAT_VERSION) { + llvm::raw_string_ostream OS(*Error); + OS << "Store format version mismatch: " << StoreFormatVersion; + OS << " , expected: " << STORE_FORMAT_VERSION; + return StreamVisit::Abort; + } + break; + } + case REC_DECLOFFSETS_BLOCK_ID: + assert(RecID == REC_DECLOFFSETS); + Reader.setDeclOffsets(makeArrayRef((uint32_t*)Blob.data(), Record[0])); + break; + + case REC_DECLS_BLOCK_ID: + case REC_DECLOCCURRENCES_BLOCK_ID: + llvm_unreachable("shouldn't visit this block'"); + } + return StreamVisit::Continue; + } +}; + +} // anonymous namespace + +std::unique_ptr +IndexRecordReader::createWithRecordFilename(StringRef RecordFilename, + StringRef StorePath, + std::string &Error) { + SmallString<128> PathBuf = StorePath; + appendRecordSubDir(PathBuf); + appendInteriorRecordPath(RecordFilename, PathBuf); + return createWithFilePath(PathBuf.str(), Error); +} + +std::unique_ptr +IndexRecordReader::createWithFilePath(StringRef FilePath, std::string &Error) { + auto ErrOrBuf = MemoryBuffer::getFile(FilePath, /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + if (!ErrOrBuf) { + raw_string_ostream(Error) << "failed opening index record '" + << FilePath << "': " << ErrOrBuf.getError().message(); + return nullptr; + } + return createWithBuffer(std::move(*ErrOrBuf), Error); +} + +std::unique_ptr +IndexRecordReader::createWithBuffer(std::unique_ptr Buffer, + std::string &Error) { + + std::unique_ptr Reader; + Reader.reset(new IndexRecordReader()); + auto &Impl = Reader->Impl; + Impl.Buffer = std::move(Buffer); + llvm::BitstreamCursor Stream(*Impl.Buffer); + + // Sniff for the signature. + if (Stream.Read(8) != 'I' || + Stream.Read(8) != 'D' || + Stream.Read(8) != 'X' || + Stream.Read(8) != 'R') { + Error = "not a serialized index record file"; + return nullptr; + } + + IndexBitstreamVisitor BitVisitor(Stream, Impl); + if (!BitVisitor.visit(Error)) + return nullptr; + + return Reader; +} + +IndexRecordReader::IndexRecordReader() + : Impl(*new Implementation()) { + +} + +IndexRecordReader::~IndexRecordReader() { + delete &Impl; +} + +bool IndexRecordReader::searchDecls( + llvm::function_ref Checker, + llvm::function_ref Receiver) { + return Impl.searchDecls(std::move(Checker), std::move(Receiver)); +} + +bool IndexRecordReader::foreachDecl(bool NoCache, + function_ref Receiver) { + return Impl.foreachDecl(NoCache, std::move(Receiver)); +} + +bool IndexRecordReader::foreachOccurrence( + ArrayRef DeclsFilter, + ArrayRef RelatedDeclsFilter, + function_ref Receiver) { + return Impl.foreachOccurrence(DeclsFilter, RelatedDeclsFilter, + std::move(Receiver)); +} + +bool IndexRecordReader::foreachOccurrence( + llvm::function_ref Receiver) { + return foreachOccurrence(None, None, std::move(Receiver)); +} + +bool IndexRecordReader::foreachOccurrenceInLineRange(unsigned lineStart, + unsigned lineCount, + llvm::function_ref Receiver) { + return Impl.foreachOccurrenceInLineRange(lineStart, lineCount, Receiver); +} Index: lib/Index/IndexRecordWriter.cpp =================================================================== --- /dev/null +++ lib/Index/IndexRecordWriter.cpp @@ -0,0 +1,366 @@ +//===--- IndexRecordWriter.cpp - Index record serialization ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexRecordWriter.h" +#include "IndexDataStoreUtils.h" +#include "indexstore/indexstore.h" +#include "clang/Index/IndexDataStoreSymbolUtils.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + +using writer::OpaqueDecl; + +namespace { +struct DeclInfo { + OpaqueDecl D; + SymbolRoleSet Roles; + SymbolRoleSet RelatedRoles; +}; + +struct OccurrenceInfo { + unsigned DeclID; + OpaqueDecl D; + SymbolRoleSet Roles; + unsigned Line; + unsigned Column; + SmallVector, 4> Related; +}; + +struct RecordState { + std::string RecordPath; + SmallString<512> Buffer; + BitstreamWriter Stream; + + DenseMap IndexForDecl; + std::vector Decls; + std::vector Occurrences; + + RecordState(std::string &&RecordPath) + : RecordPath(std::move(RecordPath)), Stream(Buffer) {} +}; +} // end anonymous namespace + +static void writeBlockInfo(BitstreamWriter &Stream) { + RecordData Record; + + Stream.EnterBlockInfoBlock(); +#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) +#define RECORD(X) emitRecordID(X, #X, Stream, Record) + + BLOCK(REC_VERSION_BLOCK); + RECORD(REC_VERSION); + + BLOCK(REC_DECLS_BLOCK); + RECORD(REC_DECLINFO); + + BLOCK(REC_DECLOFFSETS_BLOCK); + RECORD(REC_DECLOFFSETS); + + BLOCK(REC_DECLOCCURRENCES_BLOCK); + RECORD(REC_DECLOCCURRENCE); + +#undef RECORD +#undef BLOCK + Stream.ExitBlock(); +} + +static void writeVersionInfo(BitstreamWriter &Stream) { + using namespace llvm::sys; + + Stream.EnterSubblock(REC_VERSION_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(REC_VERSION)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Store format version + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + RecordData Record; + Record.push_back(REC_VERSION); + Record.push_back(STORE_FORMAT_VERSION); + Stream.EmitRecordWithAbbrev(AbbrevCode, Record); + + Stream.ExitBlock(); +} + +template +static StringRef data(const std::vector &v) { + if (v.empty()) + return StringRef(); + return StringRef(reinterpret_cast(&v[0]), sizeof(T) * v.size()); +} + +template static StringRef data(const SmallVectorImpl &v) { + return StringRef(reinterpret_cast(v.data()), + sizeof(T) * v.size()); +} + +static void writeDecls(BitstreamWriter &Stream, ArrayRef Decls, + ArrayRef Occurrences, + writer::SymbolWriterCallback GetSymbolForDecl) { + SmallVector DeclOffsets; + DeclOffsets.reserve(Decls.size()); + + //===--------------------------------------------------------------------===// + // DECLS_BLOCK_ID + //===--------------------------------------------------------------------===// + + Stream.EnterSubblock(REC_DECLS_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(REC_DECLINFO)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // Kind + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // SubKind + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // Language + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolPropertyBitNum)); // Properties + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Roles + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Related Roles + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Length of name in block + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Length of USR in block + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name + USR + CodeGen symbol name + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + +#ifndef NDEBUG + StringSet<> USRSet; +#endif + + RecordData Record; + llvm::SmallString<256> Blob; + llvm::SmallString<256> Scratch; + for (auto &Info : Decls) { + DeclOffsets.push_back(Stream.GetCurrentBitNo()); + Blob.clear(); + Scratch.clear(); + + writer::Symbol SymInfo = GetSymbolForDecl(Info.D, Scratch); + assert(SymInfo.SymInfo.Kind != SymbolKind::Unknown); + assert(!SymInfo.USR.empty() && "Recorded decl without USR!"); + + Blob += SymInfo.Name; + Blob += SymInfo.USR; + Blob += SymInfo.CodeGenName; + +#ifndef NDEBUG + bool IsNew = USRSet.insert(SymInfo.USR).second; + if (!IsNew) { + llvm::errs() << "Index: Duplicate USR! " << SymInfo.USR << "\n"; + // FIXME: print more information so it's easier to find the declaration. + } +#endif + + Record.clear(); + Record.push_back(REC_DECLINFO); + Record.push_back(getIndexStoreKind(SymInfo.SymInfo.Kind)); + Record.push_back(getIndexStoreSubKind(SymInfo.SymInfo.SubKind)); + Record.push_back(getIndexStoreLang(SymInfo.SymInfo.Lang)); + Record.push_back(getIndexStoreProperties(SymInfo.SymInfo.Properties)); + Record.push_back(getIndexStoreRoles(Info.Roles)); + Record.push_back(getIndexStoreRoles(Info.RelatedRoles)); + Record.push_back(SymInfo.Name.size()); + Record.push_back(SymInfo.USR.size()); + Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); + } + + Stream.ExitBlock(); + + //===--------------------------------------------------------------------===// + // DECLOFFSETS_BLOCK_ID + //===--------------------------------------------------------------------===// + + Stream.EnterSubblock(REC_DECLOFFSETS_BLOCK_ID, 3); + + Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(REC_DECLOFFSETS)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of Decls + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Offsets array + AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + Record.clear(); + Record.push_back(REC_DECLOFFSETS); + Record.push_back(DeclOffsets.size()); + Stream.EmitRecordWithBlob(AbbrevCode, Record, data(DeclOffsets)); + + Stream.ExitBlock(); + + //===--------------------------------------------------------------------===// + // DECLOCCURRENCES_BLOCK_ID + //===--------------------------------------------------------------------===// + + Stream.EnterSubblock(REC_DECLOCCURRENCES_BLOCK_ID, 3); + + Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(REC_DECLOCCURRENCE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Decl ID + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, SymbolRoleBitNum)); // Roles + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Line + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Column + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Num related + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // Related Roles/IDs + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // Roles or ID + AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + for (auto &Occur : Occurrences) { + Record.clear(); + Record.push_back(REC_DECLOCCURRENCE); + Record.push_back(Occur.DeclID); + Record.push_back(getIndexStoreRoles(Occur.Roles)); + Record.push_back(Occur.Line); + Record.push_back(Occur.Column); + Record.push_back(Occur.Related.size()); + for (auto &Rel : Occur.Related) { + Record.push_back(getIndexStoreRoles(Rel.first.Roles)); + Record.push_back(Rel.second); + } + Stream.EmitRecordWithAbbrev(AbbrevCode, Record); + } + Stream.ExitBlock(); +} + +IndexRecordWriter::IndexRecordWriter(StringRef IndexPath) + : RecordsPath(IndexPath) { + store::appendRecordSubDir(RecordsPath); +} + +IndexRecordWriter::Result +IndexRecordWriter::beginRecord(StringRef Filename, hash_code RecordHash, + std::string &Error, std::string *OutRecordFile) { + using namespace llvm::sys; + assert(!Record && "called beginRecord before calling endRecord on previous"); + + std::string RecordName; + { + llvm::raw_string_ostream RN(RecordName); + RN << path::filename(Filename); + RN << "-" << APInt(64, RecordHash).toString(36, /*Signed=*/false); + } + SmallString<256> RecordPath = RecordsPath.str(); + appendInteriorRecordPath(RecordName, RecordPath); + + if (OutRecordFile) + *OutRecordFile = RecordName; + + if (std::error_code EC = + fs::access(RecordPath.c_str(), fs::AccessMode::Exist)) { + if (EC != errc::no_such_file_or_directory) { + llvm::raw_string_ostream Err(Error); + Err << "could not access record '" << RecordPath + << "': " << EC.message(); + return Result::Failure; + } + } else { + return Result::AlreadyExists; + } + + // Write the record header. + auto *State = new RecordState(RecordPath.str()); + Record = State; + llvm::BitstreamWriter &Stream = State->Stream; + Stream.Emit('I', 8); + Stream.Emit('D', 8); + Stream.Emit('X', 8); + Stream.Emit('R', 8); + + writeBlockInfo(Stream); + writeVersionInfo(Stream); + + return Result::Success; +} + +IndexRecordWriter::Result +IndexRecordWriter::endRecord(std::string &Error, + writer::SymbolWriterCallback GetSymbolForDecl) { + assert(Record && "called endRecord without calling beginRecord"); + auto &State = *static_cast(Record); + Record = nullptr; + struct ScopedDelete { + RecordState *S; + ScopedDelete(RecordState *S) : S(S) {} + ~ScopedDelete() { delete S; } + } Deleter(&State); + + if (!State.Decls.empty()) { + writeDecls(State.Stream, State.Decls, State.Occurrences, GetSymbolForDecl); + } + + if (std::error_code EC = sys::fs::create_directory(sys::path::parent_path(State.RecordPath))) { + llvm::raw_string_ostream Err(Error); + Err << "failed to create directory '" << sys::path::parent_path(State.RecordPath) << "': " << EC.message(); + return Result::Failure; + } + + // Create a unique file to write to so that we can move the result into place + // atomically. If this process crashes we don't want to interfere with any + // other concurrent processes. + SmallString<128> TempPath(State.RecordPath); + TempPath += "-temp-%%%%%%%%"; + int TempFD; + if (sys::fs::createUniqueFile(TempPath.str(), TempFD, TempPath)) { + llvm::raw_string_ostream Err(Error); + Err << "failed to create temporary file: " << TempPath; + return Result::Failure; + } + + raw_fd_ostream OS(TempFD, /*shouldClose=*/true); + OS.write(State.Buffer.data(), State.Buffer.size()); + OS.close(); + + // Atomically move the unique file into place. + if (std::error_code EC = + sys::fs::rename(TempPath.c_str(), State.RecordPath.c_str())) { + llvm::raw_string_ostream Err(Error); + Err << "failed to rename '" << TempPath << "' to '" << State.RecordPath << "': " << EC.message(); + return Result::Failure; + } + + return Result::Success; +} + +void IndexRecordWriter::addOccurrence( + OpaqueDecl D, SymbolRoleSet Roles, unsigned Line, unsigned Column, + ArrayRef Related) { + assert(Record && "called addOccurrence without calling beginRecord"); + auto &State = *static_cast(Record); + + auto insertDecl = [&](OpaqueDecl D, SymbolRoleSet Roles, + SymbolRoleSet RelatedRoles) -> unsigned { + auto Insert = + State.IndexForDecl.insert(std::make_pair(D, State.Decls.size())); + unsigned Index = Insert.first->second; + + if (Insert.second) { + State.Decls.push_back(DeclInfo{D, Roles, RelatedRoles}); + } else { + State.Decls[Index].Roles |= Roles; + State.Decls[Index].RelatedRoles |= RelatedRoles; + } + return Index + 1; + }; + + unsigned DeclID = insertDecl(D, Roles, SymbolRoleSet()); + + decltype(OccurrenceInfo::Related) RelatedDecls; + RelatedDecls.reserve(Related.size()); + for (auto &Rel : Related) { + unsigned ID = insertDecl(Rel.RelatedSymbol, SymbolRoleSet(), Rel.Roles); + RelatedDecls.emplace_back(Rel, ID); + } + + State.Occurrences.push_back( + OccurrenceInfo{DeclID, D, Roles, Line, Column, std::move(RelatedDecls)}); +} Index: lib/Index/IndexUnitReader.cpp =================================================================== --- /dev/null +++ lib/Index/IndexUnitReader.cpp @@ -0,0 +1,516 @@ +//===--- IndexUnitReader.cpp - Index unit deserialization -----------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexUnitReader.h" +#include "IndexDataStoreUtils.h" +#include "BitstreamVisitor.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Support/Chrono.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +#include + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + +namespace { + +typedef function_ref DependencyReceiver; +typedef function_ref IncludeReceiver; + +class IndexUnitReaderImpl { + sys::TimePoint<> ModTime; + std::unique_ptr MemBuf; + +public: + StringRef ProviderIdentifier; + StringRef ProviderVersion; + llvm::BitstreamCursor DependCursor; + llvm::BitstreamCursor IncludeCursor; + bool IsSystemUnit; + bool IsModuleUnit; + bool IsDebugCompilation; + StringRef WorkingDir; + StringRef OutputFile; + StringRef SysrootPath; + StringRef ModuleName; + SmallString<128> MainFilePath; + StringRef Target; + std::vector Paths; + StringRef PathsBuffer; + + struct ModuleInfo { + unsigned NameOffset; + unsigned NameSize; + }; + std::vector Modules; + StringRef ModuleNamesBuffer; + + bool init(std::unique_ptr Buf, sys::TimePoint<> ModTime, + std::string &Error); + + StringRef getProviderIdentifier() const { return ProviderIdentifier; } + StringRef getProviderVersion() const { return ProviderVersion; } + + sys::TimePoint<> getModificationTime() const { return ModTime; } + StringRef getWorkingDirectory() const { return WorkingDir; } + StringRef getOutputFile() const { return OutputFile; } + StringRef getSysrootPath() const { return SysrootPath; } + StringRef getTarget() const { return Target; } + + StringRef getModuleName() const { return ModuleName; } + StringRef getMainFilePath() const { return MainFilePath.str(); } + bool hasMainFile() const { return !MainFilePath.empty(); } + bool isSystemUnit() const { return IsSystemUnit; } + bool isModuleUnit() const { return IsModuleUnit; } + bool isDebugCompilation() const { return IsDebugCompilation; } + + /// Unit dependencies are provided ahead of record ones, record ones + /// ahead of the file ones. + bool foreachDependency(DependencyReceiver Receiver); + + bool foreachInclude(IncludeReceiver Receiver); + + StringRef getPathFromBuffer(size_t Offset, size_t Size) { + return PathsBuffer.substr(Offset, Size); + } + + void constructFilePath(SmallVectorImpl &Path, int PathIndex); + + StringRef getModuleName(int ModuleIndex); +}; + +class IndexUnitBitstreamVisitor : public BitstreamVisitor { + IndexUnitReaderImpl &Reader; + size_t WorkDirOffset; + size_t WorkDirSize; + size_t OutputFileOffset; + size_t OutputFileSize; + size_t SysrootOffset; + size_t SysrootSize; + int MainPathIndex; + +public: + IndexUnitBitstreamVisitor(llvm::BitstreamCursor &Stream, + IndexUnitReaderImpl &Reader) + : BitstreamVisitor(Stream), Reader(Reader) {} + + StreamVisit visitBlock(unsigned ID) { + switch ((UnitBitBlock)ID) { + case UNIT_VERSION_BLOCK_ID: + case UNIT_INFO_BLOCK_ID: + case UNIT_PATHS_BLOCK_ID: + case UNIT_MODULES_BLOCK_ID: + return StreamVisit::Continue; + + case UNIT_DEPENDENCIES_BLOCK_ID: + Reader.DependCursor = Stream; + if (Reader.DependCursor.EnterSubBlock(ID)) { + *Error = "malformed unit dependencies block record"; + return StreamVisit::Abort; + } + readBlockAbbrevs(Reader.DependCursor); + return StreamVisit::Skip; + case UNIT_INCLUDES_BLOCK_ID: + Reader.IncludeCursor = Stream; + if (Reader.IncludeCursor.EnterSubBlock(ID)) { + *Error = "malformed unit includes block record"; + return StreamVisit::Abort; + } + readBlockAbbrevs(Reader.IncludeCursor); + return StreamVisit::Skip; + } + + // Some newly introduced block in a minor version update that we cannot + // handle. + return StreamVisit::Skip; + } + + StreamVisit visitRecord(unsigned BlockID, unsigned RecID, + RecordDataImpl &Record, StringRef Blob) { + switch (BlockID) { + case UNIT_VERSION_BLOCK_ID: { + unsigned StoreFormatVersion = Record[0]; + if (StoreFormatVersion != STORE_FORMAT_VERSION) { + llvm::raw_string_ostream OS(*Error); + OS << "Store format version mismatch: " << StoreFormatVersion; + OS << " , expected: " << STORE_FORMAT_VERSION; + return StreamVisit::Abort; + } + break; + } + + case UNIT_INFO_BLOCK_ID: { + assert(RecID == UNIT_INFO); + unsigned I = 0; + Reader.IsSystemUnit = Record[I++]; + + // Save these to lookup them up after we get the paths buffer. + WorkDirOffset = Record[I++]; + WorkDirSize = Record[I++]; + OutputFileOffset = Record[I++]; + OutputFileSize = Record[I++]; + SysrootOffset = Record[I++]; + SysrootSize = Record[I++]; + MainPathIndex = (int)Record[I++] - 1; + Reader.IsDebugCompilation = Record[I++]; + Reader.IsModuleUnit = Record[I++]; + + size_t moduleNameSize = Record[I++]; + size_t providerIdentifierSize = Record[I++]; + size_t providerVersionSize = Record[I++]; + I++; // Reserved for ProviderDataVersion. + Reader.ModuleName = Blob.substr(0, moduleNameSize); + Blob = Blob.drop_front(moduleNameSize); + Reader.ProviderIdentifier = Blob.substr(0, providerIdentifierSize); + Blob = Blob.drop_front(providerIdentifierSize); + Reader.ProviderVersion = Blob.substr(0, providerVersionSize); + Reader.Target = Blob.drop_front(providerVersionSize); + break; + } + + case UNIT_PATHS_BLOCK_ID: + switch (RecID) { + case UNIT_PATH: + { + unsigned I = 0; + UnitFilePathPrefixKind Kind = (UnitFilePathPrefixKind)Record[I++]; + size_t DirOffset = Record[I++]; + size_t DirSize = Record[I++]; + size_t FilenameOffset = Record[I++]; + size_t FilenameSize = Record[I++]; + + Reader.Paths.emplace_back(Kind, BitPathComponent(DirOffset, DirSize), + BitPathComponent(FilenameOffset, FilenameSize)); + } + break; + case UNIT_PATH_BUFFER: + Reader.PathsBuffer = Blob; + Reader.WorkingDir = Reader.getPathFromBuffer(WorkDirOffset, WorkDirSize); + Reader.OutputFile = Reader.getPathFromBuffer(OutputFileOffset, OutputFileSize); + Reader.SysrootPath = Reader.getPathFromBuffer(SysrootOffset, SysrootSize); + + // now we can populate the main file's path + Reader.constructFilePath(Reader.MainFilePath, MainPathIndex); + break; + default: + llvm_unreachable("shouldn't visit this record"); + } + break; + + case UNIT_MODULES_BLOCK_ID: + switch (RecID) { + case UNIT_MODULE: + { + unsigned I = 0; + unsigned NameOffset = Record[I++]; + unsigned NameSize = Record[I++]; + Reader.Modules.push_back({NameOffset, NameSize}); + } + break; + case UNIT_MODULE_BUFFER: + Reader.ModuleNamesBuffer = Blob; + break; + default: + llvm_unreachable("shouldn't visit this record"); + } + break; + + case UNIT_DEPENDENCIES_BLOCK_ID: + case UNIT_INCLUDES_BLOCK_ID: + llvm_unreachable("shouldn't visit this block'"); + } + return StreamVisit::Continue; + } +}; + +typedef std::function + BlockVisitorCallback; + +class IndexUnitBlockBitstreamVisitor : public BitstreamVisitor { + unsigned RecID; + BlockVisitorCallback Visit; + +public: + IndexUnitBlockBitstreamVisitor(unsigned RecID, + llvm::BitstreamCursor &BlockStream, + BlockVisitorCallback Visit) + : BitstreamVisitor(BlockStream), RecID(RecID), Visit(std::move(Visit)) {} + + StreamVisit visitRecord(unsigned BlockID, unsigned RecID, + RecordDataImpl &Record, StringRef Blob) { + if (RecID != this->RecID) + llvm_unreachable("shouldn't be called with this RecID"); + + if (Visit(Record, Blob)) + return StreamVisit::Continue; + return StreamVisit::Abort; + } +}; + +} // anonymous namespace + +bool IndexUnitReaderImpl::init(std::unique_ptr Buf, + sys::TimePoint<> ModTime, std::string &Error) { + this->ModTime = ModTime; + this->MemBuf = std::move(Buf); + llvm::BitstreamCursor Stream(*MemBuf); + + // Sniff for the signature. + if (Stream.Read(8) != 'I' || + Stream.Read(8) != 'D' || + Stream.Read(8) != 'X' || + Stream.Read(8) != 'U') { + Error = "not a serialized index unit file"; + return true; + } + + IndexUnitBitstreamVisitor BitVisitor(Stream, *this); + return !BitVisitor.visit(Error); +} + +/// Unit dependencies are provided ahead of record ones, record ones +/// ahead of the file ones. +bool IndexUnitReaderImpl::foreachDependency(DependencyReceiver Receiver) { + store::SavedStreamPosition SavedDepPosition(DependCursor); + IndexUnitBlockBitstreamVisitor Visitor(UNIT_DEPENDENCY, DependCursor, + [&](RecordDataImpl& Record, StringRef Blob) { + unsigned I = 0; + UnitDependencyKind UnitDepKind = (UnitDependencyKind)Record[I++]; + bool IsSystem = Record[I++]; + int PathIndex = (int)Record[I++] - 1; + int ModuleIndex = (int)Record[I++] - 1; + time_t ModTime = (time_t)Record[I++]; + size_t FileSize = Record[I++]; + StringRef Name = Blob; + + IndexUnitReader::DependencyKind DepKind; + switch (UnitDepKind) { + case UNIT_DEPEND_KIND_UNIT: + DepKind = IndexUnitReader::DependencyKind::Unit; break; + case UNIT_DEPEND_KIND_RECORD: + DepKind = IndexUnitReader::DependencyKind::Record; break; + case UNIT_DEPEND_KIND_FILE: + DepKind = IndexUnitReader::DependencyKind::File; break; + } + + SmallString<512> PathBuf; + this->constructFilePath(PathBuf, PathIndex); + StringRef ModuleName = this->getModuleName(ModuleIndex); + + return Receiver(IndexUnitReader::DependencyInfo{DepKind, IsSystem, Name, + PathBuf.str(), ModuleName, FileSize, ModTime}); + }); + + std::string Error; + return Visitor.visit(Error); +} + +bool IndexUnitReaderImpl::foreachInclude(IncludeReceiver Receiver) { + store::SavedStreamPosition SavedIncPosition(IncludeCursor); + IndexUnitBlockBitstreamVisitor Visitor(UNIT_INCLUDE, IncludeCursor, + [&](RecordDataImpl& Record, StringRef Blob) { + unsigned I = 0; + int SourcePathIndex = (int)Record[I++] - 1; + unsigned Line = Record[I++]; + int TargetPathIndex = (int)Record[I++] - 1; + + SmallString<512> SourceBuf, TargetBuf; + this->constructFilePath(SourceBuf, SourcePathIndex); + this->constructFilePath(TargetBuf, TargetPathIndex); + return Receiver(IndexUnitReader::IncludeInfo{SourceBuf.str(), Line, TargetBuf.str()}); + }); + + std::string Error; + return Visitor.visit(Error); +} + + +void IndexUnitReaderImpl::constructFilePath(SmallVectorImpl &PathBuf, + int PathIndex) { + + if (PathIndex < 0) return; + FileBitPath &Path = Paths[PathIndex]; + StringRef Prefix; + switch (Path.PrefixKind) { + case UNIT_PATH_PREFIX_NONE: + break; + case UNIT_PATH_PREFIX_WORKDIR: + Prefix = getWorkingDirectory(); + break; + case UNIT_PATH_PREFIX_SYSROOT: + Prefix = getSysrootPath(); + break; + } + PathBuf.append(Prefix.begin(), Prefix.end()); + sys::path::append(PathBuf, + getPathFromBuffer(Path.Dir.Offset, Path.Dir.Size), + getPathFromBuffer(Path.Filename.Offset, Path.Filename.Size)); +} + +StringRef IndexUnitReaderImpl::getModuleName(int ModuleIndex) { + if (ModuleIndex < 0) + return StringRef(); + auto &ModInfo = Modules[ModuleIndex]; + return StringRef(ModuleNamesBuffer.data()+ModInfo.NameOffset, ModInfo.NameSize); +} + + +//===----------------------------------------------------------------------===// +// IndexUnitReader +//===----------------------------------------------------------------------===// + +std::unique_ptr +IndexUnitReader::createWithUnitFilename(StringRef UnitFilename, + StringRef StorePath, + std::string &Error) { + SmallString<128> PathBuf = StorePath; + appendUnitSubDir(PathBuf); + sys::path::append(PathBuf, UnitFilename); + return createWithFilePath(PathBuf.str(), Error); +} + +std::unique_ptr +IndexUnitReader::createWithFilePath(StringRef FilePath, std::string &Error) { + int FD; + std::error_code EC = sys::fs::openFileForRead(FilePath, FD); + if (EC) { + raw_string_ostream(Error) << "Failed opening '" << FilePath << "': " + << EC.message(); + return nullptr; + } + + assert(FD != -1); + struct AutoFDClose { + int FD; + AutoFDClose(int FD) : FD(FD) {} + ~AutoFDClose() { + ::close(FD); + } + } AutoFDClose(FD); + + sys::fs::file_status FileStat; + EC = sys::fs::status(FD, FileStat); + if (EC) { + Error = EC.message(); + return nullptr; + } + + auto ErrOrBuf = MemoryBuffer::getOpenFile(FD, FilePath, /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + if (!ErrOrBuf) { + raw_string_ostream(Error) << "Failed opening '" << FilePath << "': " + << ErrOrBuf.getError().message(); + return nullptr; + } + + std::unique_ptr Impl(new IndexUnitReaderImpl()); + bool Err = Impl->init(std::move(*ErrOrBuf), FileStat.getLastModificationTime(), + Error); + if (Err) + return nullptr; + + std::unique_ptr Reader; + Reader.reset(new IndexUnitReader(Impl.release())); + return Reader; +} + +Optional> +IndexUnitReader::getModificationTimeForUnit(StringRef UnitFilename, + StringRef StorePath, + std::string &Error) { + SmallString<128> PathBuf = StorePath; + appendUnitSubDir(PathBuf); + sys::path::append(PathBuf, UnitFilename); + + sys::fs::file_status FileStat; + std::error_code EC = sys::fs::status(PathBuf.str(), FileStat); + if (EC) { + Error = EC.message(); + return None; + } + return FileStat.getLastModificationTime(); +} + +#define IMPL static_cast(Impl) + +IndexUnitReader::~IndexUnitReader() { + delete IMPL; +} + +StringRef IndexUnitReader::getProviderIdentifier() const { + return IMPL->getProviderIdentifier(); +} + +StringRef IndexUnitReader::getProviderVersion() const { + return IMPL->getProviderVersion(); +} + +llvm::sys::TimePoint<> IndexUnitReader::getModificationTime() const { + return IMPL->getModificationTime(); +} + +StringRef IndexUnitReader::getWorkingDirectory() const { + return IMPL->getWorkingDirectory(); +} + +StringRef IndexUnitReader::getOutputFile() const { + return IMPL->getOutputFile(); +} + +StringRef IndexUnitReader::getSysrootPath() const { + return IMPL->getSysrootPath(); +} + +StringRef IndexUnitReader::getMainFilePath() const { + return IMPL->getMainFilePath(); +} + +StringRef IndexUnitReader::getModuleName() const { + return IMPL->getModuleName(); +} + +StringRef IndexUnitReader::getTarget() const { + return IMPL->getTarget(); +} + +bool IndexUnitReader::hasMainFile() const { + return IMPL->hasMainFile(); +} + +bool IndexUnitReader::isSystemUnit() const { + return IMPL->isSystemUnit(); +} + +bool IndexUnitReader::isModuleUnit() const { + return IMPL->isModuleUnit(); +} + +bool IndexUnitReader::isDebugCompilation() const { + return IMPL->isDebugCompilation(); +} + +/// \c Index is the index in the \c getDependencies array. +/// Unit dependencies are provided ahead of record ones. +bool IndexUnitReader::foreachDependency(DependencyReceiver Receiver) { + return IMPL->foreachDependency(std::move(Receiver)); +} + +bool IndexUnitReader::foreachInclude(IncludeReceiver Receiver) { + return IMPL->foreachInclude(std::move(Receiver)); +} Index: lib/Index/IndexUnitWriter.cpp =================================================================== --- /dev/null +++ lib/Index/IndexUnitWriter.cpp @@ -0,0 +1,628 @@ +//===--- IndexUnitWriter.cpp - Index unit serialization -------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/Index/IndexUnitWriter.h" +#include "IndexDataStoreUtils.h" +#include "clang/Basic/FileManager.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace clang::index; +using namespace clang::index::store; +using namespace llvm; + + +class IndexUnitWriter::PathStorage { + std::string WorkDir; + std::string SysrootPath; + SmallString<512> PathsBuf; + StringMap Dirs; + std::vector FileBitPaths; + DenseMap FileToIndex; + +public: + PathStorage(StringRef workDir, StringRef sysrootPath) { + WorkDir = workDir; + if (sysrootPath == "/") + sysrootPath = StringRef(); + SysrootPath = sysrootPath; + } + + StringRef getPathsBuffer() const { return PathsBuf.str(); } + + ArrayRef getBitPaths() const { return FileBitPaths; } + + int getPathIndex(const FileEntry *FE) { + if (!FE) + return -1; + auto Pair = FileToIndex.insert(std::make_pair(FE, FileBitPaths.size())); + bool IsNew = Pair.second; + size_t Index = Pair.first->getSecond(); + + if (IsNew) { + StringRef Filename = sys::path::filename(FE->getName()); + DirBitPath Dir = getDirBitPath(sys::path::parent_path(FE->getName())); + FileBitPaths.emplace_back(Dir.PrefixKind, Dir.Dir, + BitPathComponent(getPathOffset(Filename), + Filename.size())); + } + return Index; + } + + size_t getPathOffset(StringRef Path) { + if (Path.empty()) + return 0; + size_t offset = PathsBuf.size(); + PathsBuf += Path; + return offset; + } + +private: + DirBitPath getDirBitPath(StringRef dirStr) { + auto pair = Dirs.insert(std::make_pair(dirStr, DirBitPath())); + bool isNew = pair.second; + auto &dirPath = pair.first->second; + + if (isNew) { + if (isPathInDir(SysrootPath, dirStr)) { + dirPath.PrefixKind = UNIT_PATH_PREFIX_SYSROOT; + dirStr = dirStr.drop_front(SysrootPath.size()); + while (!dirStr.empty() && dirStr[0] == '/') + dirStr = dirStr.drop_front(); + } else if (isPathInDir(WorkDir, dirStr)) { + dirPath.PrefixKind = UNIT_PATH_PREFIX_WORKDIR; + dirStr = dirStr.drop_front(WorkDir.size()); + while (!dirStr.empty() && dirStr[0] == '/') + dirStr = dirStr.drop_front(); + } + dirPath.Dir.Offset = getPathOffset(dirStr); + dirPath.Dir.Size = dirStr.size(); + } + return dirPath; + } + + static bool isPathInDir(StringRef dir, StringRef path) { + if (dir.empty() || !path.startswith(dir)) + return false; + StringRef rest = path.drop_front(dir.size()); + return !rest.empty() && sys::path::is_separator(rest.front()); + } +}; + +IndexUnitWriter::IndexUnitWriter(FileManager &FileMgr, + StringRef StorePath, + StringRef ProviderIdentifier, + StringRef ProviderVersion, + StringRef OutputFile, + StringRef ModuleName, + const FileEntry *MainFile, + bool IsSystem, + bool IsModuleUnit, + bool IsDebugCompilation, + StringRef TargetTriple, + StringRef SysrootPath, + writer::ModuleInfoWriterCallback GetInfoForModule) +: FileMgr(FileMgr) { + this->UnitsPath = StorePath; + store::appendUnitSubDir(this->UnitsPath); + this->ProviderIdentifier = ProviderIdentifier; + this->ProviderVersion = ProviderVersion; + this->OutputFile = OutputFile; + this->ModuleName = ModuleName; + this->MainFile = MainFile; + this->IsSystemUnit = IsSystem; + this->IsModuleUnit = IsModuleUnit; + this->IsDebugCompilation = IsDebugCompilation; + this->TargetTriple = TargetTriple; + this->SysrootPath = SysrootPath; + this->GetInfoForModuleFn = GetInfoForModule; +} + +IndexUnitWriter::~IndexUnitWriter() {} + +int IndexUnitWriter::addModule(writer::OpaqueModule Mod) { + if (!Mod) + return -1; + + auto Pair = IndexByModule.insert(std::make_pair(Mod, Modules.size())); + bool WasInserted = Pair.second; + if (WasInserted) { + Modules.push_back(Mod); + } + return Pair.first->second; +} + +int IndexUnitWriter::addFileDependency(const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod) { + assert(File); + auto Pair = IndexByFile.insert(std::make_pair(File, Files.size())); + bool WasInserted = Pair.second; + if (WasInserted) { + Files.push_back(FileEntryData{File, IsSystem, addModule(Mod), {}}); + } + return Pair.first->second; +} + +void IndexUnitWriter::addRecordFile(StringRef RecordFile, const FileEntry *File, + bool IsSystem, writer::OpaqueModule Mod) { + int Dep = File ? addFileDependency(File, IsSystem, /*module=*/nullptr) : -1; + Records.push_back(RecordOrUnitData{RecordFile, Dep, addModule(Mod), IsSystem}); +} + +void IndexUnitWriter::addASTFileDependency(const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod, + bool withoutUnitName) { + assert(File); + if (!SeenASTFiles.insert(File).second) + return; + + SmallString<64> UnitName; + if (!withoutUnitName) + getUnitNameForOutputFile(File->getName(), UnitName); + addUnitDependency(UnitName.str(), File, IsSystem, Mod); +} + +void IndexUnitWriter::addUnitDependency(StringRef UnitFile, + const FileEntry *File, bool IsSystem, + writer::OpaqueModule Mod) { + int Dep = File ? addFileDependency(File, IsSystem, /*module=*/nullptr) : -1; + ASTFileUnits.emplace_back(RecordOrUnitData{UnitFile, Dep, addModule(Mod), IsSystem}); +} + +bool IndexUnitWriter::addInclude(const FileEntry *Source, unsigned Line, + const FileEntry *Target) { + // FIXME: This will ignore includes of headers that resolve to module imports + // because the 'target' header has not been added as a file dependency earlier + // so it is missing from \c IndexByFile. + + auto It = IndexByFile.find(Source); + if (It == IndexByFile.end()) + return false; + int SourceIndex = It->getSecond(); + It = IndexByFile.find(Target); + if (It == IndexByFile.end()) + return false; + int TargetIndex = It->getSecond(); + Files[SourceIndex].Includes.emplace_back(FileInclude{TargetIndex, Line}); + return true; +}; + +void IndexUnitWriter::getUnitNameForOutputFile(StringRef FilePath, + SmallVectorImpl &Str) { + SmallString<256> AbsPath(FilePath); + FileMgr.makeAbsolutePath(AbsPath); + return getUnitNameForAbsoluteOutputFile(AbsPath, Str); +} + +void IndexUnitWriter::getUnitPathForOutputFile(StringRef FilePath, + SmallVectorImpl &Str) { + Str.append(UnitsPath.begin(), UnitsPath.end()); + Str.push_back('/'); + return getUnitNameForOutputFile(FilePath, Str); +} + +Optional IndexUnitWriter::isUnitUpToDateForOutputFile(StringRef FilePath, + Optional TimeCompareFilePath, + std::string &Error) { + SmallString<256> UnitPath; + getUnitPathForOutputFile(FilePath, UnitPath); + + llvm::sys::fs::file_status UnitStat; + if (std::error_code EC = llvm::sys::fs::status(UnitPath.c_str(), UnitStat)) { + if (EC != llvm::errc::no_such_file_or_directory) { + llvm::raw_string_ostream Err(Error); + Err << "could not access path '" << UnitPath + << "': " << EC.message(); + return None; + } + return false; + } + + if (!TimeCompareFilePath.hasValue()) + return true; + + llvm::sys::fs::file_status CompareStat; + if (std::error_code EC = llvm::sys::fs::status(*TimeCompareFilePath, CompareStat)) { + if (EC != llvm::errc::no_such_file_or_directory) { + llvm::raw_string_ostream Err(Error); + Err << "could not access path '" << *TimeCompareFilePath + << "': " << EC.message(); + return None; + } + return true; + } + + // Return true (unit is up-to-date) if the file to compare is older than the + // unit file. + return CompareStat.getLastModificationTime() <= UnitStat.getLastModificationTime(); +} + +void IndexUnitWriter::getUnitNameForAbsoluteOutputFile(StringRef FilePath, + SmallVectorImpl &Str) { + StringRef Fname = sys::path::filename(FilePath); + Str.append(Fname.begin(), Fname.end()); + Str.push_back('-'); + llvm::hash_code PathHashVal = llvm::hash_value(FilePath); + llvm::APInt(64, PathHashVal).toString(Str, 36, /*Signed=*/false); +} + +static void writeBlockInfo(BitstreamWriter &Stream) { + RecordData Record; + + Stream.EnterBlockInfoBlock(); +#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) +#define RECORD(X) emitRecordID(X, #X, Stream, Record) + + BLOCK(UNIT_VERSION_BLOCK); + RECORD(UNIT_VERSION); + + BLOCK(UNIT_INFO_BLOCK); + RECORD(UNIT_INFO); + + BLOCK(UNIT_DEPENDENCIES_BLOCK); + RECORD(UNIT_DEPENDENCY); + + BLOCK(UNIT_INCLUDES_BLOCK); + RECORD(UNIT_INCLUDE); + + BLOCK(UNIT_PATHS_BLOCK); + RECORD(UNIT_PATH); + RECORD(UNIT_PATH_BUFFER); + + BLOCK(UNIT_MODULES_BLOCK); + RECORD(UNIT_MODULE); + RECORD(UNIT_MODULE_BUFFER); + +#undef RECORD +#undef BLOCK + Stream.ExitBlock(); +} + +static void writeVersionInfo(BitstreamWriter &Stream) { + using namespace llvm::sys; + + Stream.EnterSubblock(UNIT_VERSION_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(UNIT_VERSION)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Store format version + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + RecordData Record; + Record.push_back(UNIT_VERSION); + Record.push_back(STORE_FORMAT_VERSION); + Stream.EmitRecordWithAbbrev(AbbrevCode, Record); + + Stream.ExitBlock(); +} + +bool IndexUnitWriter::write(std::string &Error) { + using namespace llvm::sys; + + // Determine the working directory. + SmallString<128> CWDPath; + if (!FileMgr.getFileSystemOpts().WorkingDir.empty()) { + CWDPath = FileMgr.getFileSystemOpts().WorkingDir; + if (!path::is_absolute(CWDPath)) { + fs::make_absolute(CWDPath); + } + } else { + std::error_code EC = sys::fs::current_path(CWDPath); + if (EC) { + llvm::raw_string_ostream Err(Error); + Err << "failed to determine current working directory: " << EC.message(); + return true; + } + } + WorkDir = CWDPath.str(); + + SmallString<512> Buffer; + BitstreamWriter Stream(Buffer); + Stream.Emit('I', 8); + Stream.Emit('D', 8); + Stream.Emit('X', 8); + Stream.Emit('U', 8); + + PathStorage PathStore(WorkDir, SysrootPath); + + writeBlockInfo(Stream); + writeVersionInfo(Stream); + writeUnitInfo(Stream, PathStore); + writeDependencies(Stream, PathStore); + writeIncludes(Stream, PathStore); + writePaths(Stream, PathStore); + writeModules(Stream); + + SmallString<256> UnitPath; + getUnitPathForOutputFile(OutputFile, UnitPath); + + SmallString<128> TempPath; + TempPath = path::parent_path(UnitsPath); + TempPath += '/'; + TempPath += path::filename(UnitPath); + TempPath += "-%%%%%%%%"; + int TempFD; + if (llvm::sys::fs::createUniqueFile(TempPath.str(), TempFD, TempPath)) { + llvm::raw_string_ostream Err(Error); + Err << "failed to create temporary file: " << TempPath; + return true; + } + + raw_fd_ostream OS(TempFD, /*shouldClose=*/true); + OS.write(Buffer.data(), Buffer.size()); + OS.close(); + + std::error_code EC = fs::rename(/*from=*/TempPath.c_str(), /*to=*/UnitPath.c_str()); + if (EC) { + llvm::raw_string_ostream Err(Error); + Err << "failed to rename '" << TempPath << "' to '" << UnitPath << "': " << EC.message(); + return true; + } + + return false; +} + +void IndexUnitWriter::writeUnitInfo(llvm::BitstreamWriter &Stream, + PathStorage &PathStore) { + Stream.EnterSubblock(UNIT_INFO_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(UNIT_INFO)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsSystemUnit + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // WorkDir offset + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // WorkDir size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // OutputFile offset + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // OutputFile size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // Sysroot offset + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // Sysroot size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // Main path id + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsDebugCompilation + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsModuleUnit + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // Module name size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // ProviderIdentifier size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // ProviderVersion size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 5)); // ProviderDataVersion + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Module name + ProviderIdentifier + ProviderVersion + target triple + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + RecordData Record; + Record.push_back(UNIT_INFO); + Record.push_back(IsSystemUnit); + Record.push_back(PathStore.getPathOffset(WorkDir)); + Record.push_back(WorkDir.size()); + Record.push_back(PathStore.getPathOffset(OutputFile)); + Record.push_back(OutputFile.size()); + Record.push_back(PathStore.getPathOffset(SysrootPath)); + Record.push_back(SysrootPath.size()); + Record.push_back(PathStore.getPathIndex(MainFile) + 1); // Make 1-based with 0=invalid + Record.push_back(IsDebugCompilation); + Record.push_back(IsModuleUnit); + Record.push_back(ModuleName.size()); + Record.push_back(ProviderIdentifier.size()); + Record.push_back(ProviderVersion.size()); + // ProviderDataVersion is reserved. Not sure it is a good to idea to have + // clients consider the specifics of a 'provider data version', but reserving + // to avoid store format version change in case there is a use case in the + // future. + Record.push_back(0); // ProviderDataVersion + SmallString<128> InfoStrings; + InfoStrings += ModuleName; + InfoStrings += ProviderIdentifier; + InfoStrings += ProviderVersion; + InfoStrings += TargetTriple; + Stream.EmitRecordWithBlob(AbbrevCode, Record, InfoStrings); + + Stream.ExitBlock(); +} + +void IndexUnitWriter::writeDependencies(llvm::BitstreamWriter &Stream, + PathStorage &PathStore) { + std::vector FileUsedForRecordOrUnit; + FileUsedForRecordOrUnit.resize(Files.size()); + + Stream.EnterSubblock(UNIT_DEPENDENCIES_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(UNIT_DEPENDENCY)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, UnitDependencyKindBitNum)); // Dependency kind + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // IsSystem + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // PathIndex (1-based, 0 = none) + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // ModuleIndex (1-based, 0 = none) + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // time_t + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // file size + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + RecordData Record; + + auto addRecordOrUnitData = [&](UnitDependencyKind K, const RecordOrUnitData &Data) { + Record.push_back(UNIT_DEPENDENCY); + Record.push_back(K); + Record.push_back(Data.IsSystem); + if (Data.FileIndex != -1) { + Record.push_back(PathStore.getPathIndex(Files[Data.FileIndex].File) + 1); + FileUsedForRecordOrUnit[Data.FileIndex] = true; + } else { + Record.push_back(0); + } + if (Data.ModuleIndex != -1) { + Record.push_back(Data.ModuleIndex + 1); + } else { + Record.push_back(0); + } + if (Data.FileIndex != -1) { + Record.push_back(Files[Data.FileIndex].File->getModificationTime()); + Record.push_back(Files[Data.FileIndex].File->getSize()); + } else { + Record.push_back(0); + Record.push_back(0); + } + Stream.EmitRecordWithBlob(AbbrevCode, Record, Data.Name); + }; + + for (auto &ASTData : ASTFileUnits) { + Record.clear(); + addRecordOrUnitData(UNIT_DEPEND_KIND_UNIT, ASTData); + } + for (auto &recordData : Records) { + Record.clear(); + addRecordOrUnitData(UNIT_DEPEND_KIND_RECORD, recordData); + } + size_t FileIndex = 0; + for (auto &File : Files) { + if (FileUsedForRecordOrUnit[FileIndex++]) + continue; + Record.clear(); + Record.push_back(UNIT_DEPENDENCY); + Record.push_back(UNIT_DEPEND_KIND_FILE); + Record.push_back(File.IsSystem); + Record.push_back(PathStore.getPathIndex(File.File) + 1); + if (File.ModuleIndex != -1) { + Record.push_back(File.ModuleIndex + 1); + } else { + Record.push_back(0); + } + Record.push_back(File.File->getModificationTime()); + Record.push_back(File.File->getSize()); + Stream.EmitRecordWithBlob(AbbrevCode, Record, StringRef()); + } + + Stream.ExitBlock(); +} + +void IndexUnitWriter::writeIncludes(llvm::BitstreamWriter &Stream, + PathStorage &PathStore) { + Stream.EnterSubblock(UNIT_INCLUDES_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(UNIT_INCLUDE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // source path index (1-based, 0 = no path) + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // source include line + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // target path index (1-based, 0 = no path) + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + RecordData Record; + + for (auto &Including : Files) { + for(auto &Included: Including.Includes) { + Record.clear(); + Record.push_back(UNIT_INCLUDE); + Record.push_back(PathStore.getPathIndex(Including.File) + 1); + Record.push_back(Included.Line); + Record.push_back(PathStore.getPathIndex(Files[Included.Index].File) + 1); + Stream.EmitRecordWithAbbrev(AbbrevCode, Record); + } + } + Stream.ExitBlock(); +} + +void IndexUnitWriter::writePaths(llvm::BitstreamWriter &Stream, + PathStorage &PathStore) { + Stream.EnterSubblock(UNIT_PATHS_BLOCK_ID, 3); + + auto PathAbbrev = std::make_shared(); + PathAbbrev->Add(BitCodeAbbrevOp(UNIT_PATH)); + PathAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, UnitFilePathPrefixKindBitNum)); // Path prefix kind + PathAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // DirPath offset + PathAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // DirPath size + PathAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 10)); // Filename offset + PathAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Filename size + unsigned PathAbbrevCode = Stream.EmitAbbrev(std::move(PathAbbrev)); + + auto PathBufferAbbrev = std::make_shared(); + PathBufferAbbrev->Add(BitCodeAbbrevOp(UNIT_PATH_BUFFER)); + PathBufferAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Paths buffer + unsigned PathBufferAbbrevCode = Stream.EmitAbbrev(PathBufferAbbrev); + + RecordData Record; + for(auto &BitPath: PathStore.getBitPaths()) { + Record.push_back(UNIT_PATH); + Record.push_back(BitPath.PrefixKind); + Record.push_back(BitPath.Dir.Offset); + Record.push_back(BitPath.Dir.Size); + Record.push_back(BitPath.Filename.Offset); + Record.push_back(BitPath.Filename.Size); + Stream.EmitRecordWithAbbrev(PathAbbrevCode, Record); + Record.clear(); + } + + Record.push_back(UNIT_PATH_BUFFER); + Stream.EmitRecordWithBlob(PathBufferAbbrevCode, Record, PathStore.getPathsBuffer()); + + Stream.ExitBlock(); +} + +void IndexUnitWriter::writeModules(llvm::BitstreamWriter &Stream) { + Stream.EnterSubblock(UNIT_MODULES_BLOCK_ID, 3); + + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(UNIT_MODULE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 9)); // Module name offset + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Module name size + unsigned AbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + + auto BufferAbbrev = std::make_shared(); + BufferAbbrev->Add(BitCodeAbbrevOp(UNIT_MODULE_BUFFER)); + BufferAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Module names buffer + unsigned BufferAbbrevCode = Stream.EmitAbbrev(BufferAbbrev); + + SmallString<512> ModuleNamesBuf; + + RecordData Record; + for (auto &Mod : Modules) { + SmallString<64> ModuleName; + StringRef name = GetInfoForModuleFn(Mod, ModuleName).Name; + size_t offset = ModuleNamesBuf.size(); + ModuleNamesBuf += name; + + Record.push_back(UNIT_MODULE); + Record.push_back(offset); + Record.push_back(name.size()); + Stream.EmitRecordWithAbbrev(AbbrevCode, Record); + Record.clear(); + } + + Record.push_back(UNIT_MODULE_BUFFER); + Stream.EmitRecordWithBlob(BufferAbbrevCode, Record, ModuleNamesBuf.str()); + + Stream.ExitBlock(); +} + +bool IndexUnitWriter::initIndexDirectory(StringRef StorePath, + std::string &Error) { + using namespace llvm::sys; + SmallString<128> SubPath = StorePath; + store::appendRecordSubDir(SubPath); + std::error_code EC = fs::create_directories(SubPath); + if (EC) { + llvm::raw_string_ostream Err(Error); + Err << "failed to create directory '" << SubPath << "': " << EC.message(); + return true; + } + + SubPath = StorePath; + store::appendUnitSubDir(SubPath); + EC = fs::create_directory(SubPath); + if (EC) { + llvm::raw_string_ostream Err(Error); + Err << "failed to create directory '" << SubPath << "': " << EC.message(); + return true; + } + + return false; +} Index: lib/Index/IndexingAction.cpp =================================================================== --- lib/Index/IndexingAction.cpp +++ lib/Index/IndexingAction.cpp @@ -9,9 +9,16 @@ #include "clang/Index/IndexingAction.h" #include "clang/Index/IndexDataConsumer.h" +#include "FileIndexRecord.h" #include "IndexingContext.h" +#include "ClangIndexRecordWriter.h" +#include "IndexDataStoreUtils.h" +#include "clang/Index/IndexUnitWriter.h" +#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" +#include "clang/Frontend/FrontendDiagnostic.h" #include "clang/Frontend/MultiplexConsumer.h" +#include "clang/Frontend/Utils.h" #include "clang/Lex/Preprocessor.h" #include "clang/Serialization/ASTReader.h" @@ -80,7 +87,8 @@ : DataConsumer(std::move(dataConsumer)), IndexCtx(Opts, *DataConsumer) {} - std::unique_ptr createIndexASTConsumer() { + std::unique_ptr createIndexASTConsumer(CompilerInstance &CI) { + IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); return llvm::make_unique(IndexCtx); } @@ -98,7 +106,7 @@ protected: std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override { - return createIndexASTConsumer(); + return createIndexASTConsumer(CI); } void EndSourceFileAction() override { @@ -108,7 +116,7 @@ }; class WrappingIndexAction : public WrapperFrontendAction, IndexActionBase { - bool IndexActionFailed = false; + bool CreatedASTConsumer = false; public: WrappingIndexAction(std::unique_ptr WrappedAction, @@ -128,21 +136,20 @@ void WrappingIndexAction::EndSourceFileAction() { // Invoke wrapped action's method. WrapperFrontendAction::EndSourceFileAction(); - if (!IndexActionFailed) + if (CreatedASTConsumer) finish(); } std::unique_ptr WrappingIndexAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); - if (!OtherConsumer) { - IndexActionFailed = true; + if (!OtherConsumer) return nullptr; - } + CreatedASTConsumer = true; std::vector> Consumers; Consumers.push_back(std::move(OtherConsumer)); - Consumers.push_back(createIndexASTConsumer()); + Consumers.push_back(createIndexASTConsumer(CI)); return llvm::make_unique(std::move(Consumers)); } @@ -203,3 +210,628 @@ } DataConsumer->finish(); } + +//===----------------------------------------------------------------------===// +// Index Data Recording +//===----------------------------------------------------------------------===// + +namespace { + +class IndexDataRecorder : public IndexDataConsumer { + IndexingContext *IndexCtx = nullptr; + const Preprocessor *PP = nullptr; + typedef llvm::DenseMap> RecordByFileTy; + RecordByFileTy RecordByFile; + +public: + void init(IndexingContext *idxCtx, const CompilerInstance &CI) { + IndexCtx = idxCtx; + PP = &CI.getPreprocessor(); + initialize(CI.getASTContext()); + } + + RecordByFileTy::const_iterator record_begin() const { return RecordByFile.begin(); } + RecordByFileTy::const_iterator record_end() const { return RecordByFile.end(); } + bool record_empty() const { return RecordByFile.empty(); } + +private: + bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles, + ArrayRef Relations, + FileID FID, unsigned Offset, + ASTNodeInfo ASTNode) override { + // Ignore the predefines buffer. + if (FID == PP->getPredefinesFileID()) + return true; + + FileIndexRecord &Rec = getFileIndexRecord(FID); + Rec.addDeclOccurence(Roles, Offset, D, Relations); + return true; + } + + FileIndexRecord &getFileIndexRecord(FileID FID) { + auto &Entry = RecordByFile[FID]; + if (!Entry) { + Entry.reset(new FileIndexRecord(FID, IndexCtx->isSystemFile(FID))); + } + return *Entry; + } +}; + +struct IncludeLocation { + const FileEntry *Source; + const FileEntry *Target; + unsigned Line; +}; + +class IncludePPCallbacks : public PPCallbacks { + IndexingContext &IndexCtx; + RecordingOptions RecordOpts; + std::vector &Includes; + SourceManager &SourceMgr; + +public: + IncludePPCallbacks(IndexingContext &indexCtx, RecordingOptions recordOpts, + std::vector &IncludesForFile, + SourceManager &SourceMgr) : + IndexCtx(indexCtx), RecordOpts(recordOpts), + Includes(IncludesForFile), SourceMgr(SourceMgr) {} + +private: + void addInclude(SourceLocation From, const FileEntry *To) { + assert(To); + if (RecordOpts.RecordIncludes == RecordingOptions::IncludesRecordingKind::None) + return; + + std::pair LocInfo = SourceMgr.getDecomposedExpansionLoc(From); + switch (RecordOpts.RecordIncludes) { + case RecordingOptions::IncludesRecordingKind::None: + llvm_unreachable("should have already checked in the beginning"); + case RecordingOptions::IncludesRecordingKind::UserOnly: + if (IndexCtx.isSystemFile(LocInfo.first)) + return; // Ignore includes of system headers. + break; + case RecordingOptions::IncludesRecordingKind::All: + break; + } + auto *FE = SourceMgr.getFileEntryForID(LocInfo.first); + if (!FE) + return; + auto lineNo = SourceMgr.getLineNumber(LocInfo.first, LocInfo.second); + Includes.push_back({FE, To, lineNo}); + } + + virtual void InclusionDirective(SourceLocation HashLoc, + const Token &IncludeTok, + StringRef FileName, + bool IsAngled, + CharSourceRange FilenameRange, + const FileEntry *File, + StringRef SearchPath, + StringRef RelativePath, + const Module *Imported) override { + if (HashLoc.isFileID() && File && File->isValid()) + addInclude(HashLoc, File); + } +}; + +class IndexDependencyProvider { +public: + virtual ~IndexDependencyProvider() {} + + virtual void visitFileDependencies(const CompilerInstance &CI, + llvm::function_ref visitor) = 0; + virtual void visitIncludes( + llvm::function_ref visitor) = 0; + virtual void visitModuleImports(const CompilerInstance &CI, + llvm::function_ref visitor) = 0; +}; + +class SourceFilesIndexDependencyCollector : public DependencyCollector, public IndexDependencyProvider { + IndexingContext &IndexCtx; + RecordingOptions RecordOpts; + llvm::SetVector Entries; + llvm::BitVector IsSystemByUID; + std::vector Includes; + SourceManager *SourceMgr = nullptr; + std::string SysrootPath; + +public: + SourceFilesIndexDependencyCollector(IndexingContext &indexCtx, RecordingOptions recordOpts) + : IndexCtx(indexCtx), RecordOpts(recordOpts) {} + + virtual void attachToPreprocessor(Preprocessor &PP) override { + DependencyCollector::attachToPreprocessor(PP); + PP.addPPCallbacks(llvm::make_unique(IndexCtx, + RecordOpts, + Includes, + PP.getSourceManager())); + } + + void setSourceManager(SourceManager *SourceMgr) { + this->SourceMgr = SourceMgr; + } + void setSysrootPath(StringRef sysroot) { SysrootPath = sysroot; } + + void visitFileDependencies(const CompilerInstance &CI, + llvm::function_ref visitor) override { + for (auto *FE : getEntries()) { + visitor(FE, isSystemFile(FE)); + } + } + + void visitIncludes( + llvm::function_ref visitor) override { + for (auto &Include : Includes) { + visitor(Include.Source, Include.Line, Include.Target); + } + } + + void visitModuleImports(const CompilerInstance &CI, + llvm::function_ref visitor) override { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + + if (auto Reader = CI.getModuleManager()) { + Reader->getModuleManager().visit([&](serialization::ModuleFile &Mod) -> bool { + bool isSystemMod = false; + if (Mod.isModule()) { + if (auto *M = HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false)) + isSystemMod = M->IsSystem; + } + if (!isSystemMod || needSystemDependencies()) + visitor(Mod, isSystemMod); + return true; // skip module dependencies. + }); + } + } + +private: + bool isSystemFile(const FileEntry *FE) { + auto UID = FE->getUID(); + return IsSystemByUID.size() > UID && IsSystemByUID[UID]; + } + + ArrayRef getEntries() const { + return Entries.getArrayRef(); + } + + bool needSystemDependencies() override { + return RecordOpts.RecordSystemDependencies; + } + + bool sawDependency(StringRef Filename, bool FromModule, + bool IsSystem, bool IsModuleFile, bool IsMissing) override { + bool sawIt = DependencyCollector::sawDependency(Filename, FromModule, + IsSystem, IsModuleFile, + IsMissing); + if (auto *FE = SourceMgr->getFileManager().getFile(Filename)) { + if (sawIt) + Entries.insert(FE); + // Record system-ness for all files that we pass through. + if (IsSystemByUID.size() < FE->getUID()+1) + IsSystemByUID.resize(FE->getUID()+1); + IsSystemByUID[FE->getUID()] = IsSystem || isInSysroot(Filename); + } + return sawIt; + } + + bool isInSysroot(StringRef Filename) { + return !SysrootPath.empty() && Filename.startswith(SysrootPath); + } +}; + +class IndexRecordActionBase { +protected: + RecordingOptions RecordOpts; + IndexDataRecorder Recorder; + IndexingContext IndexCtx; + SourceFilesIndexDependencyCollector DepCollector; + + IndexRecordActionBase(IndexingOptions IndexOpts, RecordingOptions recordOpts) + : RecordOpts(std::move(recordOpts)), + IndexCtx(IndexOpts, Recorder), + DepCollector(IndexCtx, RecordOpts) { + } + + std::unique_ptr + createIndexASTConsumer(CompilerInstance &CI) { + IndexCtx.setSysrootPath(CI.getHeaderSearchOpts().Sysroot); + Recorder.init(&IndexCtx, CI); + + Preprocessor &PP = CI.getPreprocessor(); + DepCollector.setSourceManager(&CI.getSourceManager()); + DepCollector.setSysrootPath(IndexCtx.getSysrootPath()); + DepCollector.attachToPreprocessor(PP); + + return llvm::make_unique(IndexCtx); + } + + void finish(CompilerInstance &CI); +}; + +class IndexRecordAction : public ASTFrontendAction, IndexRecordActionBase { +public: + IndexRecordAction(IndexingOptions IndexOpts, RecordingOptions RecordOpts) + : IndexRecordActionBase(std::move(IndexOpts), std::move(RecordOpts)) {} + +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + return createIndexASTConsumer(CI); + } + + void EndSourceFileAction() override { + FrontendAction::EndSourceFileAction(); + finish(getCompilerInstance()); + } +}; + +class WrappingIndexRecordAction : public WrapperFrontendAction, IndexRecordActionBase { + bool CreatedASTConsumer = false; + +public: + WrappingIndexRecordAction(std::unique_ptr WrappedAction, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts) + : WrapperFrontendAction(std::move(WrappedAction)), + IndexRecordActionBase(std::move(IndexOpts), std::move(RecordOpts)) {} + +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override { + auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile); + if (!OtherConsumer) + return nullptr; + + CreatedASTConsumer = true; + std::vector> Consumers; + Consumers.push_back(std::move(OtherConsumer)); + Consumers.push_back(createIndexASTConsumer(CI)); + return llvm::make_unique(std::move(Consumers)); + } + + void EndSourceFileAction() override { + // Invoke wrapped action's method. + WrapperFrontendAction::EndSourceFileAction(); + if (CreatedASTConsumer) + finish(getCompilerInstance()); + } +}; + +} // anonymous namespace + +static std::string getClangVersion() { + // Try picking the version from an Apple Clang tag. + std::string RepositoryPath = getClangRepositoryPath(); + StringRef BuildNumber = StringRef(RepositoryPath); + size_t DashOffset = BuildNumber.find('-'); + if (BuildNumber.startswith("clang") && DashOffset != StringRef::npos) { + BuildNumber = BuildNumber.substr(DashOffset + 1); + return BuildNumber; + } + // Fallback to the generic version. + return CLANG_VERSION_STRING; +} + +static void writeUnitData(const CompilerInstance &CI, + IndexDataRecorder &Recorder, + IndexDependencyProvider &DepProvider, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + StringRef OutputFile, + const FileEntry *RootFile, + Module *UnitModule, + StringRef SysrootPath); + +void IndexRecordActionBase::finish(CompilerInstance &CI) { + // We may emit more diagnostics so do the begin/end source file invocations + // on the diagnostic client. + // FIXME: FrontendAction::EndSourceFile() should probably not call + // CI.getDiagnosticClient().EndSourceFile()' until after it has called + // 'EndSourceFileAction()', so that code executing during EndSourceFileAction() + // can emit diagnostics. If this is fixed, DiagClientBeginEndRAII can go away. + struct DiagClientBeginEndRAII { + CompilerInstance &CI; + DiagClientBeginEndRAII(CompilerInstance &CI) : CI(CI) { + CI.getDiagnosticClient().BeginSourceFile(CI.getLangOpts()); + } + ~DiagClientBeginEndRAII() { + CI.getDiagnosticClient().EndSourceFile(); + } + } diagClientBeginEndRAII(CI); + + SourceManager &SM = CI.getSourceManager(); + DiagnosticsEngine &Diag = CI.getDiagnostics(); + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + StringRef DataPath = RecordOpts.DataDirPath; + + std::string Error; + if (IndexUnitWriter::initIndexDirectory(DataPath, Error)) { + unsigned DiagID = Diag.getCustomDiagID( + DiagnosticsEngine::Error, "failed creating index directory %0"); + Diag.Report(DiagID) << Error; + return; + } + + std::string OutputFile = CI.getFrontendOpts().OutputFile; + if (OutputFile.empty()) { + OutputFile = CI.getFrontendOpts().Inputs[0].getFile(); + OutputFile += ".o"; + } + + const FileEntry *RootFile = nullptr; + Module *UnitMod = nullptr; + bool isModuleGeneration = CI.getLangOpts().isCompilingModule(); + if (!isModuleGeneration && + CI.getFrontendOpts().ProgramAction != frontend::GeneratePCH) { + RootFile = SM.getFileEntryForID(SM.getMainFileID()); + } + if (isModuleGeneration) { + UnitMod = HS.lookupModule(CI.getLangOpts().CurrentModule, + /*AllowSearch=*/false); + } + + writeUnitData(CI, Recorder, DepCollector, IndexCtx.getIndexOpts(), RecordOpts, + OutputFile, RootFile, UnitMod, + IndexCtx.getSysrootPath()); +} + +/// Checks if the unit file exists for module file, if it doesn't it generates +/// index data for it. +static bool produceIndexDataForModuleFile( + serialization::ModuleFile &Mod, + const CompilerInstance &CI, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + IndexUnitWriter &ParentUnitWriter); + +static void writeUnitData(const CompilerInstance &CI, + IndexDataRecorder &Recorder, + IndexDependencyProvider &DepProvider, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + StringRef OutputFile, + const FileEntry *RootFile, + Module *UnitModule, + StringRef SysrootPath) { + + SourceManager &SM = CI.getSourceManager(); + DiagnosticsEngine &Diag = CI.getDiagnostics(); + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + StringRef DataPath = RecordOpts.DataDirPath; + bool IsSystemUnit = UnitModule ? UnitModule->IsSystem : false; + bool IsModuleUnit = UnitModule != nullptr; + bool IsDebugCompilation = CI.getCodeGenOpts().OptimizationLevel == 0; + std::string ModuleName = UnitModule ? UnitModule->getFullModuleName() : std::string(); + + auto getModuleInfo = [](writer::OpaqueModule mod, SmallVectorImpl &Scratch) -> writer::ModuleInfo { + assert(mod); + writer::ModuleInfo info; + std::string fullName = static_cast(mod)->getFullModuleName(); + unsigned offset = Scratch.size(); + Scratch.append(fullName.begin(), fullName.end()); + info.Name = StringRef(Scratch.data()+offset, fullName.size()); + return info; + }; + + auto findModuleForHeader = [&](const FileEntry *FE) -> Module * { + if (!UnitModule) + return nullptr; + if (auto Mod = HS.findModuleForHeader(FE).getModule()) + if (Mod->isSubModuleOf(UnitModule)) + return Mod; + return nullptr; + }; + + IndexUnitWriter UnitWriter(CI.getFileManager(), + DataPath, + "clang", getClangVersion(), + OutputFile, + ModuleName, + RootFile, + IsSystemUnit, + IsModuleUnit, + IsDebugCompilation, + CI.getTargetOpts().Triple, + SysrootPath, + getModuleInfo); + + DepProvider.visitFileDependencies(CI, [&](const FileEntry *FE, bool isSystemFile) { + UnitWriter.addFileDependency(FE, isSystemFile, findModuleForHeader(FE)); + }); + DepProvider.visitIncludes([&](const FileEntry *Source, unsigned Line, const FileEntry *Target) { + UnitWriter.addInclude(Source, Line, Target); + }); + DepProvider.visitModuleImports(CI, [&](serialization::ModuleFile &Mod, bool isSystemMod) { + Module *UnitMod = HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false); + UnitWriter.addASTFileDependency(Mod.File, isSystemMod, UnitMod); + if (Mod.isModule()) { + produceIndexDataForModuleFile(Mod, CI, IndexOpts, RecordOpts, UnitWriter); + } + }); + + ClangIndexRecordWriter RecordWriter(CI.getASTContext(), RecordOpts); + for (auto I = Recorder.record_begin(), E = Recorder.record_end(); I != E; ++I) { + FileID FID = I->first; + const FileIndexRecord &Rec = *I->second; + const FileEntry *FE = SM.getFileEntryForID(FID); + std::string RecordFile; + std::string Error; + + if (RecordWriter.writeRecord(FE->getName(), Rec, Error, &RecordFile)) { + unsigned DiagID = Diag.getCustomDiagID(DiagnosticsEngine::Error, + "failed writing record '%0': %1"); + Diag.Report(DiagID) << RecordFile << Error; + return; + } + UnitWriter.addRecordFile(RecordFile, FE, Rec.isSystem(), + findModuleForHeader(FE)); + } + + std::string Error; + if (UnitWriter.write(Error)) { + unsigned DiagID = Diag.getCustomDiagID(DiagnosticsEngine::Error, + "failed writing unit data: %0"); + Diag.Report(DiagID) << Error; + return; + } +} + +namespace { +class ModuleFileIndexDependencyCollector : public IndexDependencyProvider { + serialization::ModuleFile &ModFile; + RecordingOptions RecordOpts; + +public: + ModuleFileIndexDependencyCollector(serialization::ModuleFile &Mod, + RecordingOptions recordOpts) + : ModFile(Mod), RecordOpts(recordOpts) {} + + void visitFileDependencies(const CompilerInstance &CI, + llvm::function_ref visitor) override { + auto Reader = CI.getModuleManager(); + Reader->visitInputFiles(ModFile, RecordOpts.RecordSystemDependencies, + /*Complain=*/false, + [&](const serialization::InputFile &IF, bool isSystem) { + auto *FE = IF.getFile(); + if (!FE) + return; + // Ignore module map files, they are not as important to track as source + // files and they may be auto-generated which would create an undesirable + // dependency on an intermediate build byproduct. + if (FE->getName().endswith("module.modulemap")) + return; + + visitor(FE, isSystem); + }); + } + + void visitIncludes( + llvm::function_ref visitor) override { + // FIXME: Module files without a preprocessing record do not have info about + // include locations. Serialize enough data to be able to retrieve such info. + } + + void visitModuleImports(const CompilerInstance &CI, + llvm::function_ref visitor) override { + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + for (auto *Mod : ModFile.Imports) { + bool isSystemMod = false; + if (auto *M = HS.lookupModule(Mod->ModuleName, /*AllowSearch=*/false)) + isSystemMod = M->IsSystem; + if (!isSystemMod || RecordOpts.RecordSystemDependencies) + visitor(*Mod, isSystemMod); + } + } +}; +} // anonymous namespace. + +static void indexModule(serialization::ModuleFile &Mod, + const CompilerInstance &CI, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts) { + DiagnosticsEngine &Diag = CI.getDiagnostics(); + Diag.Report(Mod.ImportLoc, diag::remark_index_producing_module_file_data) + << Mod.FileName; + + StringRef SysrootPath = CI.getHeaderSearchOpts().Sysroot; + HeaderSearch &HS = CI.getPreprocessor().getHeaderSearchInfo(); + Module *UnitMod = HS.lookupModule(Mod.ModuleName, /*AllowSearch=*/false); + + IndexDataRecorder Recorder; + IndexingContext IndexCtx(IndexOpts, Recorder); + + IndexCtx.setASTContext(CI.getASTContext()); + IndexCtx.setSysrootPath(SysrootPath); + Recorder.init(&IndexCtx, CI); + + for (const Decl *D : CI.getModuleManager()->getModuleFileLevelDecls(Mod)) { + IndexCtx.indexTopLevelDecl(D); + } + Recorder.finish(); + + ModuleFileIndexDependencyCollector DepCollector(Mod, RecordOpts); + writeUnitData(CI, Recorder, DepCollector, IndexOpts, RecordOpts, + Mod.FileName, /*RootFile=*/nullptr, UnitMod, SysrootPath); + +} + +static bool produceIndexDataForModuleFile( + serialization::ModuleFile &Mod, + const CompilerInstance &CI, + IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + IndexUnitWriter &ParentUnitWriter) { + DiagnosticsEngine &Diag = CI.getDiagnostics(); + std::string Error; + // We don't do timestamp check with the PCM file, on purpose. The PCM may get + // touched for various reasons which would cause unnecessary work to emit + // index data. User modules normally will get rebuilt and their index data + // re-emitted, and system modules are generally stable (and they can also can + // get rebuilt along with their index data). + auto IsUptodateOpt = ParentUnitWriter.isUnitUpToDateForOutputFile(Mod.FileName, None, Error); + if (!IsUptodateOpt.hasValue()) { + unsigned DiagID = Diag.getCustomDiagID(DiagnosticsEngine::Error, + "failed file status check: %0"); + Diag.Report(DiagID) << Error; + return false; + } + if (*IsUptodateOpt) + return false; + + indexModule(Mod, CI, IndexOpts, RecordOpts); + return true; +} + +static std::unique_ptr +createIndexDataRecordingAction(IndexingOptions IndexOpts, + RecordingOptions RecordOpts, + std::unique_ptr WrappedAction) { + if (WrappedAction) + return llvm::make_unique(std::move(WrappedAction), + std::move(IndexOpts), + std::move(RecordOpts)); + return llvm::make_unique(std::move(IndexOpts), + std::move(RecordOpts)); +} + +static std::pair +getIndexOptionsFromFrontendOptions(const FrontendOptions &FEOpts) { + index::IndexingOptions IndexOpts; + index::RecordingOptions RecordOpts; + RecordOpts.DataDirPath = FEOpts.IndexStorePath; + if (FEOpts.IndexIgnoreSystemSymbols) { + IndexOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::None; + } + RecordOpts.RecordSymbolCodeGenName = FEOpts.IndexRecordCodegenName; + return { IndexOpts, RecordOpts }; +} + +std::unique_ptr +index::createIndexDataRecordingAction(const FrontendOptions &FEOpts, + std::unique_ptr WrappedAction) { + index::IndexingOptions IndexOpts; + index::RecordingOptions RecordOpts; + std::tie(IndexOpts, RecordOpts) = getIndexOptionsFromFrontendOptions(FEOpts); + return ::createIndexDataRecordingAction(IndexOpts, RecordOpts, + std::move(WrappedAction)); +} + +bool index::emitIndexDataForModuleFile(const Module *Mod, + const CompilerInstance &CI, + IndexUnitWriter &ParentUnitWriter) { + index::IndexingOptions IndexOpts; + index::RecordingOptions RecordOpts; + std::tie(IndexOpts, RecordOpts) = getIndexOptionsFromFrontendOptions(CI.getFrontendOpts()); + + auto astReader = CI.getModuleManager(); + serialization::ModuleFile *ModFile = astReader->getModuleManager().lookup(Mod->getASTFile()); + assert(ModFile && "no module file loaded for module ?"); + return produceIndexDataForModuleFile(*ModFile, CI, IndexOpts, RecordOpts, ParentUnitWriter); +} Index: lib/Index/IndexingContext.h =================================================================== --- lib/Index/IndexingContext.h +++ lib/Index/IndexingContext.h @@ -11,9 +11,11 @@ #define LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H #include "clang/Basic/LLVM.h" +#include "clang/Basic/SourceLocation.h" #include "clang/Index/IndexSymbol.h" #include "clang/Index/IndexingAction.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" namespace clang { class ASTContext; @@ -29,7 +31,7 @@ class Stmt; class Expr; class TypeLoc; - class SourceLocation; + class DirectoryEntry; namespace index { class IndexDataConsumer; @@ -38,6 +40,13 @@ IndexingOptions IndexOpts; IndexDataConsumer &DataConsumer; ASTContext *Ctx = nullptr; + std::string SysrootPath; + + // Records whether a directory entry is system or not. + llvm::DenseMap DirEntries; + // Keeps track of the last check for whether a FileID is system or + // not. This is used to speed up isSystemFile() call. + std::pair LastFileCheck; public: IndexingContext(IndexingOptions IndexOpts, IndexDataConsumer &DataConsumer) @@ -47,6 +56,10 @@ IndexDataConsumer &getDataConsumer() { return DataConsumer; } void setASTContext(ASTContext &ctx) { Ctx = &ctx; } + void setSysrootPath(StringRef path); + StringRef getSysrootPath() const { return SysrootPath; } + + bool isSystemFile(FileID FID); bool shouldIndex(const Decl *D); Index: lib/Index/IndexingContext.cpp =================================================================== --- lib/Index/IndexingContext.cpp +++ lib/Index/IndexingContext.cpp @@ -93,12 +93,7 @@ if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + if (isSystemFile(FID)) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; @@ -161,6 +156,56 @@ return true; } +void IndexingContext::setSysrootPath(StringRef path) { + // Ignore sysroot path if it points to root, otherwise every header will be + // treated as system one. + if (path == "/") + path = StringRef(); + SysrootPath = path; +} + +bool IndexingContext::isSystemFile(FileID FID) { + if (LastFileCheck.first == FID) + return LastFileCheck.second; + + auto result = [&](bool res) -> bool { + LastFileCheck = { FID, res }; + return res; + }; + + bool Invalid = false; + const SrcMgr::SLocEntry &SEntry = + Ctx->getSourceManager().getSLocEntry(FID, &Invalid); + if (Invalid || !SEntry.isFile()) + return result(false); + + const SrcMgr::FileInfo &FI = SEntry.getFile(); + if (FI.getFileCharacteristic() != SrcMgr::C_User) + return result(true); + + auto *CC = FI.getContentCache(); + if (!CC) + return result(false); + auto *FE = CC->OrigEntry; + if (!FE) + return result(false); + + if (SysrootPath.empty()) + return result(false); + + // Check if directory is in sysroot so that we can consider system headers + // even the headers found via a user framework search path, pointing inside + // sysroot. + auto dirEntry = FE->getDir(); + auto pair = DirEntries.insert(std::make_pair(dirEntry, false)); + bool &isSystemDir = pair.first->second; + bool wasInserted = pair.second; + if (wasInserted) { + isSystemDir = StringRef(dirEntry->getName()).startswith(SysrootPath); + } + return result(isSystemDir); +} + static const CXXRecordDecl * getDeclContextForTemplateInstationPattern(const Decl *D) { if (const auto *CTSD = @@ -321,7 +366,7 @@ const Expr *OrigE, const Decl *OrigD, const DeclContext *ContainerDC) { - if (D->isImplicit() && !isa(D)) + if (D->isImplicit() && !(isa(D) || isa(D))) return true; if (!isa(D) || shouldSkipNamelessDecl(cast(D))) return true; @@ -337,12 +382,7 @@ if (FID.isInvalid()) return true; - bool Invalid = false; - const SrcMgr::SLocEntry &SEntry = SM.getSLocEntry(FID, &Invalid); - if (Invalid || !SEntry.isFile()) - return true; - - if (SEntry.getFile().getFileCharacteristic() != SrcMgr::C_User) { + if (isSystemFile(FID)) { switch (IndexOpts.SystemSymbolFilter) { case IndexingOptions::SystemSymbolFilterKind::None: return true; Index: test/Index/Store/Inputs/head.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/head.h @@ -0,0 +1,3 @@ + +extern void test1_func(void); +extern void test2_func(void); Index: test/Index/Store/Inputs/json.c.json =================================================================== --- /dev/null +++ test/Index/Store/Inputs/json.c.json @@ -0,0 +1,151 @@ +{ + "files": [ + "/test1.o", + "/Inputs/test1.c", + "/Inputs/head.h", + "/test2.o", + "/Inputs/test2.c", + "/test3.o", + "/Inputs/test3.cpp" + ], + "symbols": [ + { + "kind": "function", + "lang": "C", + "usr": "c:@F@test1_func", + "name": "test1_func", + "codegen": "_test1_func", + "roles": "Decl,Def" + }, + { + "kind": "function", + "lang": "C", + "usr": "c:@F@test2_func", + "name": "test2_func", + "codegen": "_test2_func", + "roles": "Decl,Def" + }, + { + "kind": "class", + "lang": "C++", + "usr": "c:@S@Base", + "name": "Base", + "roles": "Def,Ref,RelBase,RelCont" + }, + { + "kind": "class", + "lang": "C++", + "usr": "c:@S@Sub", + "name": "Sub", + "roles": "Def", + "rel-roles": "RelBase,RelCont" + } + ], + "records": [ + { + "occurrences": [ + { + "symbol": 0, + "line": 3, + "col": 6, + "roles": "Def" + } + ] + }, + { + "occurrences": [ + { + "symbol": 0, + "line": 2, + "col": 13, + "roles": "Decl" + }, + { + "symbol": 1, + "line": 3, + "col": 13, + "roles": "Decl" + } + ] + }, + { + "occurrences": [ + { + "symbol": 1, + "line": 3, + "col": 6, + "roles": "Def" + } + ] + }, + { + "occurrences": [ + { + "symbol": 2, + "line": 1, + "col": 7, + "roles": "Def" + }, + { + "symbol": 3, + "line": 2, + "col": 7, + "roles": "Def" + }, + { + "symbol": 2, + "line": 2, + "col": 20, + "roles": "Ref,RelBase,RelCont", + "relations": [ + { + "symbol": 3, + "rel-roles": "RelBase,RelCont" + } + ] + + } + ] + } + ], + "units": [ + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 0, + "sources": [ + { + "file": 1, + "records": [0] + }, + { + "file": 2, + "records": [1] + } + ] + }, + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 3, + "sources": [ + { + "file": 4, + "records": [2] + }, + { + "file": 2, + "records": [1] + } + ] + }, + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 5, + "sources": [ + { + "file": 6, + "records": [3] + } + ] + } + ] +} Index: test/Index/Store/Inputs/module/ModDep.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/ModDep.h @@ -0,0 +1,3 @@ +#include "ModTop.h" + +void ModDep_func(ModTopStruct s); Index: test/Index/Store/Inputs/module/ModSystem.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/ModSystem.h @@ -0,0 +1,4 @@ + +typedef struct {} ModSystemStruct; + +void ModSystem_func(void); Index: test/Index/Store/Inputs/module/ModTop.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/ModTop.h @@ -0,0 +1,4 @@ + +typedef struct {} ModTopStruct; + +void ModTop_func(void); Index: test/Index/Store/Inputs/module/ModTopSub1.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/ModTopSub1.h @@ -0,0 +1 @@ +void ModTopSub1_func(void); Index: test/Index/Store/Inputs/module/ModTopSub2.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/ModTopSub2.h @@ -0,0 +1 @@ +// This header has no symbols, intended to show up as file dependency. Index: test/Index/Store/Inputs/module/module.modulemap =================================================================== --- /dev/null +++ test/Index/Store/Inputs/module/module.modulemap @@ -0,0 +1,12 @@ +module ModTop { + header "ModTop.h" + export * + module Sub1 { + header "ModTopSub1.h" + } + module Sub2 { + header "ModTopSub2.h" + } +} +module ModDep { header "ModDep.h" export * } +module ModSystem [system] { header "ModSystem.h" export * } Index: test/Index/Store/Inputs/overlay.yaml =================================================================== --- /dev/null +++ test/Index/Store/Inputs/overlay.yaml @@ -0,0 +1,6 @@ +{ + 'version': 0, + 'roots': [{ 'type': 'file', 'name': 'OUT_DIR/using-overlay.h', + 'external-contents': 'INPUT_DIR/using-overlay.h' + }] +} Index: test/Index/Store/Inputs/print-unit.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/print-unit.h @@ -0,0 +1,2 @@ +#include "head.h" +#include "using-overlay.h" Index: test/Index/Store/Inputs/sys/another.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/sys/another.h @@ -0,0 +1,2 @@ + +extern void sys_another_func(void); Index: test/Index/Store/Inputs/sys/syshead.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/sys/syshead.h @@ -0,0 +1,4 @@ + +#include "another.h" + +extern void sys_test1_func(void); Index: test/Index/Store/Inputs/test1.c =================================================================== --- /dev/null +++ test/Index/Store/Inputs/test1.c @@ -0,0 +1,3 @@ +#include "head.h" + +void test1_func(void) {} Index: test/Index/Store/Inputs/test2.c =================================================================== --- /dev/null +++ test/Index/Store/Inputs/test2.c @@ -0,0 +1,3 @@ +#include "head.h" + +void test2_func(void) {} Index: test/Index/Store/Inputs/test3.cpp =================================================================== --- /dev/null +++ test/Index/Store/Inputs/test3.cpp @@ -0,0 +1,2 @@ +class Base {}; +class Sub : public Base {}; Index: test/Index/Store/Inputs/using-overlay.h =================================================================== --- /dev/null +++ test/Index/Store/Inputs/using-overlay.h @@ -0,0 +1 @@ +void using_overlay(void); Index: test/Index/Store/assembly-invocation.c =================================================================== --- /dev/null +++ test/Index/Store/assembly-invocation.c @@ -0,0 +1,3 @@ +// Make sure it doesn't crash. +// RUN: %clang -target x86_64-apple-macosx10.7 -S %s -o %t.s +// RUN: env CLANG_PROJECT_INDEX_PATH=%t.idx %clang -target x86_64-apple-macosx10.7 -c %t.s -o %t.o Index: test/Index/Store/external-source-symbol-hash.m =================================================================== --- /dev/null +++ test/Index/Store/external-source-symbol-hash.m @@ -0,0 +1,47 @@ +// RUN: rm -rf %t.idx +// RUN: %clang_cc1 %s -index-store-path %t.idx -D USE_EXTERNAL +// RUN: c-index-test core -print-record %t.idx | FileCheck %s +// RUN: %clang_cc1 %s -index-store-path %t.idx +// RUN: find %t.idx/*/records -name "external-source-symbol-hash*" | count 2 + +#ifdef USE_EXTERNAL +# define EXT_DECL(mod_name) __attribute__((external_source_symbol(language="Swift", defined_in=mod_name))) +#else +# define EXT_DECL(mod_name) +#endif + +#define NS_ENUM(_name, _type) enum _name:_type _name; enum _name : _type + +// Forward declarations should pick up the attribute from later decls +@protocol P1; +// CHECK: [[@LINE-1]]:11 | protocol/Swift | c:@M@some_module@objc(pl)P1 | Ref | rel: 0 +@class I2; +// CHECK: [[@LINE-1]]:8 | class/Swift | c:@M@other_module@objc(cs)I2 | Ref | rel: 0 +enum E3: int; +// CHECK: [[@LINE-1]]:6 | enum/Swift | c:@M@third_module@E@E3 | Ref | rel: 0 + +void test(id first, I2 *second, enum E3 third) {} +// CHECK: [[@LINE-1]]:14 | protocol/Swift | c:@M@some_module@objc(pl)P1 | Ref,RelCont | rel: 1 +// CHECK: [[@LINE-2]]:25 | class/Swift | c:@M@other_module@objc(cs)I2 | Ref,RelCont | rel: 1 +// CHECK: [[@LINE-3]]:42 | enum/Swift | c:@M@third_module@E@E3 | Ref,RelCont | rel: 1 + +EXT_DECL("some_module") +@protocol P1 +// CHECK: [[@LINE-1]]:11 | protocol/Swift | c:@M@some_module@objc(pl)P1 | Decl | rel: 0 +-(void)method; +// CHECK: [[@LINE-1]]:8 | instance-method/Swift | c:@M@some_module@objc(pl)P1(im)method | Decl,Dyn,RelChild | rel: 1 +@end + +EXT_DECL("other_module") +@interface I2 +// CHECK: [[@LINE-1]]:12 | class/Swift | c:@M@other_module@objc(cs)I2 | Decl | rel: 0 +-(void)method; +// CHECK: [[@LINE-1]]:8 | instance-method/Swift | c:@M@other_module@objc(cs)I2(im)method | Decl,Dyn,RelChild | rel: 1 +@end + + +typedef NS_ENUM(E3, int) { +// CHECK: [[@LINE-1]]:17 | enum/Swift | c:@M@third_module@E@E3 | Def | rel: 0 + firstCase = 1, + // CHECK: [[@LINE-1]]:3 | enumerator/Swift | c:@M@third_module@E@E3@firstCase | Def,RelChild | rel: 1 +} EXT_DECL("third_module"); Index: test/Index/Store/handle-prebuilt-module.m =================================================================== --- /dev/null +++ test/Index/Store/handle-prebuilt-module.m @@ -0,0 +1,25 @@ +// XFAIL: linux + +// RUN: rm -rf %t +// RUN: mkdir %t +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -fsyntax-only %s -o %t.o -index-store-path %t/idx1 -fmodules -fmodules-cache-path=%t/mcp -I %S/Inputs/module -Rindex-store 2> %t.err1 +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -fsyntax-only %s -o %t.o -index-store-path %t/idx2 -fmodules -fmodules-cache-path=%t/mcp -I %S/Inputs/module -Rindex-store 2> %t.err2 +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -fsyntax-only %s -o %t.o -index-store-path %t/idx2 -fmodules -fmodules-cache-path=%t/mcp -I %S/Inputs/module -Rindex-store 2> %t.err3 +// RUN: FileCheck -input-file=%t.err1 -check-prefix=CREATING_MODULES %s -allow-empty +// RUN: FileCheck -input-file=%t.err2 -check-prefix=CREATING_INDEX_DATA_FROM_MODULE_FILES %s +// RUN: FileCheck -input-file=%t.err3 -check-prefix=EXISTING_INDEX_DATA_FROM_MODULE_FILES %s -allow-empty +// RUN: c-index-test core -print-unit %t/idx1 > %t/all-units1.txt +// RUN: c-index-test core -print-unit %t/idx2 > %t/all-units2.txt +// RUN: c-index-test core -print-record %t/idx1 > %t/all-records1.txt +// RUN: c-index-test core -print-record %t/idx2 > %t/all-records2.txt +// RUN: diff -u %t/all-units1.txt %t/all-units2.txt +// RUN: diff -u %t/all-records1.txt %t/all-records2.txt + +@import ModDep; + +// CREATING_MODULES-NOT: remark: + +// CREATING_INDEX_DATA_FROM_MODULE_FILES: remark: producing index data for module file {{.*}}ModDep{{.*}}.pcm +// CREATING_INDEX_DATA_FROM_MODULE_FILES: remark: producing index data for module file {{.*}}ModTop{{.*}}.pcm + +// EXISTING_INDEX_DATA_FROM_MODULE_FILES-NOT: remark: Index: test/Index/Store/json-with-module.m =================================================================== --- /dev/null +++ test/Index/Store/json-with-module.m @@ -0,0 +1,9 @@ +// RUN: rm -rf %t.idx %t.mcp +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -c %s -o %t.o -index-store-path %t.idx -fmodules -fmodules-cache-path=%t.mcp -Xclang -fdisable-module-hash -I %S/Inputs/module +// RUN: c-index-test core -aggregate-json %t.idx -o %t.json +// RUN: sed -e "s:%S::g" -e "s:%T::g" %t.json > %t.final.json +// RUN: diff -u %s.json %t.final.json + +// XFAIL: linux + +@import ModDep; Index: test/Index/Store/json-with-module.m.json =================================================================== --- /dev/null +++ test/Index/Store/json-with-module.m.json @@ -0,0 +1,151 @@ +{ + "files": [ + "/json-with-module.m.tmp.mcp/ModDep.pcm", + "/json-with-module.m.tmp.mcp/ModTop.pcm", + "/Inputs/module/ModDep.h", + "/Inputs/module/ModTop.h", + "/Inputs/module/ModTopSub1.h", + "/Inputs/module/ModTopSub2.h", + "/json-with-module.m.tmp.o", + "/json-with-module.m", + "/Inputs/module/module.modulemap" + ], + "symbols": [ + { + "kind": "function", + "lang": "C", + "usr": "c:@F@ModDep_func", + "name": "ModDep_func", + "roles": "Decl", + "rel-roles": "RelCont" + }, + { + "kind": "type-alias", + "lang": "C", + "usr": "c:@T@ModTopStruct", + "name": "ModTopStruct", + "roles": "Def,Ref,RelCont" + }, + { + "kind": "struct", + "lang": "C", + "usr": "c:@SA@ModTopStruct", + "name": "", + "roles": "Def" + }, + { + "kind": "function", + "lang": "C", + "usr": "c:@F@ModTop_func", + "name": "ModTop_func", + "roles": "Decl" + }, + { + "kind": "function", + "lang": "C", + "usr": "c:@F@ModTopSub1_func", + "name": "ModTopSub1_func", + "roles": "Decl" + } + ], + "records": [ + { + "occurrences": [ + { + "symbol": 0, + "line": 3, + "col": 6, + "roles": "Decl" + }, + { + "symbol": 1, + "line": 3, + "col": 18, + "roles": "Ref,RelCont", + "relations": [ + { + "symbol": 0, + "rel-roles": "RelCont" + } + ] + + } + ] + }, + { + "occurrences": [ + { + "symbol": 2, + "line": 2, + "col": 9, + "roles": "Def" + }, + { + "symbol": 1, + "line": 2, + "col": 19, + "roles": "Def" + }, + { + "symbol": 3, + "line": 4, + "col": 6, + "roles": "Decl" + } + ] + }, + { + "occurrences": [ + { + "symbol": 4, + "line": 1, + "col": 6, + "roles": "Decl" + } + ] + } + ], + "units": [ + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 0, + "unit-dependencies": [1], + "sources": [ + { + "file": 2, + "records": [0] + } + ] + }, + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 1, + "sources": [ + { + "file": 3, + "records": [1] + }, + { + "file": 4, + "records": [2] + }, + { + "file": 5 + } + ] + }, + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 6, + "unit-dependencies": [0], + "sources": [ + { + "file": 7 + }, + { + "file": 8 + } + ] + } + ] +} Index: test/Index/Store/json-with-pch.c =================================================================== --- /dev/null +++ test/Index/Store/json-with-pch.c @@ -0,0 +1,10 @@ +// RUN: rm -rf %t.idx +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -x c-header %S/Inputs/head.h -o %t.h.pch -index-store-path %t.idx +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -c %s -o %t.o -index-store-path %t.idx -include %t.h -Werror +// RUN: c-index-test core -aggregate-json %t.idx -o %t.json +// RUN: sed -e "s:%S::g" -e "s:%T::g" %t.json > %t.final.json +// RUN: diff -u %s.json %t.final.json +// XFAIL: linux +int main() { + test1_func(); +} Index: test/Index/Store/json-with-pch.c.json =================================================================== --- /dev/null +++ test/Index/Store/json-with-pch.c.json @@ -0,0 +1,96 @@ +{ + "files": [ + "/json-with-pch.c.tmp.h.pch", + "/Inputs/head.h", + "/json-with-pch.c.tmp.o", + "/json-with-pch.c" + ], + "symbols": [ + { + "kind": "function", + "lang": "C", + "usr": "c:@F@test1_func", + "name": "test1_func", + "roles": "Decl,Ref,Call,RelCall,RelCont" + }, + { + "kind": "function", + "lang": "C", + "usr": "c:@F@test2_func", + "name": "test2_func", + "roles": "Decl" + }, + { + "kind": "function", + "lang": "C", + "usr": "c:@F@main", + "name": "main", + "roles": "Def", + "rel-roles": "RelCall,RelCont" + } + ], + "records": [ + { + "occurrences": [ + { + "symbol": 0, + "line": 2, + "col": 13, + "roles": "Decl" + }, + { + "symbol": 1, + "line": 3, + "col": 13, + "roles": "Decl" + } + ] + }, + { + "occurrences": [ + { + "symbol": 2, + "line": 8, + "col": 5, + "roles": "Def" + }, + { + "symbol": 0, + "line": 9, + "col": 3, + "roles": "Ref,Call,RelCall,RelCont", + "relations": [ + { + "symbol": 2, + "rel-roles": "RelCall,RelCont" + } + ] + + } + ] + } + ], + "units": [ + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 0, + "sources": [ + { + "file": 1, + "records": [0] + } + ] + }, + { + "triple": "x86_64-apple-macosx10.7.0", + "out-file": 2, + "unit-dependencies": [0], + "sources": [ + { + "file": 3, + "records": [1] + } + ] + } + ] +} Index: test/Index/Store/json.c =================================================================== --- /dev/null +++ test/Index/Store/json.c @@ -0,0 +1,10 @@ +// RUN: rm -rf %t.idx +// RUN: mkdir -p %t.o +// RUN: env CLANG_PROJECT_INDEX_PATH=%t.idx %clang -target x86_64-apple-macosx10.7 -arch x86_64 -mmacosx-version-min=10.7 -c %S/Inputs/test1.c -o %t.o/test1.o +// RUN: env CLANG_PROJECT_INDEX_PATH=%t.idx %clang -target x86_64-apple-macosx10.7 -arch x86_64 -mmacosx-version-min=10.7 -c %S/Inputs/test2.c -o %t.o/test2.o +// RUN: env CLANG_PROJECT_INDEX_PATH=%t.idx %clang -target x86_64-apple-macosx10.7 -arch x86_64 -mmacosx-version-min=10.7 -c %S/Inputs/test3.cpp -o %t.o/test3.o +// RUN: c-index-test core -aggregate-json %t.idx -o %t.json +// RUN: sed -e "s:%S::g" -e "s:%t.o::g" %t.json > %t.final.json +// RUN: diff -u %S/Inputs/json.c.json %t.final.json + +// XFAIL: linux Index: test/Index/Store/print-record.mm =================================================================== --- /dev/null +++ test/Index/Store/print-record.mm @@ -0,0 +1,28 @@ +// RUN: rm -rf %t.idx +// RUN: %clang_cc1 %s -index-store-path %t.idx +// RUN: c-index-test core -print-record %t.idx | FileCheck %s + +// XFAIL: linux + +@class MyCls; + +@interface MyCls +@end + +// CHECK: [[@LINE+2]]:6 | function/C | c:@F@foo#*$objc(cs)MyCls# | Decl | rel: 0 +// CHECK: [[@LINE+1]]:10 | class/ObjC | c:objc(cs)MyCls | Ref,RelCont | rel: 1 +void foo(MyCls *p); + + +// RANGE-NOT: before_range +void before_range(); + +// RANGE: [[@LINE+1]]:6 | function/C | c:@F@in_range1# | Decl +void in_range1(); +// RANGE: [[@LINE+1]]:6 | function/C | c:@F@in_range2# | Decl +void in_range2(); + +// RANGE-NOT: after_range +void after_range(); + +// RUN: c-index-test core -print-record %t.idx -filepath %s:21:23 | FileCheck -check-prefix=RANGE %s Index: test/Index/Store/print-unit.c =================================================================== --- /dev/null +++ test/Index/Store/print-unit.c @@ -0,0 +1,39 @@ +// XFAIL: linux + +#include "print-unit.h" +#include "syshead.h" + +void foo(int i); + +// RUN: rm -rf %t +// RUN: %clang_cc1 -I %S/Inputs -isystem %S/Inputs/sys -index-store-path %t/idx %s -triple x86_64-apple-macosx10.8 +// RUN: c-index-test core -print-unit %t/idx | FileCheck %s +// RUN: %clang_cc1 -I %S/Inputs -isystem %S/Inputs/sys -index-store-path %t/idx_opt1 %s -triple x86_64-apple-macosx10.8 -O2 +// RUN: c-index-test core -print-unit %t/idx_opt1 | FileCheck %s -check-prefix=OPT +// RUN: %clang_cc1 -I %S/Inputs -isystem %S/Inputs/sys -index-store-path %t/idx_opt2 %s -triple x86_64-apple-macosx10.8 -Os +// RUN: c-index-test core -print-unit %t/idx_opt2 | FileCheck %s -check-prefix=OPT + +// CHECK: print-unit.c.o +// CHECK: provider: clang- +// CHECK: is-system: 0 +// CHECK: has-main: 1 +// CHECK: main-path: {{.*}}/print-unit.c +// CHECK: out-file: {{.*}}/print-unit.c.o +// CHECK: target: x86_64-apple-macosx10.8 +// CHECK: is-debug: 1 +// CHECK: DEPEND START +// CHECK: Record | user | {{.*}}/print-unit.c | print-unit.c- +// CHECK: Record | user | {{.*}}/Inputs/head.h | head.h- +// CHECK: Record | user | {{.*}}/Inputs/using-overlay.h | using-overlay.h- +// CHECK: Record | system | {{.*}}/Inputs/sys/syshead.h | syshead.h- +// CHECK: Record | system | {{.*}}/Inputs/sys/another.h | another.h- +// CHECK: File | user | {{.*}}/Inputs/print-unit.h | | {{[0-9]*$}} +// CHECK: DEPEND END (6) +// CHECK: INCLUDE START +// CHECK: {{.*}}/print-unit.c:3 | {{.*}}/Inputs/print-unit.h +// CHECK: {{.*}}/print-unit.c:4 | {{.*}}/Inputs/sys/syshead.h +// CHECK: {{.*}}/Inputs/print-unit.h:1 | {{.*}}/Inputs/head.h +// CHECK: {{.*}}/Inputs/print-unit.h:2 | {{.*}}/Inputs/using-overlay.h +// CHECK: INCLUDE END (4) + +// OPT: is-debug: 0 Index: test/Index/Store/print-units-with-modules.m =================================================================== --- /dev/null +++ test/Index/Store/print-units-with-modules.m @@ -0,0 +1,59 @@ +// RUN: rm -rf %t.idx %t.mcp +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -c %s -o %t.o -index-store-path %t.idx -fmodules -fmodules-cache-path=%t.mcp -Xclang -fdisable-module-hash -I %S/Inputs/module +// RUN: c-index-test core -print-unit %t.idx | FileCheck %s + +// XFAIL: linux + +@import ModDep; +@import ModSystem; + +// CHECK: ModDep.pcm +// CHECK: provider: clang- +// CHECK: is-system: 0 +// CHECK: is-module: 1 +// CHECK: module-name: ModDep +// CHECK: has-main: 0 +// CHECK: main-path: {{$}} +// CHECK: out-file: {{.*}}/ModDep.pcm +// CHECK: DEPEND START +// CHECK: Unit | user | ModTop | {{.*}}/ModTop.pcm | ModTop.pcm +// CHECK: Record | user | ModDep | {{.*}}/Inputs/module/ModDep.h | ModDep.h +// CHECK: DEPEND END (2) + +// CHECK: ModSystem.pcm +// CHECK: is-system: 1 +// CHECK: is-module: 1 +// CHECK: module-name: ModSystem +// CHECK: has-main: 0 +// CHECK: main-path: {{$}} +// CHECK: out-file: {{.*}}/ModSystem.pcm +// CHECK: DEPEND START +// CHECK: Record | system | ModSystem | {{.*}}/Inputs/module/ModSystem.h | ModSystem.h +// CHECK: DEPEND END (1) + +// CHECK: ModTop.pcm +// CHECK: is-system: 0 +// CHECK: is-module: 1 +// CHECK: module-name: ModTop +// CHECK: has-main: 0 +// CHECK: main-path: {{$}} +// CHECK: out-file: {{.*}}/ModTop.pcm +// CHECK: DEPEND START +// CHECK: Record | user | ModTop | {{.*}}/Inputs/module/ModTop.h | ModTop.h +// CHECK: Record | user | ModTop.Sub1 | {{.*}}/Inputs/module/ModTopSub1.h | ModTopSub1.h +// CHECK: File | user | ModTop.Sub2 | {{.*}}/Inputs/module/ModTopSub2.h | | {{[0-9]*$}} +// CHECK: DEPEND END (3) + +// CHECK: print-units-with-modules.m.tmp.o +// CHECK: is-system: 0 +// CHECK: is-module: 0 +// CHECK: module-name: +// CHECK: has-main: 1 +// CHECK: main-path: {{.*}}/print-units-with-modules.m +// CHECK: out-file: {{.*}}/print-units-with-modules.m.tmp.o +// CHECK: DEPEND START +// CHECK: Unit | user | ModDep | {{.*}}/ModDep.pcm | ModDep.pcm +// CHECK: Unit | system | ModSystem | {{.*}}/ModSystem.pcm | ModSystem.pcm +// CHECK: File | user | {{.*}}/print-units-with-modules.m | | {{[0-9]*$}} +// CHECK: File | user | {{.*}}/Inputs/module/module.modulemap | | {{[0-9]*$}} +// CHECK: DEPEND END (4) Index: test/Index/Store/print-units-with-pch.c =================================================================== --- /dev/null +++ test/Index/Store/print-units-with-pch.c @@ -0,0 +1,29 @@ +// RUN: rm -rf %t.idx +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -x c-header %S/Inputs/head.h -o %t.h.pch -index-store-path %t.idx +// RUN: %clang -arch x86_64 -mmacosx-version-min=10.7 -c %s -o %t.o -index-store-path %t.idx -include %t.h -Werror +// RUN: c-index-test core -print-unit %t.idx | FileCheck %s + +// XFAIL: linux + +int main() { + test1_func(); +} + +// CHECK: print-units-with-pch.c.tmp.h.pch +// CHECK: is-system: 0 +// CHECK: has-main: 0 +// CHECK: main-path: {{$}} +// CHECK: out-file: {{.*}}/print-units-with-pch.c.tmp.h.pch +// CHECK: DEPEND START +// CHECK: Record | user | {{.*}}/Inputs/head.h | head.h +// CHECK: DEPEND END (1) + +// CHECK: print-units-with-pch.c.tmp.o +// CHECK: is-system: 0 +// CHECK: has-main: 1 +// CHECK: main-path: {{.*}}/print-units-with-pch.c +// CHECK: out-file: {{.*}}/print-units-with-pch.c.tmp.o +// CHECK: DEPEND START +// CHECK: Unit | user | {{.*}}/print-units-with-pch.c.tmp.h.pch | print-units-with-pch.c.tmp.h.pch +// CHECK: Record | user | {{.*}}/print-units-with-pch.c | print-units-with-pch.c +// CHECK: DEPEND END (2) Index: test/Index/Store/record-hash-crash-invalid-name.cpp =================================================================== --- /dev/null +++ test/Index/Store/record-hash-crash-invalid-name.cpp @@ -0,0 +1,17 @@ +// Makes sure it doesn't crash. + +// XFAIL: linux + +// RUN: rm -rf %t +// RUN: not %clang_cc1 %s -index-store-path %t/idx -std=c++14 +// RUN: c-index-test core -print-record %t/idx | FileCheck %s + +namespace rdar32474406 { +void foo(); +typedef void (*Func_t)(); +// CHECK: [[@LINE+4]]:1 | type-alias/C | c:record-hash-crash-invalid-name.cpp@N@rdar32474406@T@Func_t | Ref,RelCont | rel: 1 +// CHECK-NEXT: RelCont | c:@N@rdar32474406 +// CHECK: [[@LINE+2]]:14 | function/C | c:@N@rdar32474406@F@foo# | Ref,RelCont | rel: 1 +// CHECK-NEXT: RelCont | c:@N@rdar32474406 +Func_t[] = { foo }; // invalid decomposition +} Index: test/Index/Store/record-hash-crash.cpp =================================================================== --- /dev/null +++ test/Index/Store/record-hash-crash.cpp @@ -0,0 +1,31 @@ +// Makes sure it doesn't crash. + +// XFAIL: linux + +// RUN: rm -rf %t +// RUN: %clang_cc1 %s -index-store-path %t/idx -std=c++14 +// RUN: c-index-test core -print-record %t/idx | FileCheck %s + +namespace crash1 { +// CHECK: [[@LINE+1]]:6 | function/C +auto getit() { return []() {}; } +} + +namespace crash2 { +// CHECK: [[@LINE+2]]:7 | class(Gen)/C++ | c:@N@crash2@ST>1#T@Foo | Ref,RelCont | rel: 1 +template +class Foo; // canonical decl + +// CHECK: [[@LINE+2]]:7 | class(Gen)/C++ | c:@N@crash2@ST>1#T@Foo | Def,RelChild | rel: 1 +template +class Foo {}; + +// CHECK: [[@LINE+2]]:8 | struct(Gen)/C++ | c:@N@crash2@ST>1#t>1#pT@Wrapper | Def,RelChild | rel: 1 +template