diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -11,97 +11,92 @@ #include "clang/Basic/LLVM.h" #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ErrorOr.h" #include "llvm/Support/VirtualFileSystem.h" #include +#include namespace clang { namespace tooling { namespace dependencies { -/// An in-memory representation of a file system entity that is of interest to -/// the dependency scanning filesystem. -/// -/// It represents one of the following: -/// - an opened source file with minimized contents and a stat value. -/// - an opened source file with original contents and a stat value. -/// - a directory entry with its stat value. -/// - an error value to represent a file system error. -/// - a placeholder with an invalid stat indicating a not yet initialized entry. -class CachedFileSystemEntry { -public: - /// Default constructor creates an entry with an invalid stat. - CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {} - - CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {} - - /// Create an entry that represents an opened source file with minimized or - /// original contents. - /// - /// The filesystem opens the file even for `stat` calls open to avoid the - /// issues with stat + open of minimized files that might lead to a - /// mismatching size of the file. If file is not minimized, the full file is - /// read and copied into memory to ensure that it's not memory mapped to avoid - /// running out of file descriptors. - static CachedFileSystemEntry createFileEntry(StringRef Filename, - llvm::vfs::FileSystem &FS, - bool Minimize = true); - - /// Create an entry that represents a directory on the filesystem. - static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat); - - /// \returns True if the entry is valid. - bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); } - - /// \returns True if the current entry points to a directory. - bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); } - - /// \returns The error or the file's contents. - llvm::ErrorOr getContents() const { - if (!MaybeStat) - return MaybeStat.getError(); - assert(!MaybeStat->isDirectory() && "not a file"); - assert(isValid() && "not initialized"); - return Contents.str(); - } - - /// \returns The error or the status of the entry. - llvm::ErrorOr getStatus() const { - assert(isValid() && "not initialized"); - return MaybeStat; - } - - /// \returns the name of the file. - StringRef getName() const { - assert(isValid() && "not initialized"); - return MaybeStat->getName(); - } - - /// Return the mapping between location -> distance that is used to speed up - /// the block skipping in the preprocessor. - const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const { - return PPSkippedRangeMapping; - } - - CachedFileSystemEntry(CachedFileSystemEntry &&) = default; - CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; - - CachedFileSystemEntry(const CachedFileSystemEntry &) = delete; - CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete; +/// The result type that represents one of: +/// - a file system error, +/// - status of a directory, +/// - status of a file, +/// - status of an opened file and its original contents, +/// - status of an opened file and its minimized contents. +struct FileSystemEntryResult { + /// The filesystem error, or status. + llvm::ErrorOr Status; + /// The (potentially minimized) file contents. + const llvm::SmallString<1> *Contents; + /// The skipped range mappings produced during minimization. + const PreprocessorSkippedRangeMapping *PPSkippedRangeMapping; + + FileSystemEntryResult( + llvm::ErrorOr Status, + const llvm::SmallString<1> *Contents, + const PreprocessorSkippedRangeMapping *PPSkippedRangeMapping) + : Status(std::move(Status)), Contents(Contents), + PPSkippedRangeMapping(PPSkippedRangeMapping) {} +}; -private: - llvm::ErrorOr MaybeStat; - // Store the contents in a small string to allow a - // move from the small string for the minimized contents. - // Note: small size of 1 allows us to store an empty string with an implicit - // null terminator without any allocations. - llvm::SmallString<1> Contents; +/// The value type for stat cache. +using MaybeStat = llvm::ErrorOr; + +/// The value type for shared stat cache. +struct SharedStat { + /// The wrapped status value. + MaybeStat Stat; + + /// The mutex that must be locked when accessing this shared stat cache entry + /// and shared original/minimized content caches for the status' UniqueID. + std::mutex Mutex; + + SharedStat() : Stat(std::error_code()) {} + + /// Check whether this cache entry is valid. + bool isValid() const { return Stat || Stat.getError() != std::error_code(); } +}; + +// Small size of 1 allows us to store an empty string with an implicit null +// terminator without any allocations. +using Contents = llvm::SmallString<1>; + +/// The value type for original contents cache. +using OriginalContents = Contents; + +/// The value type for minimized contents cache. +struct MinimizedContents { + /// The minimized contents itself. + Contents Contents; PreprocessorSkippedRangeMapping PPSkippedRangeMapping; + + MinimizedContents() : Contents(), PPSkippedRangeMapping(0) {} }; +/// Result of the "insert" operation on map and set containers that exposes +/// members with more obvious names compared to \c std::pair::{first,second}. +template struct InsertResult { + T &Value; + bool WasInserted; + + InsertResult(T &Value, bool WasInserted) + : Value(Value), WasInserted(WasInserted) {} +}; + +/// A factory function for deducing the template parameter of \c InsertResult. +template +InsertResult makeInsertResult(T &Value, bool WasInserted) { + return InsertResult(Value, WasInserted); +} + /// This class is a shared cache, that caches the 'stat' and 'open' calls to the /// underlying real file system. It distinguishes between minimized and original /// files. @@ -110,67 +105,71 @@ /// the worker threads. class DependencyScanningFilesystemSharedCache { public: - struct SharedFileSystemEntry { - std::mutex ValueLock; - CachedFileSystemEntry Value; - }; + DependencyScanningFilesystemSharedCache(); + + /// Returns the cached stat entry for directory entry. Creates invalid status + /// if not found. This is a thread safe call. + SharedStat &getStat(StringRef Path); - /// Returns a cache entry for the corresponding key. - /// - /// A new cache entry is created if the key is not in the cache. This is a - /// thread safe call. - SharedFileSystemEntry &get(StringRef Key, bool Minimized); + /// Returns the cached original contents for file. Creates empty contents if + /// not found. This is a thread safe call. + InsertResult + getOriginalContents(StringRef Filename, llvm::sys::fs::UniqueID UID); + + /// Returns the cached minimized contents for file. Creates empty contents if + /// not found. This is a thread safe call. + InsertResult + getMinimizedContents(StringRef Filename, llvm::sys::fs::UniqueID UID); private: - class SingleCache { - public: - SingleCache(); - - SharedFileSystemEntry &get(StringRef Key); - - private: - struct CacheShard { - std::mutex CacheLock; - llvm::StringMap Cache; - }; - std::unique_ptr CacheShards; - unsigned NumShards; + struct CacheShard { + /// The mutex that's locked whenever insertion into any of the caches takes + /// place. + std::mutex CacheLock; + /// Cache of stat call results. + llvm::StringMap StatCache; + /// Cache of original file contents. Using \c std::map to ensure references + /// are not invalidated on insertion. + std::map OriginalContentsCache; + /// Cache of minimized file contents. + std::map MinimizedContentsCache; }; - - SingleCache CacheMinimized; - SingleCache CacheOriginal; + std::unique_ptr CacheShards; + unsigned NumShards; }; /// This class is a local cache, that caches the 'stat' and 'open' calls to the /// underlying real file system. It distinguishes between minimized and original /// files. class DependencyScanningFilesystemLocalCache { -private: - using SingleCache = - llvm::StringMap; - - SingleCache CacheMinimized; - SingleCache CacheOriginal; - - SingleCache &selectCache(bool Minimized) { - return Minimized ? CacheMinimized : CacheOriginal; - } + /// Cache of stat call results, pointing into the shared cache. + llvm::StringMap StatCache; + /// Cache of original file contents, pointing into the shared cache. + llvm::DenseMap + OriginalContentsCache; + /// Cache of minimized file contents, pointing into the shared cache. + llvm::DenseMap + MinimizedContentsCache; public: - void setCachedEntry(StringRef Filename, bool Minimized, - const CachedFileSystemEntry *Entry) { - SingleCache &Cache = selectCache(Minimized); - bool IsInserted = Cache.try_emplace(Filename, Entry).second; - (void)IsInserted; - assert(IsInserted && "local cache is updated more than once"); - } - - const CachedFileSystemEntry *getCachedEntry(StringRef Filename, - bool Minimized) { - SingleCache &Cache = selectCache(Minimized); - auto It = Cache.find(Filename); - return It == Cache.end() ? nullptr : It->getValue(); - } + /// Returns the status result for the path, or nullptr when not found. + const MaybeStat *findStat(StringRef Path); + /// Inserts the given status result into the cache for the path. + const MaybeStat *insertStat(StringRef Path, const MaybeStat *Status); + + /// Returns the original contents for the file, or nullptr when not found. + const OriginalContents *findOriginalContents(llvm::sys::fs::UniqueID UID); + /// Inserts the given original contents into the cache for the file. + const OriginalContents * + insertOriginalContents(llvm::sys::fs::UniqueID UID, + const OriginalContents *Contents); + + /// Returns the minimized contents for the file, or nullptr when not found. + const MinimizedContents *findMinimizedContents(llvm::sys::fs::UniqueID UID); + /// Inserts the given minimized contents into the cache for the file. + const MinimizedContents * + insertMinimizedContents(llvm::sys::fs::UniqueID UID, + const MinimizedContents *Contents); }; /// A virtual file system optimized for the dependency discovery. @@ -204,13 +203,32 @@ /// Check whether the file should be minimized. bool shouldMinimize(StringRef Filename); - llvm::ErrorOr - getOrCreateFileSystemEntry(const StringRef Filename); - - /// Create a cached file system entry based on the initial status result. - CachedFileSystemEntry - createFileSystemEntry(llvm::ErrorOr &&MaybeStatus, - StringRef Filename, bool ShouldMinimize); + /// Ensure \p Lock is locked with the mutex for entry at \p Path. + /// Also assigns to \p SS if null. + void ensureLocked(StringRef Path, SharedStat *&SS, + llvm::Optional> &Lock); + + /// Get the status for \p Path from local cache, populating it if necessary. + const MaybeStat * + getOrCreateLocalStat(StringRef Path, SharedStat *&SS, + llvm::Optional> &Lock); + + /// Get the original contents for \p Filename from local cache, populating it + /// if necessary. + const OriginalContents *getOrCreateLocalOriginalContents( + StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat, + llvm::Optional> &Lock); + + /// Get the minimized contents for \p Filename from local cache, populating it + /// if necessary. + const MinimizedContents *getOrCreateLocalMinimizedContents( + StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat, + llvm::Optional> &Lock); + + /// Get the full filesystem entry for \p Path from local cache, populating it + /// if necessary. + llvm::ErrorOr + getOrCreateFileSystemEntry(StringRef Path); /// The global cache shared between worker threads. DependencyScanningFilesystemSharedCache &SharedCache; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -15,9 +15,9 @@ using namespace tooling; using namespace dependencies; -CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( - StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { - // Load the file and its content from the file system. +static llvm::ErrorOr +readFile(StringRef Filename, llvm::vfs::FileSystem &FS, + llvm::SmallString<1> &SharedOriginalContents) { llvm::ErrorOr> MaybeFile = FS.openFileForRead(Filename); if (!MaybeFile) @@ -32,31 +32,27 @@ if (!MaybeBuffer) return MaybeBuffer.getError(); + const auto &Buffer = *MaybeBuffer; + SharedOriginalContents.reserve(Buffer->getBufferSize() + 1); + SharedOriginalContents.append(Buffer->getBufferStart(), + Buffer->getBufferEnd()); + // Implicitly null terminate the contents for Clang's lexer. + SharedOriginalContents.push_back('\0'); + SharedOriginalContents.pop_back(); + return *Stat; +} + +static bool +minimizeFile(const SmallString<1> &OriginalContents, + SmallString<1> &MinimizedContents, + PreprocessorSkippedRangeMapping &PPSkippedRangeMapping) { llvm::SmallString<1024> MinimizedFileContents; // Minimize the file down to directives that might affect the dependencies. - const auto &Buffer = *MaybeBuffer; SmallVector Tokens; - if (!Minimize || minimizeSourceToDependencyDirectives( - Buffer->getBuffer(), MinimizedFileContents, Tokens)) { - // Use the original file unless requested otherwise, or - // if the minimization failed. - // FIXME: Propage the diagnostic if desired by the client. - CachedFileSystemEntry Result; - Result.MaybeStat = std::move(*Stat); - Result.Contents.reserve(Buffer->getBufferSize() + 1); - Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); - // Implicitly null terminate the contents for Clang's lexer. - Result.Contents.push_back('\0'); - Result.Contents.pop_back(); - return Result; - } + if (minimizeSourceToDependencyDirectives(OriginalContents.str(), + MinimizedFileContents, Tokens)) + return true; - CachedFileSystemEntry Result; - size_t Size = MinimizedFileContents.size(); - Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), - Stat->getLastModificationTime(), - Stat->getUser(), Stat->getGroup(), Size, - Stat->getType(), Stat->getPermissions()); // The contents produced by the minimizer must be null terminated. assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && "not null terminated contents"); @@ -65,10 +61,10 @@ // std::move will preserve it even if it needs to do a copy if the // SmallString still has the small capacity. MinimizedFileContents.push_back('\0'); - Result.Contents = std::move(MinimizedFileContents); + MinimizedContents = std::move(MinimizedFileContents); // Now make the null terminator implicit again, so that Clang's lexer can find // it right where the buffer ends. - Result.Contents.pop_back(); + MinimizedContents.pop_back(); // Compute the skipped PP ranges that speedup skipping over inactive // preprocessor blocks. @@ -86,20 +82,13 @@ } Mapping[Range.Offset] = Range.Length; } - Result.PPSkippedRangeMapping = std::move(Mapping); + PPSkippedRangeMapping = std::move(Mapping); - return Result; + return false; } -CachedFileSystemEntry -CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { - assert(Stat.isDirectory() && "not a directory!"); - auto Result = CachedFileSystemEntry(); - Result.MaybeStat = std::move(Stat); - return Result; -} - -DependencyScanningFilesystemSharedCache::SingleCache::SingleCache() { +DependencyScanningFilesystemSharedCache:: + DependencyScanningFilesystemSharedCache() { // This heuristic was chosen using a empirical testing on a // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache // sharding gives a performance edge by reducing the lock contention. @@ -110,18 +99,67 @@ CacheShards = std::make_unique(NumShards); } -DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & -DependencyScanningFilesystemSharedCache::SingleCache::get(StringRef Key) { - CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; - std::unique_lock LockGuard(Shard.CacheLock); - auto It = Shard.Cache.try_emplace(Key); - return It.first->getValue(); +SharedStat & +DependencyScanningFilesystemSharedCache::getStat(StringRef Filename) { + CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards]; + std::unique_lock Lock(Shard.CacheLock); + return Shard.StatCache.try_emplace(Filename).first->second; +} + +InsertResult +DependencyScanningFilesystemSharedCache::getOriginalContents( + StringRef Filename, llvm::sys::fs::UniqueID UID) { + CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards]; + std::unique_lock Lock(Shard.CacheLock); + auto InsertRes = Shard.OriginalContentsCache.insert({UID, OriginalContents{}}); + return makeInsertResult(InsertRes.first->second, InsertRes.second); +} + +InsertResult +DependencyScanningFilesystemSharedCache::getMinimizedContents( + StringRef Filename, llvm::sys::fs::UniqueID UID) { + CacheShard &Shard = CacheShards[llvm::hash_value(Filename) % NumShards]; + std::unique_lock Lock(Shard.CacheLock); + auto InsertRes = Shard.MinimizedContentsCache.insert({UID, MinimizedContents{}}); + return makeInsertResult(InsertRes.first->second, InsertRes.second); +} + +const MaybeStat * +DependencyScanningFilesystemLocalCache::findStat(StringRef Filename) { + auto It = StatCache.find(Filename); + return It != StatCache.end() ? It->getValue() : nullptr; +} + +const MaybeStat * +DependencyScanningFilesystemLocalCache::insertStat(StringRef Filename, + const MaybeStat *Stat) { + return StatCache.insert({Filename, Stat}).first->second; +} + +const OriginalContents * +DependencyScanningFilesystemLocalCache::findOriginalContents( + llvm::sys::fs::UniqueID UID) { + auto It = OriginalContentsCache.find(UID); + return It != OriginalContentsCache.end() ? It->second : nullptr; +} + +const OriginalContents * +DependencyScanningFilesystemLocalCache::insertOriginalContents( + llvm::sys::fs::UniqueID UID, const OriginalContents *Contents) { + return OriginalContentsCache.insert({UID, Contents}).first->second; } -DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & -DependencyScanningFilesystemSharedCache::get(StringRef Key, bool Minimized) { - SingleCache &Cache = Minimized ? CacheMinimized : CacheOriginal; - return Cache.get(Key); +const MinimizedContents * +DependencyScanningFilesystemLocalCache::findMinimizedContents( + llvm::sys::fs::UniqueID UID) { + auto It = MinimizedContentsCache.find(UID); + return It != MinimizedContentsCache.end() ? It->second : nullptr; +} + +const MinimizedContents * +DependencyScanningFilesystemLocalCache::insertMinimizedContents( + llvm::sys::fs::UniqueID UID, const MinimizedContents *Contents) { + return MinimizedContentsCache.insert({UID, Contents}).first->second; } /// Whitelist file extensions that should be minimized, treating no extension as @@ -167,66 +205,126 @@ return !NotToBeMinimized.contains(Filename); } -CachedFileSystemEntry DependencyScanningWorkerFilesystem::createFileSystemEntry( - llvm::ErrorOr &&MaybeStatus, StringRef Filename, - bool ShouldMinimize) { - if (!MaybeStatus) - return CachedFileSystemEntry(MaybeStatus.getError()); +void DependencyScanningWorkerFilesystem::ensureLocked( + StringRef Path, SharedStat *&SS, + llvm::Optional> &Lock) { + if (!SS) + SS = &SharedCache.getStat(Path); + if (!Lock) + Lock.emplace(SS->Mutex); + assert(Lock->mutex() == &SS->Mutex && "Locked the wrong mutex"); +} + +const MaybeStat *DependencyScanningWorkerFilesystem::getOrCreateLocalStat( + StringRef Path, SharedStat *&SS, + llvm::Optional> &Lock) { + if (const MaybeStat *LocalStat = Cache.findStat(Path)) + return LocalStat; + + bool CacheStatFailures = shouldCacheStatFailures(Path); - if (MaybeStatus->isDirectory()) - return CachedFileSystemEntry::createDirectoryEntry(std::move(*MaybeStatus)); + ensureLocked(Path, SS, Lock); - return CachedFileSystemEntry::createFileEntry(Filename, getUnderlyingFS(), - ShouldMinimize); + // Call stat if necessary. + if (!SS->isValid() || (!SS->Stat && !CacheStatFailures)) { + // HACK: We need to always restat non source files if the stat fails. + // This is because Clang first looks up the module cache and module + // files before building them, and then looks for them again. If we + // cache the stat failure, it won't see them the second time. + auto MaybeStatus = getUnderlyingFS().status(Path); + SS->Stat = std::move(MaybeStatus); + } + // Update local cache if possible. + return CacheStatFailures ? Cache.insertStat(Path, &SS->Stat) : &SS->Stat; } -llvm::ErrorOr -DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( - const StringRef Filename) { - bool ShouldMinimize = shouldMinimize(Filename); - - if (const auto *Entry = Cache.getCachedEntry(Filename, ShouldMinimize)) - return Entry; - - // FIXME: Handle PCM/PCH files. - // FIXME: Handle module map files. - - DependencyScanningFilesystemSharedCache::SharedFileSystemEntry - &SharedCacheEntry = SharedCache.get(Filename, ShouldMinimize); - const CachedFileSystemEntry *Result; - { - std::unique_lock LockGuard(SharedCacheEntry.ValueLock); - CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; - - if (!CacheEntry.isValid()) { - auto MaybeStatus = getUnderlyingFS().status(Filename); - if (!MaybeStatus && !shouldCacheStatFailures(Filename)) - // HACK: We need to always restat non source files if the stat fails. - // This is because Clang first looks up the module cache and module - // files before building them, and then looks for them again. If we - // cache the stat failure, it won't see them the second time. - return MaybeStatus.getError(); - CacheEntry = createFileSystemEntry(std::move(MaybeStatus), Filename, - ShouldMinimize); - } +const OriginalContents * +DependencyScanningWorkerFilesystem::getOrCreateLocalOriginalContents( + StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat, + llvm::Optional> &Lock) { + if (const OriginalContents *LocalOriginalContents = + Cache.findOriginalContents(UID)) + return LocalOriginalContents; + + ensureLocked(Filename, SharedStat, Lock); - Result = &CacheEntry; + auto SharedOriginalContents = SharedCache.getOriginalContents(Filename, UID); + // Read the file if necessary. + if (SharedOriginalContents.WasInserted) + SharedStat->Stat = + readFile(Filename, getUnderlyingFS(), SharedOriginalContents.Value); + + return Cache.insertOriginalContents(UID, &SharedOriginalContents.Value); +} + +const MinimizedContents * +DependencyScanningWorkerFilesystem::getOrCreateLocalMinimizedContents( + StringRef Filename, llvm::sys::fs::UniqueID UID, SharedStat *&SharedStat, + llvm::Optional> &Lock) { + if (const MinimizedContents *LocalMinimizedContents = + Cache.findMinimizedContents(UID)) + return LocalMinimizedContents; + + const OriginalContents *LocalOriginalContents = + getOrCreateLocalOriginalContents(Filename, UID, SharedStat, Lock); + + ensureLocked(Filename, SharedStat, Lock); + + auto SharedMinimizedContents = + SharedCache.getMinimizedContents(Filename, UID); + if (SharedMinimizedContents.WasInserted) + if (minimizeFile(*LocalOriginalContents, + SharedMinimizedContents.Value.Contents, + SharedMinimizedContents.Value.PPSkippedRangeMapping)) + // Use the original file contents if minimization failed. + SharedMinimizedContents.Value.Contents = *LocalOriginalContents; + return &SharedMinimizedContents.Value; +} + +static llvm::ErrorOr +withNewSize(const llvm::ErrorOr &Status, uint64_t NewSize) { + if (!Status) + return Status; + return llvm::vfs::Status::copyWithNewSize(*Status, NewSize); +} + +llvm::ErrorOr +DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry( + StringRef Filename) { + SharedStat *SharedStat = nullptr; + llvm::Optional> GuardLock; + + const MaybeStat *LocalStat = + getOrCreateLocalStat(Filename, SharedStat, GuardLock); + if (!(*LocalStat)) + return LocalStat->getError(); + if ((*LocalStat)->isDirectory()) + return FileSystemEntryResult(*LocalStat, nullptr, nullptr); + + llvm::sys::fs::UniqueID UID = (*LocalStat)->getUniqueID(); + + if (!shouldMinimize(Filename)) { + const OriginalContents *LocalOriginalContents = + getOrCreateLocalOriginalContents(Filename, UID, SharedStat, GuardLock); + return FileSystemEntryResult(*LocalStat, LocalOriginalContents, nullptr); } - // Store the result in the local cache. - Cache.setCachedEntry(Filename, ShouldMinimize, Result); - return Result; + const MinimizedContents *LocalMinimizedContents = + getOrCreateLocalMinimizedContents(Filename, UID, SharedStat, GuardLock); + return FileSystemEntryResult( + withNewSize(*LocalStat, LocalMinimizedContents->Contents.size()), + &LocalMinimizedContents->Contents, + &LocalMinimizedContents->PPSkippedRangeMapping); } llvm::ErrorOr DependencyScanningWorkerFilesystem::status(const Twine &Path) { SmallString<256> OwnedFilename; StringRef Filename = Path.toStringRef(OwnedFilename); - const llvm::ErrorOr Result = - getOrCreateFileSystemEntry(Filename); + auto Result = getOrCreateFileSystemEntry(Filename); if (!Result) return Result.getError(); - return (*Result)->getStatus(); + return Result->Status; } namespace { @@ -240,7 +338,7 @@ : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} static llvm::ErrorOr> - create(const CachedFileSystemEntry *Entry, + create(const FileSystemEntryResult &Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings); llvm::ErrorOr status() override { return Stat; } @@ -261,23 +359,21 @@ } // end anonymous namespace llvm::ErrorOr> MinimizedVFSFile::create( - const CachedFileSystemEntry *Entry, + const FileSystemEntryResult &Entry, ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) { - if (Entry->isDirectory()) + if (Entry.Status->isDirectory()) return llvm::ErrorOr>( std::make_error_code(std::errc::is_a_directory)); - llvm::ErrorOr Contents = Entry->getContents(); - if (!Contents) - return Contents.getError(); auto Result = std::make_unique( - llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), + llvm::MemoryBuffer::getMemBuffer(*Entry.Contents, Entry.Status->getName(), /*RequiresNullTerminator=*/false), - *Entry->getStatus()); - if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings) + *Entry.Status); + + if (Entry.PPSkippedRangeMapping && !Entry.PPSkippedRangeMapping->empty() && + PPSkipMappings) (*PPSkipMappings)[Result->Buffer->getBufferStart()] = - &Entry->getPPSkippedRangeMapping(); - return llvm::ErrorOr>( - std::unique_ptr(std::move(Result))); + Entry.PPSkippedRangeMapping; + return llvm::ErrorOr>(std::move(Result)); } llvm::ErrorOr> @@ -285,9 +381,10 @@ SmallString<256> OwnedFilename; StringRef Filename = Path.toStringRef(OwnedFilename); - const llvm::ErrorOr Result = - getOrCreateFileSystemEntry(Filename); + auto Result = getOrCreateFileSystemEntry(Filename); if (!Result) return Result.getError(); + if (!Result->Status) + return Result->Status.getError(); return MinimizedVFSFile::create(Result.get(), PPSkipMappings); } diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp --- a/clang/unittests/Tooling/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScannerTest.cpp @@ -205,9 +205,11 @@ } namespace dependencies { -TEST(DependencyScanningFilesystem, IgnoredFilesHaveSeparateCache) { +TEST(DependencyScanningFilesystem, IgnoredFilesAreCachedSeparately1) { auto VFS = llvm::makeIntrusiveRefCnt(); - VFS->addFile("/mod.h", 0, llvm::MemoryBuffer::getMemBuffer("// hi there!\n")); + VFS->addFile("/mod.h", 0, + llvm::MemoryBuffer::getMemBuffer("#include \n" + "// hi there!\n")); DependencyScanningFilesystemSharedCache SharedCache; auto Mappings = std::make_unique(); @@ -223,14 +225,35 @@ auto StatusFull3 = DepFS.status("/mod.h"); EXPECT_TRUE(StatusMinimized0); - EXPECT_EQ(StatusMinimized0->getSize(), 0u); + EXPECT_EQ(StatusMinimized0->getSize(), 17u); EXPECT_TRUE(StatusFull1); - EXPECT_EQ(StatusFull1->getSize(), 13u); + EXPECT_EQ(StatusFull1->getSize(), 30u); EXPECT_TRUE(StatusMinimized2); - EXPECT_EQ(StatusMinimized2->getSize(), 0u); + EXPECT_EQ(StatusMinimized2->getSize(), 17u); EXPECT_TRUE(StatusFull3); - EXPECT_EQ(StatusFull3->getSize(), 13u); + EXPECT_EQ(StatusFull3->getSize(), 30u); +} + +TEST(DependencyScanningFilesystem, IgnoredFilesAreCachedSeparately2) { + auto VFS = llvm::makeIntrusiveRefCnt(); + VFS->addFile("/mod.h", 0, + llvm::MemoryBuffer::getMemBuffer("#include \n" + "// hi there!\n")); + + DependencyScanningFilesystemSharedCache SharedCache; + auto Mappings = std::make_unique(); + DependencyScanningWorkerFilesystem DepFS(SharedCache, VFS, Mappings.get()); + + DepFS.disableMinimization("/mod.h"); + auto StatusFull0 = DepFS.status("/mod.h"); + DepFS.enableMinimizationOfAllFiles(); + auto StatusMinimized1 = DepFS.status("/mod.h"); + + EXPECT_TRUE(StatusFull0); + EXPECT_TRUE(StatusMinimized1); + EXPECT_EQ(StatusFull0->getSize(), 30u); + EXPECT_EQ(StatusMinimized1->getSize(), 17u); } } // end namespace dependencies diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -64,6 +64,8 @@ uint64_t Size, llvm::sys::fs::file_type Type, llvm::sys::fs::perms Perms); + /// Get a copy of a Status with a different size. + static Status copyWithNewSize(const Status &In, uint64_t NewSize); /// Get a copy of a Status with a different name. static Status copyWithNewName(const Status &In, const Twine &NewName); static Status copyWithNewName(const llvm::sys::fs::file_status &In, diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -75,6 +75,12 @@ : Name(Name.str()), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size), Type(Type), Perms(Perms) {} +Status Status::copyWithNewSize(const Status &In, uint64_t NewSize) { + return Status(In.getName(), In.getUniqueID(), In.getLastModificationTime(), + In.getUser(), In.getGroup(), NewSize, In.getType(), + In.getPermissions()); +} + Status Status::copyWithNewName(const Status &In, const Twine &NewName) { return Status(NewName, In.getUniqueID(), In.getLastModificationTime(), In.getUser(), In.getGroup(), In.getSize(), In.getType(),