Index: cfe/trunk/include/clang/Basic/FileManager.h =================================================================== --- cfe/trunk/include/clang/Basic/FileManager.h +++ cfe/trunk/include/clang/Basic/FileManager.h @@ -231,6 +231,10 @@ llvm::vfs::FileSystem &getVirtualFileSystem() const { return *FS; } + void setVirtualFileSystem(IntrusiveRefCntPtr FS) { + this->FS = std::move(FS); + } + /// Retrieve a file entry for a "virtual" file that acts as /// if there were a file with the given name on disk. /// Index: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h =================================================================== --- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -0,0 +1,168 @@ +//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H +#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/VirtualFileSystem.h" +#include + +namespace clang { +namespace tooling { +namespace dependencies { + +/// An in-memory representation of a file system entity that is of interest to +/// the dependency scanning filesystem. +/// +/// It represents one of the following: +/// - an opened source file with minimized contents and a stat value. +/// - an opened source file with original contents and a stat value. +/// - a directory entry with its stat value. +/// - an error value to represent a file system error. +/// - a placeholder with an invalid stat indicating a not yet initialized entry. +class CachedFileSystemEntry { +public: + /// Default constructor creates an entry with an invalid stat. + CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {} + + CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {} + + /// Create an entry that represents an opened source file with minimized or + /// original contents. + /// + /// The filesystem opens the file even for `stat` calls open to avoid the + /// issues with stat + open of minimized files that might lead to a + /// mismatching size of the file. If file is not minimized, the full file is + /// read and copied into memory to ensure that it's not memory mapped to avoid + /// running out of file descriptors. + static CachedFileSystemEntry createFileEntry(StringRef Filename, + llvm::vfs::FileSystem &FS, + bool Minimize = true); + + /// Create an entry that represents a directory on the filesystem. + static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat); + + /// \returns True if the entry is valid. + bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); } + + /// \returns The error or the file's contents. + llvm::ErrorOr getContents() const { + if (!MaybeStat) + return MaybeStat.getError(); + assert(!MaybeStat->isDirectory() && "not a file"); + assert(isValid() && "not initialized"); + return StringRef(Contents); + } + + /// \returns The error or the status of the entry. + llvm::ErrorOr getStatus() const { + assert(isValid() && "not initialized"); + return MaybeStat; + } + + /// \returns the name of the file. + StringRef getName() const { + assert(isValid() && "not initialized"); + return MaybeStat->getName(); + } + + CachedFileSystemEntry(CachedFileSystemEntry &&) = default; + CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; + + CachedFileSystemEntry(const CachedFileSystemEntry &) = delete; + CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete; + +private: + llvm::ErrorOr MaybeStat; + // Store the contents in a small string to allow a + // move from the small string for the minimized contents. + // Note: small size of 1 allows us to store an empty string with an implicit + // null terminator without any allocations. + llvm::SmallString<1> Contents; +}; + +/// This class is a shared cache, that caches the 'stat' and 'open' calls to the +/// underlying real file system. +/// +/// It is sharded based on the hash of the key to reduce the lock contention for +/// the worker threads. +class DependencyScanningFilesystemSharedCache { +public: + struct SharedFileSystemEntry { + std::mutex ValueLock; + CachedFileSystemEntry Value; + }; + + DependencyScanningFilesystemSharedCache(); + + /// Returns a cache entry for the corresponding key. + /// + /// A new cache entry is created if the key is not in the cache. This is a + /// thread safe call. + SharedFileSystemEntry &get(StringRef Key); + +private: + struct CacheShard { + std::mutex CacheLock; + llvm::StringMap Cache; + }; + std::unique_ptr CacheShards; + unsigned NumShards; +}; + +/// A virtual file system optimized for the dependency discovery. +/// +/// It is primarily designed to work with source files whose contents was was +/// preprocessed to remove any tokens that are unlikely to affect the dependency +/// computation. +/// +/// This is not a thread safe VFS. A single instance is meant to be used only in +/// one thread. Multiple instances are allowed to service multiple threads +/// running in parallel. +class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { +public: + DependencyScanningWorkerFilesystem( + DependencyScanningFilesystemSharedCache &SharedCache, + IntrusiveRefCntPtr FS) + : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {} + + llvm::ErrorOr status(const Twine &Path) override; + llvm::ErrorOr> + openFileForRead(const Twine &Path) override; + + /// The set of files that should not be minimized. + llvm::StringSet<> IgnoredFiles; + +private: + void setCachedEntry(StringRef Filename, const CachedFileSystemEntry *Entry) { + bool IsInserted = Cache.try_emplace(Filename, Entry).second; + (void)IsInserted; + assert(IsInserted && "local cache is updated more than once"); + } + + const CachedFileSystemEntry *getCachedEntry(StringRef Filename) { + auto It = Cache.find(Filename); + return It == Cache.end() ? nullptr : It->getValue(); + } + + DependencyScanningFilesystemSharedCache &SharedCache; + /// The local cache is used by the worker thread to cache file system queries + /// locally instead of querying the global cache every time. + llvm::StringMap Cache; +}; + +} // end namespace dependencies +} // end namespace tooling +} // end namespace clang + +#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H Index: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h =================================================================== --- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -0,0 +1,55 @@ +//===- DependencyScanningService.h - clang-scan-deps service ===-*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H +#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H + +#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" + +namespace clang { +namespace tooling { +namespace dependencies { + +/// The mode in which the dependency scanner will operate to find the +/// dependencies. +enum class ScanningMode { + /// This mode is used to compute the dependencies by running the preprocessor + /// over + /// the unmodified source files. + CanonicalPreprocessing, + + /// This mode is used to compute the dependencies by running the preprocessor + /// over + /// the source files that have been minimized to contents that might affect + /// the dependencies. + MinimizedSourcePreprocessing +}; + +/// The dependency scanning service contains the shared state that is used by +/// the invidual dependency scanning workers. +class DependencyScanningService { +public: + DependencyScanningService(ScanningMode Mode); + + ScanningMode getMode() const { return Mode; } + + DependencyScanningFilesystemSharedCache &getSharedCache() { + return SharedCache; + } + +private: + const ScanningMode Mode; + /// The global file system cache. + DependencyScanningFilesystemSharedCache SharedCache; +}; + +} // end namespace dependencies +} // end namespace tooling +} // end namespace clang + +#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H Index: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h =================================================================== --- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -10,6 +10,7 @@ #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H #include "clang/Basic/DiagnosticOptions.h" +#include "clang/Basic/FileManager.h" #include "clang/Basic/LLVM.h" #include "clang/Frontend/PCHContainerOperations.h" #include "clang/Tooling/CompilationDatabase.h" @@ -21,6 +22,9 @@ namespace tooling { namespace dependencies { +class DependencyScanningService; +class DependencyScanningWorkerFilesystem; + /// An individual dependency scanning worker that is able to run on its own /// thread. /// @@ -29,7 +33,7 @@ /// using the regular processing run. class DependencyScanningWorker { public: - DependencyScanningWorker(); + DependencyScanningWorker(DependencyScanningService &Service); /// Print out the dependency information into a string using the dependency /// file format that is specified in the options (-MD is the default) and @@ -45,10 +49,11 @@ IntrusiveRefCntPtr DiagOpts; std::shared_ptr PCHContainerOps; + llvm::IntrusiveRefCntPtr RealFS; /// The file system that is used by each worker when scanning for /// dependencies. This filesystem persists accross multiple compiler /// invocations. - llvm::IntrusiveRefCntPtr WorkerFS; + llvm::IntrusiveRefCntPtr DepFS; }; } // end namespace dependencies Index: cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt +++ cfe/trunk/lib/Tooling/DependencyScanning/CMakeLists.txt @@ -4,6 +4,8 @@ ) add_clang_library(clangDependencyScanning + DependencyScanningFilesystem.cpp + DependencyScanningService.cpp DependencyScanningWorker.cpp DEPENDS Index: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -0,0 +1,218 @@ +//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Lex/DependencyDirectivesSourceMinimizer.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Threading.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +CachedFileSystemEntry CachedFileSystemEntry::createFileEntry( + StringRef Filename, llvm::vfs::FileSystem &FS, bool Minimize) { + // Load the file and its content from the file system. + llvm::ErrorOr> MaybeFile = + FS.openFileForRead(Filename); + if (!MaybeFile) + return MaybeFile.getError(); + llvm::ErrorOr Stat = (*MaybeFile)->status(); + if (!Stat) + return Stat.getError(); + + llvm::vfs::File &F = **MaybeFile; + llvm::ErrorOr> MaybeBuffer = + F.getBuffer(Stat->getName()); + if (!MaybeBuffer) + return MaybeBuffer.getError(); + + llvm::SmallString<1024> MinimizedFileContents; + // Minimize the file down to directives that might affect the dependencies. + const auto &Buffer = *MaybeBuffer; + SmallVector Tokens; + if (!Minimize || minimizeSourceToDependencyDirectives( + Buffer->getBuffer(), MinimizedFileContents, Tokens)) { + // Use the original file unless requested otherwise, or + // if the minimization failed. + // FIXME: Propage the diagnostic if desired by the client. + CachedFileSystemEntry Result; + Result.MaybeStat = std::move(*Stat); + Result.Contents.reserve(Buffer->getBufferSize() + 1); + Result.Contents.append(Buffer->getBufferStart(), Buffer->getBufferEnd()); + // Implicitly null terminate the contents for Clang's lexer. + Result.Contents.push_back('\0'); + Result.Contents.pop_back(); + return Result; + } + + CachedFileSystemEntry Result; + size_t Size = MinimizedFileContents.size(); + Result.MaybeStat = llvm::vfs::Status(Stat->getName(), Stat->getUniqueID(), + Stat->getLastModificationTime(), + Stat->getUser(), Stat->getGroup(), Size, + Stat->getType(), Stat->getPermissions()); + // The contents produced by the minimizer must be null terminated. + assert(MinimizedFileContents.data()[MinimizedFileContents.size()] == '\0' && + "not null terminated contents"); + // Even though there's an implicit null terminator in the minimized contents, + // we want to temporarily make it explicit. This will ensure that the + // std::move will preserve it even if it needs to do a copy if the + // SmallString still has the small capacity. + MinimizedFileContents.push_back('\0'); + Result.Contents = std::move(MinimizedFileContents); + // Now make the null terminator implicit again, so that Clang's lexer can find + // it right where the buffer ends. + Result.Contents.pop_back(); + return Result; +} + +CachedFileSystemEntry +CachedFileSystemEntry::createDirectoryEntry(llvm::vfs::Status &&Stat) { + assert(Stat.isDirectory() && "not a directory!"); + auto Result = CachedFileSystemEntry(); + Result.MaybeStat = std::move(Stat); + return Result; +} + +DependencyScanningFilesystemSharedCache:: + DependencyScanningFilesystemSharedCache() { + // This heuristic was chosen using a empirical testing on a + // reasonably high core machine (iMacPro 18 cores / 36 threads). The cache + // sharding gives a performance edge by reducing the lock contention. + // FIXME: A better heuristic might also consider the OS to account for + // the different cost of lock contention on different OSes. + NumShards = std::max(2u, llvm::hardware_concurrency() / 4); + CacheShards = llvm::make_unique(NumShards); +} + +/// Returns a cache entry for the corresponding key. +/// +/// A new cache entry is created if the key is not in the cache. This is a +/// thread safe call. +DependencyScanningFilesystemSharedCache::SharedFileSystemEntry & +DependencyScanningFilesystemSharedCache::get(StringRef Key) { + CacheShard &Shard = CacheShards[llvm::hash_value(Key) % NumShards]; + std::unique_lock LockGuard(Shard.CacheLock); + auto It = Shard.Cache.try_emplace(Key); + return It.first->getValue(); +} + +llvm::ErrorOr +DependencyScanningWorkerFilesystem::status(const Twine &Path) { + SmallString<256> OwnedFilename; + StringRef Filename = Path.toStringRef(OwnedFilename); + + // Check the local cache first. + if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) + return Entry->getStatus(); + + // FIXME: Handle PCM/PCH files. + // FIXME: Handle module map files. + + bool KeepOriginalSource = IgnoredFiles.count(Filename); + DependencyScanningFilesystemSharedCache::SharedFileSystemEntry + &SharedCacheEntry = SharedCache.get(Filename); + const CachedFileSystemEntry *Result; + { + std::unique_lock LockGuard(SharedCacheEntry.ValueLock); + CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; + + if (!CacheEntry.isValid()) { + llvm::vfs::FileSystem &FS = getUnderlyingFS(); + auto MaybeStatus = FS.status(Filename); + if (!MaybeStatus) + CacheEntry = CachedFileSystemEntry(MaybeStatus.getError()); + else if (MaybeStatus->isDirectory()) + CacheEntry = CachedFileSystemEntry::createDirectoryEntry( + std::move(*MaybeStatus)); + else + CacheEntry = CachedFileSystemEntry::createFileEntry( + Filename, FS, !KeepOriginalSource); + } + + Result = &CacheEntry; + } + + // Store the result in the local cache. + setCachedEntry(Filename, Result); + return Result->getStatus(); +} + +namespace { + +/// The VFS that is used by clang consumes the \c CachedFileSystemEntry using +/// this subclass. +class MinimizedVFSFile final : public llvm::vfs::File { +public: + MinimizedVFSFile(std::unique_ptr Buffer, + llvm::vfs::Status Stat) + : Buffer(std::move(Buffer)), Stat(std::move(Stat)) {} + + llvm::ErrorOr status() override { return Stat; } + + const llvm::MemoryBuffer *getBufferPtr() const { return Buffer.get(); } + + llvm::ErrorOr> + getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, + bool IsVolatile) override { + return std::move(Buffer); + } + + std::error_code close() override { return {}; } + +private: + std::unique_ptr Buffer; + llvm::vfs::Status Stat; +}; + +llvm::ErrorOr> +createFile(const CachedFileSystemEntry *Entry) { + llvm::ErrorOr Contents = Entry->getContents(); + if (!Contents) + return Contents.getError(); + return llvm::make_unique( + llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), + /*RequiresNullTerminator=*/false), + *Entry->getStatus()); +} + +} // end anonymous namespace + +llvm::ErrorOr> +DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) { + SmallString<256> OwnedFilename; + StringRef Filename = Path.toStringRef(OwnedFilename); + + // Check the local cache first. + if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) + return createFile(Entry); + + // FIXME: Handle PCM/PCH files. + // FIXME: Handle module map files. + + bool KeepOriginalSource = IgnoredFiles.count(Filename); + DependencyScanningFilesystemSharedCache::SharedFileSystemEntry + &SharedCacheEntry = SharedCache.get(Filename); + const CachedFileSystemEntry *Result; + { + std::unique_lock LockGuard(SharedCacheEntry.ValueLock); + CachedFileSystemEntry &CacheEntry = SharedCacheEntry.Value; + + if (!CacheEntry.isValid()) { + CacheEntry = CachedFileSystemEntry::createFileEntry( + Filename, getUnderlyingFS(), !KeepOriginalSource); + } + + Result = &CacheEntry; + } + + // Store the result in the local cache. + setCachedEntry(Filename, Result); + return createFile(Result); +} Index: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -0,0 +1,16 @@ +//===- DependencyScanningService.cpp - clang-scan-deps service ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +DependencyScanningService::DependencyScanningService(ScanningMode Mode) + : Mode(Mode) {} Index: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp =================================================================== --- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -8,9 +8,11 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/Tooling.h" using namespace clang; @@ -62,10 +64,12 @@ /// dependency scanning for the given compiler invocation. class DependencyScanningAction : public tooling::ToolAction { public: - DependencyScanningAction(StringRef WorkingDirectory, - std::string &DependencyFileContents) + DependencyScanningAction( + StringRef WorkingDirectory, std::string &DependencyFileContents, + llvm::IntrusiveRefCntPtr DepFS) : WorkingDirectory(WorkingDirectory), - DependencyFileContents(DependencyFileContents) {} + DependencyFileContents(DependencyFileContents), + DepFS(std::move(DepFS)) {} bool runInvocation(std::shared_ptr Invocation, FileManager *FileMgr, @@ -74,8 +78,6 @@ // Create a compiler instance to handle the actual work. CompilerInstance Compiler(std::move(PCHContainerOps)); Compiler.setInvocation(std::move(Invocation)); - FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; - Compiler.setFileManager(FileMgr); // Don't print 'X warnings and Y errors generated'. Compiler.getDiagnosticOpts().ShowCarets = false; @@ -84,6 +86,27 @@ if (!Compiler.hasDiagnostics()) return false; + // Use the dependency scanning optimized file system if we can. + if (DepFS) { + // FIXME: Purge the symlink entries from the stat cache in the FM. + const CompilerInvocation &CI = Compiler.getInvocation(); + // Add any filenames that were explicity passed in the build settings and + // that might be opened, as we want to ensure we don't run source + // minimization on them. + DepFS->IgnoredFiles.clear(); + for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) + DepFS->IgnoredFiles.insert(Entry.Path); + for (const auto &Entry : CI.getHeaderSearchOpts().VFSOverlayFiles) + DepFS->IgnoredFiles.insert(Entry); + + // Support for virtual file system overlays on top of the caching + // filesystem. + FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation( + CI, Compiler.getDiagnostics(), DepFS)); + } + + FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; + Compiler.setFileManager(FileMgr); Compiler.createSourceManager(*FileMgr); // Create the dependency collector that will collect the produced @@ -103,7 +126,8 @@ auto Action = llvm::make_unique(); const bool Result = Compiler.ExecuteAction(*Action); - FileMgr->clearStatCache(); + if (!DepFS) + FileMgr->clearStatCache(); return Result; } @@ -111,16 +135,19 @@ StringRef WorkingDirectory; /// The dependency file will be written to this string. std::string &DependencyFileContents; + llvm::IntrusiveRefCntPtr DepFS; }; } // end anonymous namespace -DependencyScanningWorker::DependencyScanningWorker() { +DependencyScanningWorker::DependencyScanningWorker( + DependencyScanningService &Service) { DiagOpts = new DiagnosticOptions(); PCHContainerOps = std::make_shared(); - /// FIXME: Use the shared file system from the service for fast scanning - /// mode. - WorkerFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); + RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); + if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing) + DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(), + RealFS); } llvm::Expected @@ -133,14 +160,17 @@ llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.get()); - WorkerFS->setCurrentWorkingDirectory(WorkingDirectory); - tooling::ClangTool Tool(CDB, Input, PCHContainerOps, WorkerFS); + RealFS->setCurrentWorkingDirectory(WorkingDirectory); + /// Create the tool that uses the underlying file system to ensure that any + /// file system requests that are made by the driver do not go through the + /// dependency scanning filesystem. + tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS); Tool.clearArgumentsAdjusters(); Tool.setRestoreWorkingDir(false); Tool.setPrintErrorMessage(false); Tool.setDiagnosticConsumer(&DiagPrinter); std::string Output; - DependencyScanningAction Action(WorkingDirectory, Output); + DependencyScanningAction Action(WorkingDirectory, Output, DepFS); if (Tool.run(&Action)) { return llvm::make_error(DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); Index: cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/Headers/Framework.h =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/Headers/Framework.h +++ cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/Headers/Framework.h @@ -0,0 +1,2 @@ +// This comment is stripped, so size is changed when file is opened +#define FRAMEWORK 0 Index: cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h +++ cfe/trunk/test/ClangScanDeps/Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h @@ -0,0 +1,2 @@ +// This comment is stripped when file is opened, so size will change +#define PRIV 0 Index: cfe/trunk/test/ClangScanDeps/Inputs/header_stat_before_open_cdb.json =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/header_stat_before_open_cdb.json +++ cfe/trunk/test/ClangScanDeps/Inputs/header_stat_before_open_cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/header_stat_before_open.m -iframework Inputs/frameworks", + "file": "DIR/header_stat_before_open.m" +} +] Index: cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay.yaml =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay.yaml +++ cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay.yaml @@ -0,0 +1,12 @@ +{ + 'version': 0, + 'roots': [ + { 'name': 'DIR', 'type': 'directory', + 'contents': [ + { 'name': 'not_real.h', 'type': 'file', + 'external-contents': 'DIR/Inputs/header.h' + } + ] + } + ] +} Index: cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay_cdb.json =================================================================== --- cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay_cdb.json +++ cfe/trunk/test/ClangScanDeps/Inputs/vfsoverlay_cdb.json @@ -0,0 +1,7 @@ +[ +{ + "directory": "DIR", + "command": "clang -E DIR/vfsoverlay.cpp -IInputs -ivfsoverlay DIR/vfsoverlay.yaml", + "file": "DIR/vfsoverlay.cpp" +} +] Index: cfe/trunk/test/ClangScanDeps/header_stat_before_open.m =================================================================== --- cfe/trunk/test/ClangScanDeps/header_stat_before_open.m +++ cfe/trunk/test/ClangScanDeps/header_stat_before_open.m @@ -0,0 +1,18 @@ +// RUN: rm -rf %t.dir +// RUN: rm -rf %t.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/header_stat_before_open.m +// RUN: mkdir %t.dir/Inputs +// RUN: cp -R %S/Inputs/frameworks %t.dir/Inputs/frameworks +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/header_stat_before_open_cdb.json > %t.cdb +// +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | \ +// RUN: FileCheck %s + +#include "Framework/Framework.h" +#include "Framework/PrivateHeader.h" + +// CHECK: clang-scan-deps dependency +// CHECK-NEXT: header_stat_before_open.m +// CHECK-NEXT: Inputs/frameworks/Framework.framework/Headers/Framework.h +// CHECK-NEXT: Inputs/frameworks/Framework.framework/PrivateHeaders/PrivateHeader.h Index: cfe/trunk/test/ClangScanDeps/regular_cdb.cpp =================================================================== --- cfe/trunk/test/ClangScanDeps/regular_cdb.cpp +++ cfe/trunk/test/ClangScanDeps/regular_cdb.cpp @@ -8,7 +8,9 @@ // RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h // RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources | \ +// RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s // // Make sure we didn't produce any dependency files! @@ -20,9 +22,13 @@ // as it might fail if the results for `regular_cdb.cpp` are reported before // `regular_cdb2.cpp`. // -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: FileCheck --check-prefix=CHECK1 %s +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK1 %s -// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 | \ +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess-minimized-sources | \ +// RUN: FileCheck --check-prefix=CHECK2 %s +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 -mode preprocess | \ // RUN: FileCheck --check-prefix=CHECK2 %s #include "header.h" Index: cfe/trunk/test/ClangScanDeps/vfsoverlay.cpp =================================================================== --- cfe/trunk/test/ClangScanDeps/vfsoverlay.cpp +++ cfe/trunk/test/ClangScanDeps/vfsoverlay.cpp @@ -0,0 +1,17 @@ +// RUN: rm -rf %t.dir +// RUN: rm -rf %t.cdb +// RUN: mkdir -p %t.dir +// RUN: cp %s %t.dir/vfsoverlay.cpp +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/vfsoverlay.yaml > %t.dir/vfsoverlay.yaml +// RUN: mkdir %t.dir/Inputs +// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/vfsoverlay_cdb.json > %t.cdb +// +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | \ +// RUN: FileCheck %s + +#include "not_real.h" + +// CHECK: clang-scan-deps dependency +// CHECK-NEXT: vfsoverlay.cpp +// CHECK-NEXT: Inputs{{/|\\}}header.h Index: cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp =================================================================== --- cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp +++ cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp @@ -8,6 +8,7 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Tooling/JSONCompilationDatabase.h" #include "llvm/Support/InitLLVM.h" @@ -45,9 +46,10 @@ /// /// \param Compilations The reference to the compilation database that's /// used by the clang tool. - DependencyScanningTool(const tooling::CompilationDatabase &Compilations, + DependencyScanningTool(DependencyScanningService &Service, + const tooling::CompilationDatabase &Compilations, SharedStream &OS, SharedStream &Errs) - : Compilations(Compilations), OS(OS), Errs(Errs) {} + : Worker(Service), Compilations(Compilations), OS(OS), Errs(Errs) {} /// Computes the dependencies for the given file and prints them out. /// @@ -80,6 +82,20 @@ llvm::cl::OptionCategory DependencyScannerCategory("Tool options"); +static llvm::cl::opt ScanMode( + "mode", + llvm::cl::desc("The preprocessing mode used to compute the dependencies"), + llvm::cl::values( + clEnumValN(ScanningMode::MinimizedSourcePreprocessing, + "preprocess-minimized-sources", + "The set of dependencies is computed by preprocessing the " + "source files that were minimized to only include the " + "contents that might affect the dependencies"), + clEnumValN(ScanningMode::CanonicalPreprocessing, "preprocess", + "The set of dependencies is computed by preprocessing the " + "unmodified source files")), + llvm::cl::init(ScanningMode::MinimizedSourcePreprocessing)); + llvm::cl::opt NumThreads("j", llvm::cl::Optional, llvm::cl::desc("Number of worker threads to use (default: use " @@ -136,12 +152,14 @@ SharedStream Errs(llvm::errs()); // Print out the dependency results to STDOUT by default. SharedStream DependencyOS(llvm::outs()); + + DependencyScanningService Service(ScanMode); unsigned NumWorkers = NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads; std::vector> WorkerTools; for (unsigned I = 0; I < NumWorkers; ++I) WorkerTools.push_back(llvm::make_unique( - *AdjustingCompilations, DependencyOS, Errs)); + Service, *AdjustingCompilations, DependencyOS, Errs)); std::vector WorkerThreads; std::atomic HadErrors(false);