diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -46,6 +46,7 @@ class FrontendAction; class InMemoryModuleCache; class Module; +class PPCallbacks; class Preprocessor; class Sema; class SourceManager; @@ -180,6 +181,9 @@ /// Force an output buffer. std::unique_ptr OutputStream; + /// Additional PPCallbacks to add to the created preprocessor. + std::unique_ptr AdditionalPPCallbacks; + CompilerInstance(const CompilerInstance &) = delete; void operator=(const CompilerInstance &) = delete; public: @@ -247,6 +251,10 @@ BuildGlobalModuleIndex = Build; } + /// Set the additional PP callbacks that will be added to the created + /// preprocessor. + void setAdditionalPPCallbacks(std::unique_ptr PPC); + /// } /// @name Forwarding Methods /// { diff --git a/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h b/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h --- a/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h +++ b/clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h @@ -66,6 +66,24 @@ Token(TokenKind K, int Offset) : K(K), Offset(Offset) {} }; +/// Simplified token range to track the range of a potentially skippable PP +/// directive. +struct SkippedRange { + /// Offset into the output byte stream of where the skipped directive begins. + int Offset; + + /// The number of bytes that can be skipped before the preprocessing must + /// resume. + int Length; +}; + +/// Computes the potential source ranges that can be skipped by the preprocessor +/// when skipping a directive like #if, #ifdef or #elsif. +/// +/// \returns false on success, true on error. +bool computeSkippedRanges(ArrayRef Input, + llvm::SmallVectorImpl &Range); + } // end namespace minimize_source_to_dependency_directives /// Minimize the input down to the preprocessor directives that might have diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h --- a/clang/include/clang/Lex/Lexer.h +++ b/clang/include/clang/Lex/Lexer.h @@ -265,6 +265,21 @@ /// Return the current location in the buffer. const char *getBufferLocation() const { return BufferPtr; } + /// Returns the current lexing offset. + unsigned getCurrentBufferOffset() { + assert(BufferPtr >= BufferStart && "Invalid buffer state"); + return BufferPtr - BufferStart; + } + + /// Skip over \p NumBytes bytes. + /// + /// If the skip is successful, the next token will be lexed from the new + /// offset. The lexer also assumes that we skipped to the start of the line. + /// + /// \returns true if the skip failed (new offset would have been past the + /// end of the buffer), false otherwise. + bool skipOver(unsigned NumBytes); + /// Stringify - Convert the specified string into a C string by i) escaping /// '\\' and " characters and ii) replacing newline character(s) with "\\n". /// If Charify is true, this escapes the ' character instead of ". diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h --- a/clang/include/clang/Lex/PPCallbacks.h +++ b/clang/include/clang/Lex/PPCallbacks.h @@ -370,6 +370,12 @@ /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive. virtual void Endif(SourceLocation Loc, SourceLocation IfLoc) { } + + /// This skipped range class returns an optional thing. + virtual Optional getSkippedRangeForExcludedConditionalBlock( + Preprocessor &PP, SourceLocation HashLoc, unsigned CurLexerBufferOffset) { + return None; + } }; /// Simple wrapper class for chaining callbacks. @@ -606,6 +612,16 @@ First->Endif(Loc, IfLoc); Second->Endif(Loc, IfLoc); } + + Optional getSkippedRangeForExcludedConditionalBlock( + Preprocessor &PP, SourceLocation HashLoc, + unsigned CurLexerBufferOffset) override { + if (auto I = First->getSkippedRangeForExcludedConditionalBlock( + PP, HashLoc, CurLexerBufferOffset)) + return I; + return Second->getSkippedRangeForExcludedConditionalBlock( + PP, HashLoc, CurLexerBufferOffset); + } }; } // end namespace clang diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -10,6 +10,7 @@ #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H #include "clang/Basic/LLVM.h" +#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Allocator.h" @@ -76,6 +77,12 @@ return MaybeStat->getName(); } + /// Return the mapping between location -> distance that is used to speed up + /// the block skipping in the preprocessor. + const SkippedRangeMapping &getPPSkippedRangeMapping() const { + return PPSkippedRangeMapping; + } + CachedFileSystemEntry(CachedFileSystemEntry &&) = default; CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; @@ -89,6 +96,7 @@ // Note: small size of 1 allows us to store an empty string with an implicit // null terminator without any allocations. llvm::SmallString<1> Contents; + SkippedRangeMapping PPSkippedRangeMapping; }; /// This class is a shared cache, that caches the 'stat' and 'open' calls to the @@ -133,8 +141,10 @@ public: DependencyScanningWorkerFilesystem( DependencyScanningFilesystemSharedCache &SharedCache, - IntrusiveRefCntPtr FS) - : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {} + IntrusiveRefCntPtr FS, + PreprocessorSkippedMappings &PPSkippedRanges) + : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache), + PPSkippedRanges(PPSkippedRanges) {} llvm::ErrorOr status(const Twine &Path) override; llvm::ErrorOr> @@ -159,6 +169,7 @@ /// The local cache is used by the worker thread to cache file system queries /// locally instead of querying the global cache every time. llvm::StringMap Cache; + PreprocessorSkippedMappings &PPSkippedRanges; }; } // end namespace dependencies diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -34,12 +34,15 @@ /// the invidual dependency scanning workers. class DependencyScanningService { public: - DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true); + DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true, + bool SkipExcludedPPRanges = true); ScanningMode getMode() const { return Mode; } bool canReuseFileManager() const { return ReuseFileManager; } + bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; } + DependencyScanningFilesystemSharedCache &getSharedCache() { return SharedCache; } @@ -47,6 +50,10 @@ private: const ScanningMode Mode; const bool ReuseFileManager; + /// Set to true to use the preprocessor optimization that skips excluded PP + /// ranges by bumping the buffer pointer in the lexer instead of lexing the + /// tokens in the range until reaching the corresponding directive. + const bool SkipExcludedPPRanges; /// The global file system cache. DependencyScanningFilesystemSharedCache SharedCache; }; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -14,6 +14,7 @@ #include "clang/Basic/LLVM.h" #include "clang/Frontend/PCHContainerOperations.h" #include "clang/Tooling/CompilationDatabase.h" +#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include @@ -62,6 +63,10 @@ private: IntrusiveRefCntPtr DiagOpts; std::shared_ptr PCHContainerOps; + /// The mapping that holds the active 'skip' mappings that are used by a + /// particular preprocessor. + PreprocessorSkippedMappings PPSkippedRanges; + const bool SkipExcludedPPRanges; llvm::IntrusiveRefCntPtr RealFS; /// The file system that is used by each worker when scanning for diff --git a/clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h b/clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h new file mode 100644 --- /dev/null +++ b/clang/include/clang/Tooling/DependencyScanning/PPRangeSkipping.h @@ -0,0 +1,87 @@ +//===- PPRangeSkipping.h - PP Callbacks to skip inactive range --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H +#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H + +#include "clang/Basic/LLVM.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Lex/PPCallbacks.h" +#include "llvm/ADT/DenseMap.h" +#include + +namespace clang { +namespace tooling { +namespace dependencies { + +/// A mapping from an offset into a buffer to the number of bytes that can be +/// skipped by the preprocessor when skipping over inactive preprocessor ranges. +using SkippedRangeMapping = llvm::DenseMap; + +/// Contains the currently active skipped range mappings for a particular +/// preprocessor instance. +class PreprocessorSkippedMappings { +public: + /// Clears the buffer -> skipped range mappings. + void reset() { MappingForBuffer.clear(); } + + /// Inserts a new mapping that maps from the specified memory buffer to the + /// specified skipped range mappings. + void setSkippedRanges(const llvm::MemoryBuffer *Buf, + const SkippedRangeMapping *Mapping) { + MappingForBuffer[Buf] = Mapping; + } + + /// Returns the skipped range mappings for the given buffer or null if none + /// exist. + const SkippedRangeMapping * + getSkippedRanges(const llvm::MemoryBuffer *Buf) const { + auto It = MappingForBuffer.find(Buf); + if (It != MappingForBuffer.end()) + return It->getSecond(); + return nullptr; + } + +private: + llvm::DenseMap + MappingForBuffer; +}; + +/// A PP callbacks instance that implements fast PP skipping. +class PPRangeSkippingCallbacks final : public PPCallbacks { +public: + PPRangeSkippingCallbacks(PreprocessorSkippedMappings &BufferMappings) + : BufferMappings(BufferMappings) {} + + Optional getSkippedRangeForExcludedConditionalBlock( + Preprocessor &PP, SourceLocation HashLoc, + unsigned CurLexerBufferOffset) override; + + void FileChanged(SourceLocation Loc, FileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, + FileID PrevFID) override; + +private: + PreprocessorSkippedMappings &BufferMappings; + struct FileInfo { + SourceLocation Loc; + FileID File; + const SkippedRangeMapping *PPSkippedRanges = nullptr; + + FileInfo(SourceLocation Loc) : Loc(Loc) {} + + bool isValid() const { return File.isValid(); } + }; + llvm::SmallVector FileStack; +}; + +} // end namespace dependencies +} // end namespace tooling +} // end namespace clang + +#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_PP_RANGE_SKIPPING_H diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -75,6 +75,15 @@ Invocation = std::move(Value); } +void CompilerInstance::setAdditionalPPCallbacks( + std::unique_ptr PPC) { + if (AdditionalPPCallbacks) + AdditionalPPCallbacks = std::make_unique( + std::move(AdditionalPPCallbacks), std::move(PPC)); + else + AdditionalPPCallbacks = std::move(PPC); +} + bool CompilerInstance::shouldBuildGlobalModuleIndex() const { return (BuildGlobalModuleIndex || (ModuleManager && ModuleManager->isGlobalIndexUnavailable() && @@ -457,6 +466,9 @@ /*ShowAllHeaders=*/true, /*OutputPath=*/"", /*ShowDepth=*/true, /*MSStyle=*/true); } + + if (AdditionalPPCallbacks) + PP->addPPCallbacks(std::move(AdditionalPPCallbacks)); } std::string CompilerInstance::getSpecificModuleCachePath() { diff --git a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp b/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp --- a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp +++ b/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp @@ -865,6 +865,54 @@ return Error; } +bool clang::minimize_source_to_dependency_directives::computeSkippedRanges( + ArrayRef Input, llvm::SmallVectorImpl &Range) { + struct Directive { + enum DirectiveKind { + If, // if/ifdef/ifndef + Else // elif,else + }; + int Offset; + DirectiveKind Kind; + }; + llvm::SmallVector Offsets; + for (const Token &T : Input) { + switch (T.K) { + case pp_if: + case pp_ifdef: + case pp_ifndef: + Offsets.push_back({T.Offset, Directive::If}); + break; + + case pp_elif: + case pp_else: { + if (Offsets.empty()) + return true; + int PreviousOffset = Offsets.back().Offset; + Range.push_back({PreviousOffset, T.Offset - PreviousOffset}); + Offsets.push_back({T.Offset, Directive::Else}); + break; + } + + case pp_endif: { + if (Offsets.empty()) + return true; + int PreviousOffset = Offsets.back().Offset; + Range.push_back({PreviousOffset, T.Offset - PreviousOffset}); + do { + Directive::DirectiveKind Kind = Offsets.pop_back_val().Kind; + if (Kind == Directive::If) + break; + } while (!Offsets.empty()); + break; + } + default: + break; + } + } + return false; +} + bool clang::minimizeSourceToDependencyDirectives( StringRef Input, SmallVectorImpl &Output, SmallVectorImpl &Tokens, DiagnosticsEngine *Diags, diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp --- a/clang/lib/Lex/Lexer.cpp +++ b/clang/lib/Lex/Lexer.cpp @@ -218,6 +218,15 @@ return L; } +bool Lexer::skipOver(unsigned NumBytes) { + IsAtPhysicalStartOfLine = true; + IsAtStartOfLine = true; + if ((BufferPtr + NumBytes) > BufferEnd) + return true; + BufferPtr += NumBytes; + return false; +} + template static void StringifyImpl(T &Str, char Quote) { typename T::size_type i = 0, e = Str.size(); while (i < e) { diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -396,6 +396,13 @@ // disabling warnings, etc. CurPPLexer->LexingRawMode = true; Token Tok; + if (Callbacks && HashTokenLoc.isFileID()) { + if (auto SkipLength = Callbacks->getSkippedRangeForExcludedConditionalBlock( + *this, HashTokenLoc, CurLexer->getCurrentBufferOffset())) { + // Skip to the next '#endif' / '#else' / '#elif'. + CurLexer->skipOver(*SkipLength); + } + } while (true) { CurLexer->Lex(Tok); diff --git a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt b/clang/lib/Tooling/DependencyScanning/CMakeLists.txt --- a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt +++ b/clang/lib/Tooling/DependencyScanning/CMakeLists.txt @@ -7,6 +7,7 @@ DependencyScanningFilesystem.cpp DependencyScanningService.cpp DependencyScanningWorker.cpp + PPRangeSkipping.cpp DEPENDS ClangDriverOptions diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -69,6 +69,25 @@ // Now make the null terminator implicit again, so that Clang's lexer can find // it right where the buffer ends. Result.Contents.pop_back(); + + // Compute the skipped PP ranges that speedup skipping over inactive + // preprocessor blocks. + llvm::SmallVector + SkippedRanges; + minimize_source_to_dependency_directives::computeSkippedRanges(Tokens, + SkippedRanges); + SkippedRangeMapping Mapping; + for (const auto &Range : SkippedRanges) { + if (Range.Length < 16) { + // Ignore small ranges as non-profitable. + // FIXME: This is a heuristic, its worth investigating the tradeoffs + // when it should be applied. + continue; + } + Mapping[Range.Offset] = Range.Length; + } + Result.PPSkippedRangeMapping = std::move(Mapping); + return Result; } @@ -172,14 +191,19 @@ }; llvm::ErrorOr> -createFile(const CachedFileSystemEntry *Entry) { +createFile(const CachedFileSystemEntry *Entry, + PreprocessorSkippedMappings &PPSkippedRanges) { llvm::ErrorOr Contents = Entry->getContents(); if (!Contents) return Contents.getError(); - return std::make_unique( + auto Result = std::make_unique( llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(), /*RequiresNullTerminator=*/false), *Entry->getStatus()); + if (!Entry->getPPSkippedRangeMapping().empty()) + PPSkippedRanges.setSkippedRanges(Result->getBufferPtr(), + &Entry->getPPSkippedRangeMapping()); + return Result; } } // end anonymous namespace @@ -191,7 +215,7 @@ // Check the local cache first. if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename)) - return createFile(Entry); + return createFile(Entry, PPSkippedRanges); // FIXME: Handle PCM/PCH files. // FIXME: Handle module map files. @@ -214,5 +238,5 @@ // Store the result in the local cache. setCachedEntry(Filename, Result); - return createFile(Result); + return createFile(Result, PPSkippedRanges); } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -13,5 +13,7 @@ using namespace dependencies; DependencyScanningService::DependencyScanningService(ScanningMode Mode, - bool ReuseFileManager) - : Mode(Mode), ReuseFileManager(ReuseFileManager) {} + bool ReuseFileManager, + bool SkipExcludedPPRanges) + : Mode(Mode), ReuseFileManager(ReuseFileManager), + SkipExcludedPPRanges(SkipExcludedPPRanges) {} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -66,9 +66,10 @@ public: DependencyScanningAction( StringRef WorkingDirectory, DependencyConsumer &Consumer, - llvm::IntrusiveRefCntPtr DepFS) + llvm::IntrusiveRefCntPtr DepFS, + PreprocessorSkippedMappings *PPSkippedRanges) : WorkingDirectory(WorkingDirectory), Consumer(Consumer), - DepFS(std::move(DepFS)) {} + DepFS(std::move(DepFS)), PPSkippedRanges(PPSkippedRanges) {} bool runInvocation(std::shared_ptr Invocation, FileManager *FileMgr, @@ -107,6 +108,14 @@ Compiler.setFileManager(FileMgr); Compiler.createSourceManager(*FileMgr); + // Set the PP Callbacks which should speed up excluded conditional block + // skipping in the preprocessor. + if (PPSkippedRanges) { + PPSkippedRanges->reset(); + Compiler.setAdditionalPPCallbacks( + std::make_unique(*PPSkippedRanges)); + } + // Create the dependency collector that will collect the produced // dependencies. // @@ -134,18 +143,20 @@ StringRef WorkingDirectory; DependencyConsumer &Consumer; llvm::IntrusiveRefCntPtr DepFS; + PreprocessorSkippedMappings *PPSkippedRanges; }; } // end anonymous namespace DependencyScanningWorker::DependencyScanningWorker( - DependencyScanningService &Service) { + DependencyScanningService &Service) + : SkipExcludedPPRanges(Service.canSkipExcludedPPRanges()) { DiagOpts = new DiagnosticOptions(); PCHContainerOps = std::make_shared(); RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing) DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(), - RealFS); + RealFS, PPSkippedRanges); if (Service.canReuseFileManager()) Files = new FileManager(FileSystemOptions(), RealFS); } @@ -178,7 +189,9 @@ Tool.setRestoreWorkingDir(false); Tool.setPrintErrorMessage(false); Tool.setDiagnosticConsumer(&DC); - DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS); + DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS, + SkipExcludedPPRanges ? &PPSkippedRanges + : nullptr); return !Tool.run(&Action); }); } diff --git a/clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp b/clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Tooling/DependencyScanning/PPRangeSkipping.cpp @@ -0,0 +1,71 @@ +//===- PPRangeSkipping.cpp - PP Callbacks to skip inactive range ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanning/PPRangeSkipping.h" +#include "clang/Lex/Preprocessor.h" + +using namespace clang; +using namespace tooling; +using namespace dependencies; + +Optional +PPRangeSkippingCallbacks::getSkippedRangeForExcludedConditionalBlock( + Preprocessor &PP, SourceLocation HashLoc, unsigned CurLexerBufferOffset) { + if (FileStack.empty()) + return None; + const SourceManager &SM = PP.getSourceManager(); + + // Retrieve the information about the file we're currently lexing. + auto &Info = FileStack.back(); + if (!Info.isValid()) { + std::pair F = SM.getDecomposedLoc(Info.Loc); + Info.File = F.first; + const auto *Buf = SM.getBuffer(F.first); + if (const auto *PPSkippedRanges = BufferMappings.getSkippedRanges(Buf)) { + Info.PPSkippedRanges = PPSkippedRanges; + } + } + + if (!Info.PPSkippedRanges) + return None; + + auto HashFileOffset = SM.getDecomposedLoc(HashLoc); + if (HashFileOffset.first != Info.File) + return None; + // Check if the offset of '#' is mapped in the skipped ranges. + auto It = Info.PPSkippedRanges->find(HashFileOffset.second); + if (It == Info.PPSkippedRanges->end()) { + return None; + } + unsigned BytesToSkip = It->getSecond(); + assert(CurLexerBufferOffset >= HashFileOffset.second && + "lexer is before the hash?"); + // Take into account the fact that the lexer has already advanced, so the + // number of bytes to skip must be adjusted. + unsigned LengthDiff = CurLexerBufferOffset - HashFileOffset.second; + assert(BytesToSkip >= LengthDiff && "lexer is after the skipped range?"); + return BytesToSkip - LengthDiff; +} + +void PPRangeSkippingCallbacks::FileChanged(SourceLocation Loc, + FileChangeReason Reason, + SrcMgr::CharacteristicKind FileType, + FileID PrevFID) { + switch (Reason) { + case EnterFile: + if (Loc.isValid()) + FileStack.push_back(Loc); + break; + case ExitFile: + if (!FileStack.empty()) + FileStack.pop_back(); + break; + default: + break; + } +} diff --git a/clang/test/ClangScanDeps/regular_cdb.cpp b/clang/test/ClangScanDeps/regular_cdb.cpp --- a/clang/test/ClangScanDeps/regular_cdb.cpp +++ b/clang/test/ClangScanDeps/regular_cdb.cpp @@ -12,6 +12,8 @@ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s // RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \ // RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources \ +// RUN: -skip-excluded-pp-ranges=0 | FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s // // Make sure we didn't produce any dependency files! // RUN: not cat %t.dir/regular_cdb.d diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -168,6 +168,14 @@ llvm::cl::desc("Reuse the file manager and its cache between invocations."), llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory)); +llvm::cl::opt SkipExcludedPPRanges( + "skip-excluded-pp-ranges", + llvm::cl::desc( + "Use the preprocessor optimization that skips excluded conditionals by " + "bumping the buffer pointer in the lexer instead of lexing the tokens " + "until reaching the end directive."), + llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory)); + } // end anonymous namespace int main(int argc, const char **argv) { @@ -214,7 +222,8 @@ // Print out the dependency results to STDOUT by default. SharedStream DependencyOS(llvm::outs()); - DependencyScanningService Service(ScanMode, ReuseFileManager); + DependencyScanningService Service(ScanMode, ReuseFileManager, + SkipExcludedPPRanges); #if LLVM_ENABLE_THREADS unsigned NumWorkers = NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads; diff --git a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp b/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp --- a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp +++ b/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp @@ -617,4 +617,46 @@ minimize_source_to_dependency_directives::cxx_module_decl); } +TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesBasic) { + SmallString<128> Out; + SmallVector Toks; + StringRef Source = "#ifndef GUARD\n" + "#define GUARD\n" + "void foo();\n" + "#endif\n"; + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks)); + SmallVector Ranges; + ASSERT_FALSE(computeSkippedRanges(Toks, Ranges)); + EXPECT_EQ(Ranges.size(), 1u); + EXPECT_EQ(Ranges[0].Offset, 0); + EXPECT_EQ(Ranges[0].Length, (int)Out.find("#endif")); +} + +TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesNested) { + SmallString<128> Out; + SmallVector Toks; + StringRef Source = "#ifndef GUARD\n" + "#define GUARD\n" + "#if FOO\n" + "#include hello\n" + "#elif BAR\n" + "#include bye\n" + "#endif\n" + "#else\n" + "#include nothing\n" + "#endif\n"; + ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks)); + SmallVector Ranges; + ASSERT_FALSE(computeSkippedRanges(Toks, Ranges)); + EXPECT_EQ(Ranges.size(), 4u); + EXPECT_EQ(Ranges[0].Offset, (int)Out.find("#if FOO")); + EXPECT_EQ(Ranges[0].Offset + Ranges[0].Length, (int)Out.find("#elif")); + EXPECT_EQ(Ranges[1].Offset, (int)Out.find("#elif BAR")); + EXPECT_EQ(Ranges[1].Offset + Ranges[1].Length, (int)Out.find("#endif")); + EXPECT_EQ(Ranges[2].Offset, 0); + EXPECT_EQ(Ranges[2].Length, (int)Out.find("#else")); + EXPECT_EQ(Ranges[3].Offset, (int)Out.find("#else")); + EXPECT_EQ(Ranges[3].Offset + Ranges[3].Length, (int)Out.rfind("#endif")); +} + } // end anonymous namespace