diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -33,6 +33,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/FoldingSet.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerUnion.h" @@ -48,8 +49,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -124,6 +125,7 @@ friend class VAOptDefinitionContext; friend class VariadicMacroScopeGuard; + llvm::unique_function OnToken; std::shared_ptr PPOpts; DiagnosticsEngine *Diags; LangOptions &LangOpts; @@ -997,6 +999,12 @@ } /// \} + /// Register a function that would be called on each token in the final + /// expanded token stream. + void setTokenWatcher(llvm::unique_function F) { + OnToken = std::move(F); + } + bool isMacroDefined(StringRef Id) { return isMacroDefined(&Identifiers.get(Id)); } diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h --- a/clang/include/clang/Lex/TokenLexer.h +++ b/clang/include/clang/Lex/TokenLexer.h @@ -147,6 +147,10 @@ /// preprocessor directive. bool isParsingPreprocessorDirective() const; + /// Returns true iff the TokenLexer is expanding a macro and not replaying a + /// stream of tokens. + bool isMacroExpansion() const { return Macro != nullptr; } + private: void destroy(); diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp --- a/clang/lib/Lex/Preprocessor.cpp +++ b/clang/lib/Lex/Preprocessor.cpp @@ -867,20 +867,27 @@ // We loop here until a lex function returns a token; this avoids recursion. bool ReturnedToken; bool IsNewToken = true; + // Whether to call the OnToken callback on this token. + bool Report; do { switch (CurLexerKind) { case CLK_Lexer: + Report = true; ReturnedToken = CurLexer->Lex(Result); break; case CLK_TokenLexer: + Report = CurTokenLexer->isMacroExpansion(); ReturnedToken = CurTokenLexer->Lex(Result); break; case CLK_CachingLexer: CachingLex(Result, IsNewToken); + Report = IsNewToken; ReturnedToken = true; break; case CLK_LexAfterModuleImport: - ReturnedToken = LexAfterModuleImport(Result); + LexAfterModuleImport(Result); + Report = true; + ReturnedToken = true; break; } } while (!ReturnedToken); @@ -937,6 +944,8 @@ LastTokenWasAt = Result.is(tok::at); --LexLevel; + if (OnToken && LexLevel == 0 && Report) + OnToken(Result); } /// Lex a header-name token (including one formed from header-name-tokens if