Index: cfe/trunk/include/clang/Lex/Preprocessor.h =================================================================== --- cfe/trunk/include/clang/Lex/Preprocessor.h +++ cfe/trunk/include/clang/Lex/Preprocessor.h @@ -1077,6 +1077,24 @@ /// \brief Disable the last EnableBacktrackAtThisPos call. void CommitBacktrackedTokens(); + struct CachedTokensRange { + CachedTokensTy::size_type Begin, End; + }; + +private: + /// \brief A range of cached tokens that should be erased after lexing + /// when backtracking requires the erasure of such cached tokens. + Optional CachedTokenRangeToErase; + +public: + /// \brief Returns the range of cached tokens that were lexed since + /// EnableBacktrackAtThisPos() was previously called. + CachedTokensRange LastCachedTokenRange(); + + /// \brief Erase the range of cached tokens that were lexed since + /// EnableBacktrackAtThisPos() was previously called. + void EraseCachedTokens(CachedTokensRange TokenRange); + /// \brief Make Preprocessor re-lex the tokens that were lexed since /// EnableBacktrackAtThisPos() was previously called. void Backtrack(); Index: cfe/trunk/lib/Lex/PPCaching.cpp =================================================================== --- cfe/trunk/lib/Lex/PPCaching.cpp +++ cfe/trunk/lib/Lex/PPCaching.cpp @@ -35,6 +35,29 @@ BacktrackPositions.pop_back(); } +Preprocessor::CachedTokensRange Preprocessor::LastCachedTokenRange() { + assert(isBacktrackEnabled()); + auto PrevCachedLexPos = BacktrackPositions.back(); + return CachedTokensRange{PrevCachedLexPos, CachedLexPos}; +} + +void Preprocessor::EraseCachedTokens(CachedTokensRange TokenRange) { + assert(TokenRange.Begin <= TokenRange.End); + if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) { + // We have backtracked to the start of the token range as we want to consume + // them again. Erase the tokens only after consuming then. + assert(!CachedTokenRangeToErase); + CachedTokenRangeToErase = TokenRange; + return; + } + // The cached tokens were committed, so they should be erased now. + assert(TokenRange.End == CachedLexPos); + CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin, + CachedTokens.begin() + TokenRange.End); + CachedLexPos = TokenRange.Begin; + ExitCachingLexMode(); +} + // Make Preprocessor re-lex the tokens that were lexed since // EnableBacktrackAtThisPos() was previously called. void Preprocessor::Backtrack() { @@ -51,6 +74,13 @@ if (CachedLexPos < CachedTokens.size()) { Result = CachedTokens[CachedLexPos++]; + // Erase the some of the cached tokens after they are consumed when + // asked to do so. + if (CachedTokenRangeToErase && + CachedTokenRangeToErase->End == CachedLexPos) { + EraseCachedTokens(*CachedTokenRangeToErase); + CachedTokenRangeToErase = None; + } return; } Index: cfe/trunk/lib/Lex/Pragma.cpp =================================================================== --- cfe/trunk/lib/Lex/Pragma.cpp +++ cfe/trunk/lib/Lex/Pragma.cpp @@ -160,12 +160,23 @@ ~LexingFor_PragmaRAII() { if (InMacroArgPreExpansion) { + // When committing/backtracking the cached pragma tokens in a macro + // argument pre-expansion we want to ensure that either the tokens which + // have been committed will be removed from the cache or that the tokens + // over which we just backtracked won't remain in the cache after they're + // consumed and that the caching will stop after consuming them. + // Otherwise the caching will interfere with the way macro expansion + // works, because we will continue to cache tokens after consuming the + // backtracked tokens, which shouldn't happen when we're dealing with + // macro argument pre-expansion. + auto CachedTokenRange = PP.LastCachedTokenRange(); if (Failed) { PP.CommitBacktrackedTokens(); } else { PP.Backtrack(); OutTok = PragmaTok; } + PP.EraseCachedTokens(CachedTokenRange); } } Index: cfe/trunk/test/CodeCompletion/pragma-macro-token-caching.c =================================================================== --- cfe/trunk/test/CodeCompletion/pragma-macro-token-caching.c +++ cfe/trunk/test/CodeCompletion/pragma-macro-token-caching.c @@ -0,0 +1,18 @@ + +#define Outer(action) action + +void completeParam(int param) { + ; + Outer(__extension__({ _Pragma("clang diagnostic push") })); + param; +} + +// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:7:1 %s | FileCheck %s +// CHECK: param : [#int#]param + +void completeParamPragmaError(int param) { + Outer(__extension__({ _Pragma(2) })); // expected-error {{_Pragma takes a parenthesized string literal}} + param; +} + +// RUN: %clang_cc1 -fsyntax-only -verify -code-completion-at=%s:16:1 %s | FileCheck %s