Prepare ground for re-lexing modular headers.

[alexfh]

Summary

When a clang tool runs on a translation unit that uses modular headers, no
PPCallbacks will be invoked for the code in the modular headers (since they are
not parsed, but loaded as AST). Since some tools depend on PPCallbacks for all
transitively included headers, a solution is to re-lex the modular headers
from the sources stored along with AST inside module files. This patch makes it
possible to re-attach PPCallbacks from the compiler's Preprocessor to a custom
one that will be used to re-lex all transitively included headers. It also
updates clang-tidy checks to be compatible with this mode of operation.

[usaxena95]

+cc myself

[alexfh]

Friendly ping.

[sammccall]

clang/include/clang/Lex/PPCallbacks.h

346

This seems pretty tangly from a layering point of view, giving each object a reference to the other. I always find it hard to reason about ownership in such cases.

What about lifecycle methods beginPreprocessing(Preprocessor*) and endPreprocessing(Preprocessor*)? It will look similar in practice: subclasses that need it can still track the PP instance, but the circular references will only be there when needed, and will be explicit.

[alexfh]

clang/include/clang/Lex/PPCallbacks.h

346

We can do that as well. However, adding another couple of overrides to each PPCallbacks implementation will result in quite some boilerplate. How would you look at creating an intermediate class in the PPCallbacks -> <users' PPCallbacks handlers> hierarchy that will handle beginPreprocessing/endPreprocessing and expose the preprocessor pointer via a protected getPreprocessor() method?

[sammccall]

clang/include/clang/Lex/PPCallbacks.h

346

I think that would probably obscure the intent, I really think this is subtle enough to be explicit, and would be only about as boilerplatey as today (the PP pointer is set by a method rather than in the constructor).

But I'm more concerned about the interfaces in Lex/ than the specific callsites in clang-tidy, so if such a helper class were private to clang-tidy, that seems ok.

[alexfh]

I've found an alternative way to deal with PPCallbacks when analyzing code in modular mode. See https://reviews.llvm.org/D59528. Abandoning this revision.

clang-tools-extra/clang-tidy/bugprone/MacroParenthesesCheck.cpp

 
 namespace clang {
 namespace tidy {
 namespace bugprone {
 
 namespace {
 class MacroParenthesesPPCallbacks : public PPCallbacks {
 public:
-  MacroParenthesesPPCallbacks(Preprocessor *PP, MacroParenthesesCheck *Check)
-      : PP(PP), Check(Check) {}
+  MacroParenthesesPPCallbacks(MacroParenthesesCheck *Check) : Check(Check) {}
 
   void MacroDefined(const Token &MacroNameTok,
                     const MacroDirective *MD) override {
     replacementList(MacroNameTok, MD->getMacroInfo());
     argument(MacroNameTok, MD->getMacroInfo());
   }
 
 private:
   /// Replacement list with calculations should be enclosed in parentheses.
   void replacementList(const Token &MacroNameTok, const MacroInfo *MI);
 
   /// Arguments should be enclosed in parentheses.
   void argument(const Token &MacroNameTok, const MacroInfo *MI);
 
-  Preprocessor *PP;
   MacroParenthesesCheck *Check;
 };
 } // namespace
 
 /// Is argument surrounded properly with parentheses/braces/squares/commas?
 static bool isSurroundedLeft(const Token &T) {
   return T.isOneOf(tok::l_paren, tok::l_brace, tok::l_square, tok::comma,
                    tok::semi);
if (Tok.isOneOf(tok::l_paren, tok::l_brace, tok::l_square)) {
 
       Loc = Tok.getLocation();
     }
   }
   if (Loc.isValid()) {
     const Token &Last = *(MI->tokens_end() - 1);
     Check->diag(Loc, "macro replacement list should be enclosed in parentheses")
         << FixItHint::CreateInsertion(MI->tokens_begin()->getLocation(), "(")
-        << FixItHint::CreateInsertion(Last.getLocation().getLocWithOffset(
-                                          PP->getSpelling(Last).length()),
-                                      ")");
+        << FixItHint::CreateInsertion(
+               Last.getLocation().getLocWithOffset(
+                   getPreprocessor()->getSpelling(Last).length()),
   }
 }
 
 void MacroParenthesesPPCallbacks::argument(const Token &MacroNameTok,
                                            const MacroInfo *MI) {
 
   // Skip variable declaration.
   bool VarDecl = possibleVarDecl(MI, MI->tokens_begin());
if (Prev.is(tok::l_paren) && Next.is(tok::star) &&
         TI + 2 != MI->tokens_end() && (TI + 2)->is(tok::r_paren))
       continue;
 
     // Assignment/return, i.e. '=x;' or 'return x;'.
     if (Prev.isOneOf(tok::equal, tok::kw_return) && Next.is(tok::semi))
       continue;
 
     // C++ template parameters.
-    if (PP->getLangOpts().CPlusPlus && Prev.isOneOf(tok::comma, tok::less) &&
+    if (getPreprocessor()->getLangOpts().CPlusPlus &&
+        Prev.isOneOf(tok::comma, tok::less) &&
         Next.isOneOf(tok::comma, tok::greater))
       continue;
 
     // Namespaces.
     if (Prev.is(tok::kw_namespace))
       continue;
 
     // Variadic templates
     if (MI->isVariadic())
       continue;
 
     Check->diag(Tok.getLocation(), "macro argument should be enclosed in "
                                    "parentheses")
         << FixItHint::CreateInsertion(Tok.getLocation(), "(")
-        << FixItHint::CreateInsertion(Tok.getLocation().getLocWithOffset(
-                                          PP->getSpelling(Tok).length()),
-                                      ")");
+        << FixItHint::CreateInsertion(
+               Tok.getLocation().getLocWithOffset(
+                   getPreprocessor()->getSpelling(Tok).length()),
   }
 }
 
 void MacroParenthesesCheck::registerPPCallbacks(CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      llvm::make_unique<MacroParenthesesPPCallbacks>(
-          &Compiler.getPreprocessor(), this));
+      llvm::make_unique<MacroParenthesesPPCallbacks>(this));
 }
 
 } // namespace bugprone
 } // namespace tidy
 } // namespace clang

clang-tools-extra/clang-tidy/bugprone/MacroRepeatedSideEffectsCheck.cpp

 
 namespace clang {
 namespace tidy {
 namespace bugprone {
 
 namespace {
 class MacroRepeatedPPCallbacks : public PPCallbacks {
 public:
-  MacroRepeatedPPCallbacks(ClangTidyCheck &Check, Preprocessor &PP)
-      : Check(Check), PP(PP) {}
+  MacroRepeatedPPCallbacks(ClangTidyCheck &Check) : Check(Check) {}
 
   void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
                     SourceRange Range, const MacroArgs *Args) override;
 
 private:
   ClangTidyCheck &Check;
-  Preprocessor &PP;
 
   unsigned countArgumentExpansions(const MacroInfo *MI,
                                    const IdentifierInfo *Arg) const;
 
   bool hasSideEffects(const Token *ResultArgToks) const;
 };
 } // End of anonymous namespace.
 
if (TII->getBuiltinID() == Builtin::BI__builtin_constant_p) {
       FoundBuiltin = true;
       SkipParen = true;
       continue;
     }
 
     // If another macro is found within the macro definition, skip the macro
     // and the eventual arguments.
     if (TII->hasMacroDefinition()) {
-      const MacroInfo *M = PP.getMacroDefinition(TII).getMacroInfo();
+      const MacroInfo *M =
+          getPreprocessor()->getMacroDefinition(TII).getMacroInfo();
       if (M != nullptr && M->isFunctionLike())
         SkipParen = true;
       continue;
     }
 
     // Count argument.
     if (TII == Arg) {
       Current++;
if (ResultArgToks->isOneOf(tok::plusplus, tok::minusminus))
       return true;
   }
   return false;
 }
 
 void MacroRepeatedSideEffectsCheck::registerPPCallbacks(
     CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      ::llvm::make_unique<MacroRepeatedPPCallbacks>(
-          *this, Compiler.getPreprocessor()));
+      ::llvm::make_unique<MacroRepeatedPPCallbacks>(*this));
 }
 
 } // namespace bugprone
 } // namespace tidy
 } // namespace clang

clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.cpp

static bool isGoogletestTestMacro(StringRef MacroName) {
   return MacroNames.find(MacroName) != MacroNames.end();
 }
 
 namespace {
 
 class AvoidUnderscoreInGoogletestNameCallback : public PPCallbacks {
 public:
   AvoidUnderscoreInGoogletestNameCallback(
-      Preprocessor *PP, AvoidUnderscoreInGoogletestNameCheck *Check)
-      : PP(PP), Check(Check) {}
+      AvoidUnderscoreInGoogletestNameCheck *Check)
+      : Check(Check) {}
 
   // Detects expansions of the TEST, TEST_F, TEST_P, TYPED_TEST, TYPED_TEST_P
   // macros and checks that their arguments do not have any underscores.
   void MacroExpands(const Token &MacroNameToken,
                     const MacroDefinition &MacroDefinition, SourceRange Range,
                     const MacroArgs *Args) override {
     IdentifierInfo *NameIdentifierInfo = MacroNameToken.getIdentifierInfo();
     if (!NameIdentifierInfo)
       return;
     StringRef MacroName = NameIdentifierInfo->getName();
     if (!isGoogletestTestMacro(MacroName) || !Args ||
         Args->getNumMacroArguments() < 2)
       return;
     const Token *TestCaseNameToken = Args->getUnexpArgument(0);
     const Token *TestNameToken = Args->getUnexpArgument(1);
     if (!TestCaseNameToken || !TestNameToken)
       return;
-    std::string TestCaseName = PP->getSpelling(*TestCaseNameToken);
+    std::string TestCaseName =
+        getPreprocessor()->getSpelling(*TestCaseNameToken);
     if (TestCaseName.find('_') != std::string::npos)
       Check->diag(TestCaseNameToken->getLocation(),
                   "avoid using \"_\" in test case name \"%0\" according to "
                   "Googletest FAQ")
           << TestCaseName;
 
-    std::string TestNameMaybeDisabled = PP->getSpelling(*TestNameToken);
+    std::string TestNameMaybeDisabled =
+        getPreprocessor()->getSpelling(*TestNameToken);
     StringRef TestName = TestNameMaybeDisabled;
     TestName.consume_front(kDisabledTestPrefix);
     if (TestName.contains('_'))
       Check->diag(TestNameToken->getLocation(),
                   "avoid using \"_\" in test name \"%0\" according to "
                   "Googletest FAQ")
           << TestName;
   }
 
 private:
-  Preprocessor *PP;
   AvoidUnderscoreInGoogletestNameCheck *Check;
 };
 
 } // namespace
 
 void AvoidUnderscoreInGoogletestNameCheck::registerPPCallbacks(
     CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      llvm::make_unique<AvoidUnderscoreInGoogletestNameCallback>(
-          &Compiler.getPreprocessor(), this));
+      llvm::make_unique<AvoidUnderscoreInGoogletestNameCallback>(this));
 }
 
 } // namespace readability
 } // namespace google
 } // namespace tidy
 } // namespace clang

clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp

 
 #undef NAMING_KEYS
 // clang-format on
 
 namespace {
 /// Callback supplies macros to IdentifierNamingCheck::checkMacro
 class IdentifierNamingCheckPPCallbacks : public PPCallbacks {
 public:
-  IdentifierNamingCheckPPCallbacks(Preprocessor *PP,
-                                   IdentifierNamingCheck *Check)
-      : PP(PP), Check(Check) {}
+  IdentifierNamingCheckPPCallbacks(IdentifierNamingCheck *Check)
+      : Check(Check) {}
 
   /// MacroDefined calls checkMacro for macros in the main file
   void MacroDefined(const Token &MacroNameTok,
                     const MacroDirective *MD) override {
-    Check->checkMacro(PP->getSourceManager(), MacroNameTok, MD->getMacroInfo());
+    Check->checkMacro(getPreprocessor()->getSourceManager(), MacroNameTok,
+                      MD->getMacroInfo());
   }
 
   /// MacroExpands calls expandMacro for macros in the main file
   void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
                     SourceRange /*Range*/,
                     const MacroArgs * /*Args*/) override {
     Check->expandMacro(MacroNameTok, MD.getMacroInfo());
   }
 
 private:
-  Preprocessor *PP;
   IdentifierNamingCheck *Check;
 };
 } // namespace
 
 IdentifierNamingCheck::IdentifierNamingCheck(StringRef Name,
                                              ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context) {
   auto const fromString = [](StringRef Str) {
void IdentifierNamingCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(cxxConstructorDecl().bind("classRef"), this);
   Finder->addMatcher(cxxDestructorDecl().bind("classRef"), this);
   Finder->addMatcher(typeLoc().bind("typeLoc"), this);
   Finder->addMatcher(nestedNameSpecifierLoc().bind("nestedNameLoc"), this);
 }
 
 void IdentifierNamingCheck::registerPPCallbacks(CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      llvm::make_unique<IdentifierNamingCheckPPCallbacks>(
-          &Compiler.getPreprocessor(), this));
+      llvm::make_unique<IdentifierNamingCheckPPCallbacks>(this));
 }
 
 static bool matchesStyle(StringRef Name,
                          IdentifierNamingCheck::NamingStyle Style) {
   static llvm::Regex Matchers[] = {
       llvm::Regex("^.*$"),
       llvm::Regex("^[a-z][a-z0-9_]*$"),
       llvm::Regex("^[a-z][a-zA-Z0-9]*$"),

clang-tools-extra/clang-tidy/readability/RedundantPreprocessorCheck.cpp

struct PreprocessorEntry {
   /// Condition used after the preprocessor directive.
   std::string Condition;
 };
 
 class RedundantPreprocessorCallbacks : public PPCallbacks {
   enum DirectiveKind { DK_If = 0, DK_Ifdef = 1, DK_Ifndef = 2 };
 
 public:
-  explicit RedundantPreprocessorCallbacks(ClangTidyCheck &Check,
-                                          Preprocessor &PP)
-      : Check(Check), PP(PP),
+  explicit RedundantPreprocessorCallbacks(ClangTidyCheck &Check)
+      : Check(Check),
         WarningDescription("nested redundant %select{#if|#ifdef|#ifndef}0; "
                            "consider removing it"),
         NoteDescription("previous %select{#if|#ifdef|#ifndef}0 was here") {}
 
   void If(SourceLocation Loc, SourceRange ConditionRange,
           ConditionValueKind ConditionValue) override {
     StringRef Condition =
         Lexer::getSourceText(CharSourceRange::getTokenRange(ConditionRange),
-                             PP.getSourceManager(), PP.getLangOpts());
+                             getPreprocessor()->getSourceManager(),
+                             getPreprocessor()->getLangOpts());
     CheckMacroRedundancy(Loc, Condition, IfStack, DK_If, DK_If, true);
   }
 
   void Ifdef(SourceLocation Loc, const Token &MacroNameTok,
              const MacroDefinition &MacroDefinition) override {
-    std::string MacroName = PP.getSpelling(MacroNameTok);
+    std::string MacroName = getPreprocessor()->getSpelling(MacroNameTok);
     CheckMacroRedundancy(Loc, MacroName, IfdefStack, DK_Ifdef, DK_Ifdef, true);
     CheckMacroRedundancy(Loc, MacroName, IfndefStack, DK_Ifdef, DK_Ifndef,
                          false);
   }
 
   void Ifndef(SourceLocation Loc, const Token &MacroNameTok,
               const MacroDefinition &MacroDefinition) override {
-    std::string MacroName = PP.getSpelling(MacroNameTok);
+    std::string MacroName = getPreprocessor()->getSpelling(MacroNameTok);
     CheckMacroRedundancy(Loc, MacroName, IfndefStack, DK_Ifndef, DK_Ifndef,
                          true);
     CheckMacroRedundancy(Loc, MacroName, IfdefStack, DK_Ifndef, DK_Ifdef,
                          false);
   }
 
   void Endif(SourceLocation Loc, SourceLocation IfLoc) override {
     if (!IfStack.empty() && IfLoc == IfStack.back().Loc)
       IfStack.pop_back();
     if (!IfdefStack.empty() && IfLoc == IfdefStack.back().Loc)
       IfdefStack.pop_back();
     if (!IfndefStack.empty() && IfLoc == IfndefStack.back().Loc)
       IfndefStack.pop_back();
   }
 
 private:
   void CheckMacroRedundancy(SourceLocation Loc, StringRef MacroName,
                             SmallVector<PreprocessorEntry, 4> &Stack,
                             DirectiveKind WarningKind, DirectiveKind NoteKind,
                             bool Store) {
-    if (PP.getSourceManager().isInMainFile(Loc)) {
+    if (getPreprocessor()->getSourceManager().isInMainFile(Loc)) {
       for (const auto &Entry : Stack) {
         if (Entry.Condition == MacroName) {
           Check.diag(Loc, WarningDescription) << WarningKind;
           Check.diag(Entry.Loc, NoteDescription, DiagnosticIDs::Note)
               << NoteKind;
         }
       }
     }
 
     if (Store)
       // This is an actual directive to be remembered.
       Stack.push_back({Loc, MacroName});
   }
 
   ClangTidyCheck &Check;
-  Preprocessor &PP;
   SmallVector<PreprocessorEntry, 4> IfStack;
   SmallVector<PreprocessorEntry, 4> IfdefStack;
   SmallVector<PreprocessorEntry, 4> IfndefStack;
   const std::string WarningDescription;
   const std::string NoteDescription;
 };
 } // namespace
 
 void RedundantPreprocessorCheck::registerPPCallbacks(
     CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      ::llvm::make_unique<RedundantPreprocessorCallbacks>(
-          *this, Compiler.getPreprocessor()));
+      ::llvm::make_unique<RedundantPreprocessorCallbacks>(*this));
 }
 
 } // namespace readability
 } // namespace tidy
 } // namespace clang

clang-tools-extra/clang-tidy/utils/HeaderGuard.cpp

static std::string cleanPath(StringRef Path) {
   SmallString<256> Result = Path;
   llvm::sys::path::remove_dots(Result, true);
   return Result.str();
 }
 
 namespace {
 class HeaderGuardPPCallbacks : public PPCallbacks {
 public:
-  HeaderGuardPPCallbacks(Preprocessor *PP, HeaderGuardCheck *Check)
-      : PP(PP), Check(Check) {}
+  HeaderGuardPPCallbacks(HeaderGuardCheck *Check) : Check(Check) {}
 
   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                    SrcMgr::CharacteristicKind FileType,
                    FileID PrevFID) override {
     // Record all files we enter. We'll need them to diagnose headers without
     // guards.
-    SourceManager &SM = PP->getSourceManager();
+    SourceManager &SM = getPreprocessor()->getSourceManager();
     if (Reason == EnterFile && FileType == SrcMgr::C_User) {
       if (const FileEntry *FE = SM.getFileEntryForID(SM.getFileID(Loc))) {
         std::string FileName = cleanPath(FE->getName());
         Files[FileName] = FE;
       }
     }
   }
 
public:
 
   void Endif(SourceLocation Loc, SourceLocation IfLoc) override {
     // Record all #endif and the corresponding #ifs (including #ifndefs).
     EndIfs[IfLoc] = Loc;
   }
 
   void EndOfMainFile() override {
     // Now that we have all this information from the preprocessor, use it!
-    SourceManager &SM = PP->getSourceManager();
+    SourceManager &SM = getPreprocessor()->getSourceManager();
 
     for (const auto &MacroEntry : Macros) {
       const MacroInfo *MI = MacroEntry.second;
 
       // We use clang's header guard detection. This has the advantage of also
       // emitting a warning for cases where a pseudo header guard is found but
       // preceded by something blocking the header guard optimization.
       if (!MI->isUsedForHeaderGuard())
void EndOfMainFile() override {
     EndIfs.clear();
   }
 
   bool wouldFixEndifComment(StringRef FileName, SourceLocation EndIf,
                             StringRef HeaderGuard,
                             size_t *EndIfLenPtr = nullptr) {
     if (!EndIf.isValid())
       return false;
-    const char *EndIfData = PP->getSourceManager().getCharacterData(EndIf);
+    const char *EndIfData =
+        getPreprocessor()->getSourceManager().getCharacterData(EndIf);
     size_t EndIfLen = std::strcspn(EndIfData, "\r\n");
     if (EndIfLenPtr)
       *EndIfLenPtr = EndIfLen;
 
     StringRef EndIfStr(EndIfData, EndIfLen);
     EndIfStr = EndIfStr.substr(EndIfStr.find_first_not_of("#endif \t"));
 
     // Give up if there's an escaped newline.
void checkGuardlessHeaders() {
     // Look for header files that didn't have a header guard. Emit a warning and
     // fix-its to add the guard.
     // TODO: Insert the guard after top comments.
     for (const auto &FE : Files) {
       StringRef FileName = FE.getKey();
       if (!Check->shouldSuggestToAddHeaderGuard(FileName))
         continue;
 
-      SourceManager &SM = PP->getSourceManager();
+      SourceManager &SM = getPreprocessor()->getSourceManager();
       FileID FID = SM.translateFile(FE.getValue());
       SourceLocation StartLoc = SM.getLocForStartOfFile(FID);
       if (StartLoc.isInvalid())
         continue;
 
       std::string CPPVar = Check->getHeaderGuard(FileName);
       std::string CPPVarUnder = CPPVar + '_'; // Allow a trailing underscore.
       // If there's a macro with a name that follows the header guard convention

 
 private:
   std::vector<std::pair<Token, const MacroInfo *>> Macros;
   llvm::StringMap<const FileEntry *> Files;
   std::map<const IdentifierInfo *, std::pair<SourceLocation, SourceLocation>>
       Ifndefs;
   std::map<SourceLocation, SourceLocation> EndIfs;
 
-  Preprocessor *PP;
   HeaderGuardCheck *Check;
 };
 } // namespace
 
 void HeaderGuardCheck::registerPPCallbacks(CompilerInstance &Compiler) {
   Compiler.getPreprocessor().addPPCallbacks(
-      llvm::make_unique<HeaderGuardPPCallbacks>(&Compiler.getPreprocessor(),
-                                                this));
+      llvm::make_unique<HeaderGuardPPCallbacks>(this));
 }
 
 bool HeaderGuardCheck::shouldSuggestEndifComment(StringRef FileName) {
   return utils::isHeaderFileExtension(FileName, HeaderFileExtensions);
 }
 
 bool HeaderGuardCheck::shouldFixHeaderGuard(StringRef FileName) { return true; }
 

clang/include/clang/Lex/PPCallbacks.h

 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
   class Token;
   class IdentifierInfo;
   class MacroDefinition;
   class MacroDirective;
   class MacroArgs;
+  class Preprocessor;
 
 /// This interface provides a way to observe the actions of the
 /// preprocessor as it does its thing.
 ///
 /// Clients can define their hooks here to implement preprocessor level tools.
 class PPCallbacks {
 public:
   virtual ~PPCallbacks();
public:
   virtual void Else(SourceLocation Loc, SourceLocation IfLoc) {
   }
 
   /// Hook called whenever an \#endif is seen.
   /// \param Loc the source location of the directive.
   /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
   virtual void Endif(SourceLocation Loc, SourceLocation IfLoc) {
   }
+
+  virtual void setPreprocessor(Preprocessor *preprocessor) {
+    preprocessor_ = preprocessor;
+  }
+
+protected:
+  Preprocessor *getPreprocessor() const { return preprocessor_; }
+
+private:
+  Preprocessor *preprocessor_;
 };
 
 /// Simple wrapper class for chaining callbacks.
 class PPChainedCallbacks : public PPCallbacks {
   virtual void anchor();
   std::unique_ptr<PPCallbacks> First, Second;
 
 public:
void Else(SourceLocation Loc, SourceLocation IfLoc) override {
     Second->Else(Loc, IfLoc);
   }
 
   /// Hook called whenever an \#endif is seen.
   void Endif(SourceLocation Loc, SourceLocation IfLoc) override {
     First->Endif(Loc, IfLoc);
     Second->Endif(Loc, IfLoc);
   }
+
+  void setPreprocessor(Preprocessor *preprocessor) override {
+    PPCallbacks::setPreprocessor(preprocessor);
+    First->setPreprocessor(preprocessor);
+    Second->setPreprocessor(preprocessor);
+  }
 };
 
 }  // end namespace clang
 
 #endif

clang/include/clang/Lex/Preprocessor.h

public:
 
   /// \{
   /// Accessors for preprocessor callbacks.
   ///
   /// Note that this class takes ownership of any PPCallbacks object given to
   /// it.
   PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
   void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
+    assert(C && "Callback cannot be null.");
     if (Callbacks)
       C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
                                                 std::move(Callbacks));
+    C->setPreprocessor(this);
     Callbacks = std::move(C);
   }
+
+  std::unique_ptr<PPCallbacks> takePPCallbacks() {
+    if (Callbacks)
+      Callbacks->setPreprocessor(nullptr);
+    return std::move(Callbacks);
+  }
   /// \}
 
   bool isMacroDefined(StringRef Id) {
     return isMacroDefined(&Identifiers.get(Id));
   }
   bool isMacroDefined(const IdentifierInfo *II) {
     return II->hasMacroDefinition() &&
            (!getLangOpts().Modules || (bool)getMacroDefinition(II));