Fix evaluation of `__has_include_next` during -frewrite-includes.

[vsapsai]

Summary

__has_include_next requires correct DirectoryLookup for being
evaluated correctly. We were using Preprocessor::GetCurDirLookup() but
we were calling it after the preprocessor finished its work. And in this
case CurDirLookup is always nullptr which makes __has_include_next
behave as __has_include.

Fix by storing and using CurDirLookup when preprocessor enters a file,
not when we rewrite the includes.

rdar://problem/36305026

[vsapsai]

To simplify reviewing additions to the test, here is the corresponding output

# 23 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/rewrite-includes.c" 2
#if 0 /* expanded by -frewrite-includes */
#include "rewrite-includes9.h"
#endif /* expanded by -frewrite-includes */
# 23 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/rewrite-includes.c"
# 1 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/Inputs/rewrite-includes9.h" 1
#if (1)/*__has_include_next(<rewrite-includes9.h>)*/
#if 0 /* expanded by -frewrite-includes */
#include_next <rewrite-includes9.h>
#endif /* expanded by -frewrite-includes */
# 2 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/Inputs/rewrite-includes9.h"
# 1 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h" 1
included_line9
# 3 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/Inputs/rewrite-includes9.h" 2
#endif
# 4 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/Inputs/rewrite-includes9.h"
# 24 "/Users/vsapsai/Projects/llvm/llvm-project/clang/test/Frontend/rewrite-includes.c" 2

#if 0 /* expanded by -frewrite-includes */
#include "rewrite-includes9.h"
#endif /* expanded by -frewrite-includes */
#if (1)/*__has_include_next(<rewrite-includes9.h>)*/
#if 0 /* expanded by -frewrite-includes */
#include_next <rewrite-includes9.h>
#endif /* expanded by -frewrite-includes */
included_line9
#endif

[bkramer]

lg

[vsapsai]

Thanks for the prompt review.

clang/lib/Frontend/Rewrite/InclusionRewriter.cpp

 namespace {
 
 class InclusionRewriter : public PPCallbacks {
   /// Information about which #includes were actually performed,
   /// created by preprocessor callbacks.
   struct IncludedFile {
     FileID Id;
     SrcMgr::CharacteristicKind FileType;
-    IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
-        : Id(Id), FileType(FileType) {}
+    const DirectoryLookup *DirLookup;
+    IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType,
+                 const DirectoryLookup *DirLookup)
+        : Id(Id), FileType(FileType), DirLookup(DirLookup) {}
   };
   Preprocessor &PP; ///< Used to find inclusion directives.
   SourceManager &SM; ///< Used to read and manage source files.
   raw_ostream &OS; ///< The destination stream for rewritten contents.
   StringRef MainEOL; ///< The line ending marker to use.
   const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
   bool ShowLineMarkers; ///< Show #line markers.
   bool UseLineDirectives; ///< Use of line directives or line markers.
   /// Tracks where inclusions that change the file are found.
   std::map<unsigned, IncludedFile> FileIncludes;
   /// Tracks where inclusions that import modules are found.
   std::map<unsigned, const Module *> ModuleIncludes;
   /// Tracks where inclusions that enter modules (in a module build) are found.
   std::map<unsigned, const Module *> ModuleEntryIncludes;
   /// Used transitively for building up the FileIncludes mapping over the
   /// various \c PPCallbacks callbacks.
   SourceLocation LastInclusionLocation;
 public:
   InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
                     bool UseLineDirectives);
-  void Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
+  void Process(FileID FileId, SrcMgr::CharacteristicKind FileType,
+               const DirectoryLookup *DirLookup);
   void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
     PredefinesBuffer = Buf;
   }
   void detectMainFileEOL();
   void handleModuleBegin(Token &Tok) {
     assert(Tok.getKind() == tok::annot_module_begin);
     ModuleEntryIncludes.insert({Tok.getLocation().getRawEncoding(),
                                 (Module *)Tok.getAnnotationValue()});
void InclusionRewriter::FileChanged(SourceLocation Loc,
                                     SrcMgr::CharacteristicKind NewFileType,
                                     FileID) {
   if (Reason != EnterFile)
     return;
   if (LastInclusionLocation.isInvalid())
     // we didn't reach this file (eg: the main file) via an inclusion directive
     return;
   FileID Id = FullSourceLoc(Loc, SM).getFileID();
-  auto P = FileIncludes.insert(std::make_pair(
-      LastInclusionLocation.getRawEncoding(), IncludedFile(Id, NewFileType)));
+  auto P = FileIncludes.insert(
+      std::make_pair(LastInclusionLocation.getRawEncoding(),
+                     IncludedFile(Id, NewFileType, PP.GetCurDirLookup())));
   (void)P;
   assert(P.second && "Unexpected revisitation of the same include directive");
   LastInclusionLocation = SourceLocation();
 }
 
 /// Called whenever an inclusion is skipped due to canonical header protection
 /// macros.
 void InclusionRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
bool InclusionRewriter::HandleHasInclude(
   bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
   const DirectoryLookup *CurDir;
   const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
   SmallVector<std::pair<const FileEntry *, const DirectoryEntry *>, 1>
       Includers;
   Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
   // FIXME: Why don't we call PP.LookupFile here?
   const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
-      Filename, SourceLocation(), isAngled, nullptr, CurDir, Includers, nullptr,
+      Filename, SourceLocation(), isAngled, Lookup, CurDir, Includers, nullptr,
       nullptr, nullptr, nullptr, nullptr);
 
   FileExists = File != nullptr;
   return true;
 }
 
 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
 /// and including content of included files recursively.
 void InclusionRewriter::Process(FileID FileId,
-                                SrcMgr::CharacteristicKind FileType) {
+                                SrcMgr::CharacteristicKind FileType,
+                                const DirectoryLookup *DirLookup) {
   bool Invalid;
   const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
   assert(!Invalid && "Attempting to process invalid inclusion");
   StringRef FileName = FromFile.getBufferIdentifier();
   Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
   RawLex.SetCommentRetentionState(false);
 
   StringRef LocalEOL = DetectEOL(FromFile);
if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
               WriteImplicitModuleImport(Mod);
             else if (const IncludedFile *Inc = FindIncludeAtLocation(Loc)) {
               const Module *Mod = FindEnteredModule(Loc);
               if (Mod)
                 OS << "#pragma clang module begin "
                    << Mod->getFullModuleName(true) << "\n";
 
               // Include and recursively process the file.
-              Process(Inc->Id, Inc->FileType);
+              Process(Inc->Id, Inc->FileType, Inc->DirLookup);
 
               if (Mod)
                 OS << "#pragma clang module end /*"
                    << Mod->getFullModuleName(true) << "*/\n";
 
               // Add line marker to indicate we're returning from an included
               // file.
               LineInfoExtra = " 2";
if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
                 // Rewrite __has_include(x)
                 if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
                   if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
                                         HasFile))
                     continue;
                   // Rewrite __has_include_next(x)
                 } else if (RawToken.getIdentifierInfo()->isStr(
                                "__has_include_next")) {
-                  const DirectoryLookup *Lookup = PP.GetCurDirLookup();
-                  if (Lookup)
-                    ++Lookup;
+                  if (DirLookup)
+                    ++DirLookup;
 
-                  if (!HandleHasInclude(FileId, RawLex, Lookup, RawToken,
+                  if (!HandleHasInclude(FileId, RawLex, DirLookup, RawToken,
                                         HasFile))
                     continue;
                 } else {
                   continue;
                 }
                 // Replace the macro with (0) or (1), followed by the commented
                 // out macro for reference.
                 OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
   // nothing else matters for parsing or processing.
   PP.SetMacroExpansionOnlyInDirectives();
   do {
     PP.Lex(Tok);
     if (Tok.is(tok::annot_module_begin))
       Rewrite->handleModuleBegin(Tok);
   } while (Tok.isNot(tok::eof));
   Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
-  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User);
-  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
+  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User, nullptr);
+  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User, nullptr);
   OS->flush();
 }

clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h

+included_line9

clang/test/Frontend/Inputs/rewrite-includes9.h

+#if __has_include_next(<rewrite-includes9.h>)
+#include_next <rewrite-includes9.h>
+#endif

clang/test/Frontend/rewrite-includes.c

-// RUN: not %clang_cc1 -verify -E -frewrite-includes -DFIRST -I %S/Inputs %s -o - | FileCheck -strict-whitespace %s
-// RUN: not %clang_cc1 -verify -E -frewrite-includes -P -DFIRST -I %S/Inputs %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s
+// RUN: not %clang_cc1 -verify -E -frewrite-includes -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -strict-whitespace %s
+// RUN: not %clang_cc1 -verify -E -frewrite-includes -P -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s
 // STARTCOMPARE
 #define A(a,b) a ## b
 A(1,2)
 #include "rewrite-includes1.h"
 #ifdef FIRST
 #define HEADER "rewrite-includes3.h"
 #include HEADER
 #else
 #include "rewrite-includes4.h"
 #endif
   // indented
 #/**/include /**/ "rewrite-includes5.h" /**/ \
  
 #include "rewrite-includes6.h" // comment
  
 #include "rewrite-includes6.h" /* comment
                                   continues */
 #include "rewrite-includes7.h"
 #include "rewrite-includes7.h"
 #include "rewrite-includes8.h"
+#include "rewrite-includes9.h"
 // ENDCOMPARE
 // CHECK: {{^}}# 1 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK: {{^}}// STARTCOMPARE{{$}}
 // CHECK-NEXT: {{^}}#define A(a,b) a ## b{{$}}
 // CHECK-NEXT: {{^}}A(1,2){{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}

 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 22 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 22 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h" 1{{$}}
-// CHECK-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
+// CHECK-NEXT: {{^}}#if (0)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
 // CHECK-NEXT: {{^}}#elif (0)/*__has_include(<rewrite-includes8.hfail>)*/{{$}}
 // CHECK-NEXT: {{^}}# 3 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 4 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#if !(1)/*__has_include("rewrite-includes8.h")*/{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 6 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include "rewrite-includes9.h"{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h" 1{{$}}
+// CHECK-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes9.h>)*/{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#include_next <rewrite-includes9.h>{{$}}
+// CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h"{{$}}
+// CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)NextIncludes(/|\\\\)}}rewrite-includes9.h" 1{{$}}
+// CHECK-NEXT: {{^}}included_line9{{$}}
+// CHECK-NEXT: {{^}}# 3 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h" 2{{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}# 4 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h"{{$}}
+// CHECK-NEXT: {{^}}# 24 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}// ENDCOMPARE{{$}}
 
 // CHECKNL: {{^}}// STARTCOMPARE{{$}}
 // CHECKNL-NEXT: {{^}}#define A(a,b) a ## b{{$}}
 // CHECKNL-NEXT: {{^}}A(1,2){{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}

 // CHECKNL-NEXT: {{^}}included_line7{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes8.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
+// CHECKNL-NEXT: {{^}}#if (0)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
 // CHECKNL-NEXT: {{^}}#elif (0)/*__has_include(<rewrite-includes8.hfail>)*/{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
 // CHECKNL-NEXT: {{^}}#if !(1)/*__has_include("rewrite-includes8.h")*/{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include "rewrite-includes9.h"{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes9.h>)*/{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#include_next <rewrite-includes9.h>{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}included_line9{{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
 // CHECKNL-NEXT: {{^}}// ENDCOMPARE{{$}}