Fix PCH skipping to handle all Lexers

[mikerice]

Summary

When skipping code at the start of a file during PCH use, Preprocessor::Lex is not used since it consumes all preprocessor directives until it returns a real token. Using the specific Lexer (i.e. CurLexer->Lex) makes it possible to stop skipping after an #include or #pragma hdrstop. Previously the skipping code was only handling CurLexer, now all will be handled correctly.

Fixes: llvm.org/PR41585

[rnk]

lgtm, thanks!

lib/Lex/Preprocessor.cpp

 /// is reached, skipping continues into the main file. If the end of the
 /// main file is reached, it's a fatal error.
 void Preprocessor::SkipTokensWhileUsingPCH() {
   bool ReachedMainFileEOF = false;
   bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
   bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
   Token Tok;
   while (true) {
-    bool InPredefines = (CurLexer->getFileID() == getPredefinesFileID());
+    bool InPredefines =
+        (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
+    switch (CurLexerKind) {
+    case CLK_Lexer:
       CurLexer->Lex(Tok);
+     break;
+    case CLK_TokenLexer:
+      CurTokenLexer->Lex(Tok);
+      break;
+    case CLK_CachingLexer:
+      bool IsNewToken;
+      CachingLex(Tok, IsNewToken);
+      break;
+    case CLK_LexAfterModuleImport:
+      LexAfterModuleImport(Tok);
+      break;
+    }
     if (Tok.is(tok::eof) && !InPredefines) {
       ReachedMainFileEOF = true;
       break;
     }
     if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
       break;
     if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
       break;

test/PCH/Inputs/pch-through-macro.h

+#pragma once
+#define Source(x,y)
+#define InOut(size) Source(InOut, (size))

test/PCH/pch-through4.cpp

+// expected-no-diagnostics
+// Create PCH with #pragma hdrstop processing.
+// RUN: %clang_cc1 -verify -I %S -emit-pch -pch-through-hdrstop-create \
+// RUN:   -fms-extensions -o %t.pch -x c++-header %s
+
+// Create the PCH object
+// RUN: %clang_cc1 -verify -I %S -emit-obj -include-pch %t.pch \
+// RUN:   -pch-through-hdrstop-create -fms-extensions -o %t.obj -x c++ %s
+
+#pragma once
+#include "Inputs/pch-through-macro.h"
+void f(InOut(a) char *b, unsigned long a);

test/PCH/pch-through4a.cpp

+// expected-no-diagnostics
+// Create PCH with a through header.
+// RUN: %clang_cc1 -verify -I %S -emit-pch \
+// RUN: -pch-through-header=Inputs/pch-through1.h \
+// RUN:   -fms-extensions -o %t.pch -x c++-header %s
+
+// Create the PCH object
+// RUN: %clang_cc1 -verify -I %S -emit-obj -include-pch %t.pch \
+// RUN:   -pch-through-header=Inputs/pch-through1.h \
+// RUN:   -fms-extensions -o %t.obj -x c++ %s
+
+#define Source(x,y)
+#define InOut(size) Source(InOut, (size))
+void f(InOut(a) char *b, unsigned long a);
+#include "Inputs/pch-through1.h"
+int other;