Index: include/clang/Lex/Preprocessor.h =================================================================== --- include/clang/Lex/Preprocessor.h +++ include/clang/Lex/Preprocessor.h @@ -726,6 +726,9 @@ /// Whether tokens are being skipped until the through header is seen. bool SkippingUntilPCHThroughHeader = false; + /// XXX + bool SawHdrstop = false; + /// \{ /// Cache of macro expanders to reduce malloc traffic. enum { TokenLexerCacheSize = 8 }; @@ -1783,7 +1786,8 @@ /// This either returns the EOF token and returns true, or /// pops a level off the include stack and returns false, at which point the /// client should call lex again. - bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false); + bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false, + bool IsHdrstop = false); /// Callback invoked when the current TokenLexer hits the end of its /// token stream. @@ -2178,7 +2182,7 @@ // Pragmas. void HandlePragmaDirective(SourceLocation IntroducerLoc, - PragmaIntroducerKind Introducer); + PragmaIntroducerKind Introducer, Token &Result); public: void HandlePragmaOnce(Token &OnceTok); @@ -2190,6 +2194,7 @@ void HandlePragmaPopMacro(Token &Tok); void HandlePragmaIncludeAlias(Token &Tok); void HandlePragmaModuleBuild(Token &Tok); + void HandlePragmaHdrstop(Token &Tok); IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok); // Return true and store the first token only if any CommentHandler Index: lib/Lex/Lexer.cpp =================================================================== --- lib/Lex/Lexer.cpp +++ lib/Lex/Lexer.cpp @@ -698,6 +698,8 @@ break; } while (true); +// XXX might hook into this? + SourceLocation End; if (ActiveCommentLoc.isValid()) End = ActiveCommentLoc; // don't truncate a decl comment. Index: lib/Lex/PPDirectives.cpp =================================================================== --- lib/Lex/PPDirectives.cpp +++ lib/Lex/PPDirectives.cpp @@ -896,9 +896,11 @@ if (const IdentifierInfo *II = Result.getIdentifierInfo()) { if (II->getPPKeywordID() == tok::pp_include) return HandleIncludeDirective(HashLoc, Result); - if (II->getPPKeywordID() == tok::pp_define) - return HandleDefineDirective(Result, - /*ImmediatelyAfterHeaderGuard=*/false); + //if (II->getPPKeywordID() == tok::pp_define) + //return HandleDefineDirective(Result, + ///*ImmediatelyAfterHeaderGuard=*/false); + if (II->getPPKeywordID() == tok::pp_pragma) + return HandlePragmaDirective(HashLoc, PIK_HashPragma, Result); } DiscardUntilEndOfDirective(); } @@ -1027,7 +1029,8 @@ // C99 6.10.6 - Pragma Directive. case tok::pp_pragma: - return HandlePragmaDirective(SavedHash.getLocation(), PIK_HashPragma); + return HandlePragmaDirective(SavedHash.getLocation(), PIK_HashPragma, + Result); // GNU Extensions. case tok::pp_import: @@ -1955,7 +1958,7 @@ if (hadModuleLoaderFatalFailure()) { // With a fatal failure in the module loader, we abort parsing. Token &Result = IncludeTok; - if (CurLexer) { + if (CurLexer) { // XXX something like this Result.startToken(); CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof); CurLexer->cutOffLexing(); Index: lib/Lex/PPLexerChange.cpp =================================================================== --- lib/Lex/PPLexerChange.cpp +++ lib/Lex/PPLexerChange.cpp @@ -333,9 +333,19 @@ /// HandleEndOfFile - This callback is invoked when the lexer hits the end of /// the current file. This either returns the EOF token or pops a level off /// the include stack and keeps going. -bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) { +bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro, + bool IsHdrstop) { assert(!CurTokenLexer && "Ending a file when currently in a macro!"); + if (CurTokenLexer) +fprintf(stderr, "HandleEndOfFile CurTokenLexer\n"); + else if (CurPPLexer) { + if (CurPPLexer->getFileEntry()) +fprintf(stderr, "HandleEndOfFile %s\n", CurPPLexer->getFileEntry()->getName().str().c_str()); + else +fprintf(stderr, "HandleEndOfFile no file\n"); + }else +fprintf(stderr, "HandleEndOfFile whatnow\n"); // If we have an unclosed module region from a pragma at the end of a // module, complain and close it now. @@ -426,7 +436,7 @@ PragmaAssumeNonNullLoc = SourceLocation(); } - bool LeavingPCHThroughHeader = false; + bool LeavingPCHThroughHeader = SawHdrstop; // If this is a #include'd file, pop it off the include stack and continue // lexing the #includer file. @@ -486,9 +496,10 @@ bool FoundPCHThroughHeader = false; if (CurPPLexer && creatingPCHWithThroughHeader() && - isPCHThroughHeader( - SourceMgr.getFileEntryForID(CurPPLexer->getFileID()))) + isPCHThroughHeader(CurPPLexer->getFileEntry())) { +fprintf(stderr, "FoundPCHThroughHeader\n"); FoundPCHThroughHeader = true; + } // We're done with the #included file. RemoveTopOfLexerStack(); @@ -516,13 +527,17 @@ // processing. LeavingPCHThroughHeader = true; } else { +fprintf(stderr, "LeavingSubmodule\n"); // Client should lex another token unless we generated an EOM. return LeavingSubmodule; } } +fprintf(stderr, "hi\n"); + // If this is the end of the main file, form an EOF token. if (CurLexer) { +fprintf(stderr, "ho\n"); const char *EndPos = getCurLexerEndPos(); Result.startToken(); CurLexer->BufferPtr = EndPos; Index: lib/Lex/PPMacroExpansion.cpp =================================================================== --- lib/Lex/PPMacroExpansion.cpp +++ lib/Lex/PPMacroExpansion.cpp @@ -431,7 +431,7 @@ else if (CurPTHLexer) Val = CurPTHLexer->isNextPPTokenLParen(); else - Val = CurTokenLexer->isNextTokenLParen(); + Val = CurTokenLexer->isNextTokenLParen(); // XXX if (Val == 2) { // We have run off the end. If it's a source file we don't @@ -762,7 +762,9 @@ // Read arguments as unexpanded tokens. This avoids issues, e.g., where // an argument value in a macro could expand to ',' or '(' or ')'. - LexUnexpandedToken(Tok); + // XXX look at ( from macro too + //LexUnexpandedToken(Tok); + LexNonComment(Tok); assert(Tok.is(tok::l_paren) && "Error computing l-paren-ness?"); // ArgTokens - Build up a list of tokens that make up each argument. Each Index: lib/Lex/Pragma.cpp =================================================================== --- lib/Lex/Pragma.cpp +++ lib/Lex/Pragma.cpp @@ -127,7 +127,8 @@ /// HandlePragmaDirective - The "\#pragma" directive has been parsed. Lex the /// rest of the pragma, passing it to the registered pragma handlers. void Preprocessor::HandlePragmaDirective(SourceLocation IntroducerLoc, - PragmaIntroducerKind Introducer) { + PragmaIntroducerKind Introducer, + Token &Result) { if (Callbacks) Callbacks->PragmaDirective(IntroducerLoc, Introducer); @@ -137,8 +138,8 @@ ++NumPragma; // Invoke the first level of pragma handlers which reads the namespace id. - Token Tok; - PragmaHandlers->HandlePragma(*this, Introducer, Tok); + //Token Tok; + PragmaHandlers->HandlePragma(*this, Introducer, Result); // If the pragma handler didn't read the rest of the line, consume it now. if ((CurTokenLexer && CurTokenLexer->isParsingPreprocessorDirective()) @@ -331,7 +332,7 @@ EnterSourceFileWithLexer(TL, nullptr); // With everything set up, lex this as a #pragma directive. - HandlePragmaDirective(PragmaLoc, PIK__Pragma); + HandlePragmaDirective(PragmaLoc, PIK__Pragma, Tok); // Finally, return whatever came after the pragma directive. return Lex(Tok); @@ -380,7 +381,7 @@ EnterTokenStream(TokArray, PragmaToks.size(), true, true); // With everything set up, lex this as a #pragma directive. - HandlePragmaDirective(PragmaLoc, PIK___pragma); + HandlePragmaDirective(PragmaLoc, PIK___pragma, Tok); // Finally, return whatever came after the pragma directive. return Lex(Tok); @@ -876,6 +877,72 @@ StringRef(Start, End - Start)); } +void Preprocessor::HandlePragmaHdrstop(Token &Tok) { + // XXX do this right in HandlePragma() instead (see other LexStringLiteral() + // clients in lib/Parse/ParsePragma.cpp) + // `#pragma hdrstop` can optionally be followed by '(' string_literal ')', + // check if that's present and parse it if it's there. + Lex(Tok); + if (Tok.isNot(tok::l_paren)) { + // XXX diag if this isn't eod + //Lex(Tok); // consuming eod pops the macro off the stack + // XXX wrong already popped (?) + // XXX only do anything if we're in the main tu? + // XXX probably also for ( arg ) form below + if (usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) { +fprintf(stderr, "bar %s\n", Tok.getName()); + // XXX probably also for ( arg ) form below + SkippingUntilPCHThroughHeader = false; + } + // XXX make sure this is only called once if there are several + // pragma hdrstops or the pragma and an include? or neuter logic in + // HandleEndOfFile? + if (creatingPCHWithThroughHeader()) { +fprintf(stderr, "foo %s\n", Tok.getName()); + // XXX need to get the eof Tok to the other lexing loop + //HandleEndOfFile(Tok, /*isEndOfMacro=*/false, /*IsHdrstop=*/true); + Token& Result = Tok; + //Token Result; + if (CurLexer) { // XXX something like this + Result.startToken(); + CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof); + CurLexer->cutOffLexing(); + } else { + assert(CurPTHLexer && "#include but no current lexer set!"); + CurPTHLexer->getEOF(Result); + } + // XXX calling this seems to prevent HandleEndOfFile() being called, + // which is good in that it suppresses the "didn't see through header" + // diag, but HandleEndOfFile() might do other important things + // Looks like Preprocessor::CachingLex() doesn't call HandleEndOfFile() + // on eof tokens, only Lexer::LexEndOfFile calls it. + //EnterToken(Result); // XXX need? + SawHdrstop = true; + } + return; + } + + std::string Name; + // XXX Preprocessor::LexStringLiteral() doesn't handle wide, raw, unicode + // string literals (Lexer::LexStringLiteral() does). + if (!LexStringLiteral(Tok, Name, "pragma hdrstop", + /*AllowMacroExpansion=*/true)) + return; + + if (Tok.isNot(tok::r_paren)) { + Diag(Tok, diag::err_expected) << tok::r_paren; + return; + } + + // XXX diag if there are more tokens + + // There was a syntactically valid '(' string_literal ')' after + // `#pragma hdrstop`. After going through all this work, diag that that's + // not supported in clang. (XXX: unless it's the /Fp arg?) + // XXX: real diag (XXX warning or err?) + Diag(Tok, diag::err_pp_expected_rparen); +} + /// AddPragmaHandler - Add the specified pragma handler to the preprocessor. /// If 'Namespace' is non-null, then it is a token required to exist on the /// pragma line before the pragma string starts, e.g. "STDC" or "GCC". @@ -1220,6 +1287,16 @@ } }; +/// "\#pragma hdrstop". +struct PragmaHdrstopHandler : public PragmaHandler { + PragmaHdrstopHandler() : PragmaHandler("hdrstop") {} + + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &DepToken) override { + PP.HandlePragmaHdrstop(DepToken); + } +}; + /// "\#pragma warning(...)". MSVC's diagnostics do not map cleanly to clang's /// diagnostics, so we don't really implement this pragma. We parse it and /// ignore it to avoid -Wunknown-pragma warnings. @@ -1799,6 +1876,7 @@ if (LangOpts.MicrosoftExt) { AddPragmaHandler(new PragmaWarningHandler()); AddPragmaHandler(new PragmaIncludeAliasHandler()); + AddPragmaHandler(new PragmaHdrstopHandler()); } // Pragmas added by plugins Index: lib/Lex/Preprocessor.cpp =================================================================== --- lib/Lex/Preprocessor.cpp +++ lib/Lex/Preprocessor.cpp @@ -619,6 +619,7 @@ if (!SkippingUntilPCHThroughHeader) break; } +// XXX if (ReachedMainFileEOF) Diag(SourceLocation(), diag::err_pp_through_header_not_seen) << PPOpts->PCHThroughHeader << 1; @@ -787,6 +788,7 @@ if (!Identifier.isExpandDisabled() && MI->isEnabled()) { // C99 6.10.3p10: If the preprocessing token immediately after the // macro name isn't a '(', this macro should not be expanded. + // XXX I think this means '(' coming from a macro doesn't count if (!MI->isFunctionLike() || isNextPPTokenLParen()) return HandleMacroExpandedIdentifier(Identifier, MD); } else { Index: lib/Lex/TokenLexer.cpp =================================================================== --- lib/Lex/TokenLexer.cpp +++ lib/Lex/TokenLexer.cpp @@ -138,6 +138,7 @@ if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) return false; +// XXX // In Microsoft-compatibility mode, a comma is removed in the expansion // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is // not supported by gcc. @@ -662,6 +663,7 @@ HasLeadingSpace = false; // Handle recursive expansion! + // XXX this recurses ([cpp.rescan] i think) if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) { // Change the kind of this identifier to the appropriate token kind, e.g. // turning "for" into a keyword. @@ -890,6 +892,19 @@ // Out of tokens? if (isAtEnd()) return 2; +#if 0 + if (Tokens[CurToken].is(tok::identifier)) { + // XXX need to expand to know + // XXX several levels + IdentifierInfo *II = Tokens[CurToken].getIdentifierInfo(); + MacroDefinition Macro = PP.getMacroDefinition(II); + if (Macro.getMacroInfo()) { + if (Macro.getMacroInfo()->getReplacementToken(0).is(tok::l_paren)) + return true; + } + } + return Tokens[CurToken].is(tok::l_paren); +#endif return Tokens[CurTokenIdx].is(tok::l_paren); }