diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst --- a/clang/docs/ClangCommandLineReference.rst +++ b/clang/docs/ClangCommandLineReference.rst @@ -2326,6 +2326,12 @@ Use #line in preprocessed output +.. option:: -fnormalize-whitespace, -fno-normalize-whitespace + +Ignore the whitespace from the input file when emitting preprocessor output. It will only contain whitespace when necessary, e.g. to keep two minus signs from merging into to an increment operator. Useful with the -P option to normlize whitespace such that two files with only formatted changes are equal. + +Incompatible with -traditional-cpp. + .. option:: -fvalidate-ast-input-files-content Compute and store the hash of input files used to build an AST. Files with mismatching mtime's are considered valid if both contents is identical diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1793,6 +1793,9 @@ defm use_line_directives : BoolFOption<"use-line-directives", PreprocessorOutputOpts<"UseLineDirectives">, DefaultFalse, PosFlag, NegFlag>; +defm normalize_whitespace : BoolFOption<"normalize-whitespace", + PreprocessorOutputOpts<"NormalizeWhitespace">, DefaultFalse, + PosFlag, NegFlag>; def ffreestanding : Flag<["-"], "ffreestanding">, Group, Flags<[CC1Option]>, HelpText<"Assert that the compilation takes place in a freestanding environment">, diff --git a/clang/include/clang/Frontend/PreprocessorOutputOptions.h b/clang/include/clang/Frontend/PreprocessorOutputOptions.h --- a/clang/include/clang/Frontend/PreprocessorOutputOptions.h +++ b/clang/include/clang/Frontend/PreprocessorOutputOptions.h @@ -24,6 +24,7 @@ unsigned ShowIncludeDirectives : 1; ///< Print includes, imports etc. within preprocessed output. unsigned RewriteIncludes : 1; ///< Preprocess include directives only. unsigned RewriteImports : 1; ///< Include contents of transitively-imported modules. + unsigned NormalizeWhitespace : 1; ///< Ignore whitespace from input. public: PreprocessorOutputOptions() { @@ -36,6 +37,7 @@ ShowIncludeDirectives = 0; RewriteIncludes = 0; RewriteImports = 0; + NormalizeWhitespace = 0; } }; diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -6025,6 +6025,9 @@ if (Args.hasFlag(options::OPT_fuse_line_directives, options::OPT_fno_use_line_directives, false)) CmdArgs.push_back("-fuse-line-directives"); + if (Args.hasFlag(options::OPT_fnormalize_whitespace, + options::OPT_fno_normalize_whitespace, false)) + CmdArgs.push_back("-fnormalize-whitespace"); // -fms-extensions=0 is default. if (Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions, diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp --- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp +++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp @@ -95,14 +95,20 @@ bool DumpIncludeDirectives; bool UseLineDirectives; bool IsFirstFileEntered; + bool NormalizeWhitespace; + + Token PrevTok; + Token PrevPrevTok; + public: PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream &os, bool lineMarkers, bool defines, bool DumpIncludeDirectives, - bool UseLineDirectives) + bool UseLineDirectives, bool NormalizeWhitespace) : PP(pp), SM(PP.getSourceManager()), ConcatInfo(PP), OS(os), DisableLineMarkers(lineMarkers), DumpDefines(defines), DumpIncludeDirectives(DumpIncludeDirectives), - UseLineDirectives(UseLineDirectives) { + UseLineDirectives(UseLineDirectives), + NormalizeWhitespace(NormalizeWhitespace) { CurLine = 0; CurFilename += ""; EmittedTokensOnThisLine = false; @@ -110,8 +116,13 @@ FileType = SrcMgr::C_User; Initialized = false; IsFirstFileEntered = false; + + PrevTok.startToken(); + PrevPrevTok.startToken(); } + bool isNormalizingWhitespace() const { return NormalizeWhitespace; } + void setEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; } bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; } @@ -120,7 +131,12 @@ return EmittedDirectiveOnThisLine; } - bool startNewLineIfNeeded(bool ShouldUpdateCurrentLine = true); + /// Ensure that the output stream position is at the beginning of a new line + /// and inserts one if it does not.It is intended to ensure that directives + /// inserted by the directives not from the input source (such as #line) are + /// in the first column. To insert newlines the represent the input, use + /// MoveToLine(/*...*/, /*RequireStartOfLine=*/true). + void startNewLineIfNeeded(); void FileChanged(SourceLocation Loc, FileChangeReason Reason, SrcMgr::CharacteristicKind FileType, @@ -148,18 +164,45 @@ void PragmaAssumeNonNullBegin(SourceLocation Loc) override; void PragmaAssumeNonNullEnd(SourceLocation Loc) override; - bool HandleFirstTokOnLine(Token &Tok); + /// Insert whitespace before emitting the next token. + /// + /// @param Tok Next token to be emitted. + /// @param RequireSpace Ensure at least one whitespace is emitted. Useful + /// if non-tokens have been emitted to the stream. + /// @param RequireSameLine Never emit newlines. Useful when semantics depend + /// on being on the same line, such as directives. + void HandleWhitespaceBeforeTok(Token &Tok, bool RequireSpace, + bool RequireSameLine); /// Move to the line of the provided source location. This will - /// return true if the output stream required adjustment or if - /// the requested location is on the first line. - bool MoveToLine(SourceLocation Loc) { + /// return true if a newline was inserted or if + /// the requested location is the first token on the first line. + /// In these cases the next output will be the first column on the line and + /// make it possible to insert indention. The newline is was inserted + /// implicitly when at the beginning of the file. + /// + /// @param Tok Token where to move to. + /// @param RequiresStartOfLine Whether the next line depends on being in the + /// first column, such as a directive. + /// + /// @return Whether column adjustments are necessary. + bool MoveToLine(const Token &Tok, bool RequireStartOfLine) { + PresumedLoc PLoc = SM.getPresumedLoc(Tok.getLocation()); + if (PLoc.isInvalid()) + return false; + bool IsFirstInFile = Tok.isAtStartOfLine() && PLoc.getLine() == 1; + return MoveToLine(PLoc.getLine(), RequireStartOfLine) || IsFirstInFile; + } + + /// Move to the line of the provided source location. Returns true if a new + /// line was inserted. + bool MoveToLine(SourceLocation Loc, bool RequireStartOfLine) { PresumedLoc PLoc = SM.getPresumedLoc(Loc); if (PLoc.isInvalid()) return false; - return MoveToLine(PLoc.getLine()) || (PLoc.getLine() == 1); + return MoveToLine(PLoc.getLine(), RequireStartOfLine); } - bool MoveToLine(unsigned LineNo); + bool MoveToLine(unsigned LineNo, bool RequireStartOfLine); bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) { @@ -187,7 +230,7 @@ void PrintPPOutputPPCallbacks::WriteLineInfo(unsigned LineNo, const char *Extra, unsigned ExtraLen) { - startNewLineIfNeeded(/*ShouldUpdateCurrentLine=*/false); + startNewLineIfNeeded(); // Emit #line directives or GNU line markers depending on what mode we're in. if (UseLineDirectives) { @@ -214,43 +257,57 @@ /// object. We can do this by emitting some number of \n's, or be emitting a /// #line directive. This returns false if already at the specified line, true /// if some newlines were emitted. -bool PrintPPOutputPPCallbacks::MoveToLine(unsigned LineNo) { +bool PrintPPOutputPPCallbacks::MoveToLine(unsigned LineNo, + bool RequireStartOfLine) { + // If it is required to start a new line or finish the current, insert + // vertical whitespace now and take it into account when moving to the + // expected line. + bool StartedNewLine = false; + if ((RequireStartOfLine && EmittedTokensOnThisLine) || + EmittedDirectiveOnThisLine) { + OS << '\n'; + StartedNewLine = true; + CurLine += 1; + EmittedTokensOnThisLine = false; + EmittedDirectiveOnThisLine = false; + } + // If this line is "close enough" to the original line, just print newlines, // otherwise print a #line directive. - if (LineNo-CurLine <= 8) { - if (LineNo-CurLine == 1) - OS << '\n'; - else if (LineNo == CurLine) - return false; // Spelling line moved, but expansion line didn't. - else { - const char *NewLines = "\n\n\n\n\n\n\n\n"; - OS.write(NewLines, LineNo-CurLine); - } + if (CurLine == LineNo) { + // Nothing to do if we are already on the correct line. + } else if (!StartedNewLine && (!NormalizeWhitespace || !DisableLineMarkers) && + LineNo - CurLine == 1) { + // Printing a single line has priority over printing a #line directive, even + // when normalizing whitespace which otherwise would print #line directives + // for every single line. + OS << '\n'; + StartedNewLine = true; + } else if (!NormalizeWhitespace && LineNo - CurLine <= 8) { + const char *NewLines = "\n\n\n\n\n\n\n\n"; + OS.write(NewLines, LineNo - CurLine); + StartedNewLine = true; } else if (!DisableLineMarkers) { // Emit a #line or line marker. WriteLineInfo(LineNo, nullptr, 0); - } else { - // Okay, we're in -P mode, which turns off line markers. However, we still - // need to emit a newline between tokens on different lines. - startNewLineIfNeeded(/*ShouldUpdateCurrentLine=*/false); + StartedNewLine = true; + } + + if (StartedNewLine) { + EmittedTokensOnThisLine = false; + EmittedDirectiveOnThisLine = false; } CurLine = LineNo; - return true; + return StartedNewLine; } -bool -PrintPPOutputPPCallbacks::startNewLineIfNeeded(bool ShouldUpdateCurrentLine) { +void PrintPPOutputPPCallbacks::startNewLineIfNeeded() { if (EmittedTokensOnThisLine || EmittedDirectiveOnThisLine) { OS << '\n'; EmittedTokensOnThisLine = false; EmittedDirectiveOnThisLine = false; - if (ShouldUpdateCurrentLine) - ++CurLine; - return true; } - - return false; } /// FileChanged - Whenever the preprocessor enters or exits a #include file @@ -273,7 +330,7 @@ if (Reason == PPCallbacks::EnterFile) { SourceLocation IncludeLoc = UserLoc.getIncludeLoc(); if (IncludeLoc.isValid()) - MoveToLine(IncludeLoc); + MoveToLine(IncludeLoc, /*RequireStartOfLine=*/false); } else if (Reason == PPCallbacks::SystemHeaderPragma) { // GCC emits the # directive for this directive on the line AFTER the // directive and emits a bunch of spaces that aren't needed. This is because @@ -290,7 +347,8 @@ FileType = NewFileType; if (DisableLineMarkers) { - startNewLineIfNeeded(/*ShouldUpdateCurrentLine=*/false); + if (!NormalizeWhitespace) + startNewLineIfNeeded(); return; } @@ -336,15 +394,13 @@ // In -dI mode, dump #include directives prior to dumping their content or // interpretation. if (DumpIncludeDirectives) { - startNewLineIfNeeded(); - MoveToLine(HashLoc); + MoveToLine(HashLoc, /*RequireStartOfLine=*/true); const std::string TokenText = PP.getSpelling(IncludeTok); assert(!TokenText.empty()); OS << "#" << TokenText << " " << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"') << " /* clang -E -dI */"; setEmittedDirectiveOnThisLine(); - startNewLineIfNeeded(); } // When preprocessing, turn implicit imports into module import pragmas. @@ -353,17 +409,13 @@ case tok::pp_include: case tok::pp_import: case tok::pp_include_next: - startNewLineIfNeeded(); - MoveToLine(HashLoc); + MoveToLine(HashLoc, /*RequireStartOfLine=*/true); OS << "#pragma clang module import " << Imported->getFullModuleName(true) << " /* clang -E: implicit import for " << "#" << PP.getSpelling(IncludeTok) << " " << (IsAngled ? '<' : '"') << FileName << (IsAngled ? '>' : '"') << " */"; - // Since we want a newline after the pragma, but not a #, start a - // new line immediately. - EmittedTokensOnThisLine = true; - startNewLineIfNeeded(); + setEmittedDirectiveOnThisLine(); break; case tok::pp___include_macros: @@ -398,11 +450,11 @@ /// Ident - Handle #ident directives when read by the preprocessor. /// void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, StringRef S) { - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS.write("#ident ", strlen("#ident ")); OS.write(S.begin(), S.size()); - EmittedTokensOnThisLine = true; + setEmittedTokensOnThisLine(); } /// MacroDefined - This hook is called whenever a macro definition is seen. @@ -414,7 +466,7 @@ // Ignore __FILE__ etc. MI->isBuiltinMacro()) return; - MoveToLine(MI->getDefinitionLoc()); + MoveToLine(MI->getDefinitionLoc(), /*RequireStartOfLine=*/true); PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS); setEmittedDirectiveOnThisLine(); } @@ -425,7 +477,7 @@ // Only print out macro definitions in -dD mode. if (!DumpDefines) return; - MoveToLine(MacroNameTok.getLocation()); + MoveToLine(MacroNameTok.getLocation(), /*RequireStartOfLine=*/true); OS << "#undef " << MacroNameTok.getIdentifierInfo()->getName(); setEmittedDirectiveOnThisLine(); } @@ -446,8 +498,7 @@ StringRef Namespace, PragmaMessageKind Kind, StringRef Str) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma "; if (!Namespace.empty()) OS << Namespace << ' '; @@ -472,8 +523,7 @@ void PrintPPOutputPPCallbacks::PragmaDebug(SourceLocation Loc, StringRef DebugType) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma clang __debug "; OS << DebugType; @@ -483,16 +533,14 @@ void PrintPPOutputPPCallbacks:: PragmaDiagnosticPush(SourceLocation Loc, StringRef Namespace) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma " << Namespace << " diagnostic push"; setEmittedDirectiveOnThisLine(); } void PrintPPOutputPPCallbacks:: PragmaDiagnosticPop(SourceLocation Loc, StringRef Namespace) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma " << Namespace << " diagnostic pop"; setEmittedDirectiveOnThisLine(); } @@ -501,8 +549,7 @@ StringRef Namespace, diag::Severity Map, StringRef Str) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma " << Namespace << " diagnostic "; switch (Map) { case diag::Severity::Remark: @@ -528,8 +575,7 @@ void PrintPPOutputPPCallbacks::PragmaWarning(SourceLocation Loc, StringRef WarningSpec, ArrayRef Ids) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma warning(" << WarningSpec << ':'; for (ArrayRef::iterator I = Ids.begin(), E = Ids.end(); I != E; ++I) OS << ' ' << *I; @@ -539,8 +585,7 @@ void PrintPPOutputPPCallbacks::PragmaWarningPush(SourceLocation Loc, int Level) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma warning(push"; if (Level >= 0) OS << ", " << Level; @@ -549,16 +594,14 @@ } void PrintPPOutputPPCallbacks::PragmaWarningPop(SourceLocation Loc) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma warning(pop)"; setEmittedDirectiveOnThisLine(); } void PrintPPOutputPPCallbacks::PragmaExecCharsetPush(SourceLocation Loc, StringRef Str) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma character_execution_set(push"; if (!Str.empty()) OS << ", " << Str; @@ -567,64 +610,79 @@ } void PrintPPOutputPPCallbacks::PragmaExecCharsetPop(SourceLocation Loc) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma character_execution_set(pop)"; setEmittedDirectiveOnThisLine(); } void PrintPPOutputPPCallbacks:: PragmaAssumeNonNullBegin(SourceLocation Loc) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma clang assume_nonnull begin"; setEmittedDirectiveOnThisLine(); } void PrintPPOutputPPCallbacks:: PragmaAssumeNonNullEnd(SourceLocation Loc) { - startNewLineIfNeeded(); - MoveToLine(Loc); + MoveToLine(Loc, /*RequireStartOfLine=*/true); OS << "#pragma clang assume_nonnull end"; setEmittedDirectiveOnThisLine(); } -/// HandleFirstTokOnLine - When emitting a preprocessed file in -E mode, this -/// is called for the first token on each new line. If this really is the start -/// of a new logical line, handle it and return true, otherwise return false. -/// This may not be the start of a logical line because the "start of line" -/// marker is set for spelling lines, not expansion ones. -bool PrintPPOutputPPCallbacks::HandleFirstTokOnLine(Token &Tok) { - // Figure out what line we went to and insert the appropriate number of - // newline characters. - if (!MoveToLine(Tok.getLocation())) - return false; - - // Print out space characters so that the first token on a line is - // indented for easy reading. - unsigned ColNo = SM.getExpansionColumnNumber(Tok.getLocation()); - - // The first token on a line can have a column number of 1, yet still expect - // leading white space, if a macro expansion in column 1 starts with an empty - // macro argument, or an empty nested macro expansion. In this case, move the - // token to column 2. - if (ColNo == 1 && Tok.hasLeadingSpace()) - ColNo = 2; - - // This hack prevents stuff like: - // #define HASH # - // HASH define foo bar - // From having the # character end up at column 1, which makes it so it - // is not handled as a #define next time through the preprocessor if in - // -fpreprocessed mode. - if (ColNo <= 1 && Tok.is(tok::hash)) - OS << ' '; +void PrintPPOutputPPCallbacks::HandleWhitespaceBeforeTok(Token &Tok, + bool RequireSpace, + bool RequireSameLine) { + // These tokens are not expand to anything and don't need whitespace before + // them. + if (Tok.is(tok::eof) || + (Tok.isAnnotation() && !Tok.is(tok::annot_header_unit) && + !Tok.is(tok::annot_module_begin) && !Tok.is(tok::annot_module_end))) + return; - // Otherwise, indent the appropriate number of spaces. - for (; ColNo > 1; --ColNo) - OS << ' '; + if (!RequireSameLine && MoveToLine(Tok, /*RequireStartOfLine=*/false)) { + if (NormalizeWhitespace) { + // Avoid interpreting hash as a directive under -fpreprocessed. + if (Tok.is(tok::hash)) + OS << ' '; + } else { + // Print out space characters so that the first token on a line is + // indented for easy reading. + unsigned ColNo = SM.getExpansionColumnNumber(Tok.getLocation()); + + // The first token on a line can have a column number of 1, yet still + // expect leading white space, if a macro expansion in column 1 starts + // with an empty macro argument, or an empty nested macro expansion. In + // this case, move the token to column 2. + if (ColNo == 1 && Tok.hasLeadingSpace()) + ColNo = 2; + + // This hack prevents stuff like: + // #define HASH # + // HASH define foo bar + // From having the # character end up at column 1, which makes it so it + // is not handled as a #define next time through the preprocessor if in + // -fpreprocessed mode. + if (ColNo <= 1 && Tok.is(tok::hash)) + OS << ' '; + + // Otherwise, indent the appropriate number of spaces. + for (; ColNo > 1; --ColNo) + OS << ' '; + } + } else { + // Insert whitespace between the previous and next token if either + // - The input had whitespace between them and we are not in + // whitespace-normalization mode + // - The whitespace is necessary to keep the tokens apart and there is not + // already a newline between them + if (RequireSpace || (!NormalizeWhitespace && Tok.hasLeadingSpace()) || + ((EmittedTokensOnThisLine || EmittedTokensOnThisLine) && + AvoidConcat(PrevPrevTok, PrevTok, Tok))) + OS << ' '; + } - return true; + PrevPrevTok = PrevTok; + PrevTok = Tok; } void PrintPPOutputPPCallbacks::HandleNewlinesInToken(const char *TokStr, @@ -668,9 +726,9 @@ Token &PragmaTok) override { // Figure out what line we went to and insert the appropriate number of // newline characters. - Callbacks->startNewLineIfNeeded(); - Callbacks->MoveToLine(PragmaTok.getLocation()); + Callbacks->MoveToLine(PragmaTok.getLocation(), /*RequireStartOfLine=*/true); Callbacks->OS.write(Prefix, strlen(Prefix)); + Callbacks->setEmittedTokensOnThisLine(); if (ShouldExpandTokens) { // The first token does not have expanded macros. Expand them, if @@ -682,21 +740,16 @@ /*IsReinject=*/false); PP.Lex(PragmaTok); } - Token PrevToken; - Token PrevPrevToken; - PrevToken.startToken(); - PrevPrevToken.startToken(); // Read and print all of the pragma tokens. + bool IsFirst = true; while (PragmaTok.isNot(tok::eod)) { - if (PragmaTok.hasLeadingSpace() || - Callbacks->AvoidConcat(PrevPrevToken, PrevToken, PragmaTok)) - Callbacks->OS << ' '; + Callbacks->HandleWhitespaceBeforeTok(PragmaTok, /*RequireSpace=*/IsFirst, + /*RequireSameLine=*/true); + IsFirst = false; std::string TokSpell = PP.getSpelling(PragmaTok); Callbacks->OS.write(&TokSpell[0], TokSpell.size()); - - PrevPrevToken = PrevToken; - PrevToken = PragmaTok; + Callbacks->setEmittedTokensOnThisLine(); if (ShouldExpandTokens) PP.Lex(PragmaTok); @@ -716,33 +769,14 @@ !PP.getCommentRetentionState(); char Buffer[256]; - Token PrevPrevTok, PrevTok; - PrevPrevTok.startToken(); - PrevTok.startToken(); while (1) { - if (Callbacks->hasEmittedDirectiveOnThisLine()) { - Callbacks->startNewLineIfNeeded(); - Callbacks->MoveToLine(Tok.getLocation()); - } - - // If this token is at the start of a line, emit newlines if needed. - if (Tok.isAtStartOfLine() && Callbacks->HandleFirstTokOnLine(Tok)) { - // done. - } else if (Tok.hasLeadingSpace() || - // If we haven't emitted a token on this line yet, PrevTok isn't - // useful to look at and no concatenation could happen anyway. - (Callbacks->hasEmittedTokensOnThisLine() && - // Don't print "-" next to "-", it would form "--". - Callbacks->AvoidConcat(PrevPrevTok, PrevTok, Tok))) { - OS << ' '; - } + Callbacks->HandleWhitespaceBeforeTok(Tok, /*RequireSpace=*/false, + /*RequireSameLine=*/false); if (DropComments && Tok.is(tok::comment)) { // Skip comments. Normally the preprocessor does not generate // tok::comment nodes at all when not keeping comments, but under // -traditional-cpp the lexer keeps /all/ whitespace, including comments. - SourceLocation StartLoc = Tok.getLocation(); - Callbacks->MoveToLine(StartLoc.getLocWithOffset(Tok.getLength())); } else if (Tok.is(tok::eod)) { // Don't print end of directive tokens, since they are typically newlines // that mess up our line tracking. These come from unknown pre-processor @@ -798,6 +832,12 @@ // line number. if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown) Callbacks->HandleNewlinesInToken(TokPtr, Len); + if (Tok.is(tok::comment) && Len >= 2 && TokPtr[0] == '/' && + TokPtr[1] == '/') { + // It's a line comment; + // Ensure that we don't concatenate anything behind it. + Callbacks->setEmittedDirectiveOnThisLine(); + } } else { std::string S = PP.getSpelling(Tok); OS.write(S.data(), S.size()); @@ -806,13 +846,16 @@ // line number. if (Tok.getKind() == tok::comment || Tok.getKind() == tok::unknown) Callbacks->HandleNewlinesInToken(S.data(), S.size()); + if (Tok.is(tok::comment) && S.size() >= 2 && S[0] == '/' && S[1] == '/') { + // It's a line comment; + // Ensure that we don't concatenate anything behind it. + Callbacks->setEmittedDirectiveOnThisLine(); + } } Callbacks->setEmittedTokensOnThisLine(); if (Tok.is(tok::eof)) break; - PrevPrevTok = PrevTok; - PrevTok = Tok; PP.Lex(Tok); } } @@ -870,7 +913,8 @@ PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks( PP, *OS, !Opts.ShowLineMarkers, Opts.ShowMacros, - Opts.ShowIncludeDirectives, Opts.UseLineDirectives); + Opts.ShowIncludeDirectives, Opts.UseLineDirectives, + Opts.NormalizeWhitespace); // Expand macros in pragmas with -fms-extensions. The assumption is that // the majority of pragmas in such a file will be Microsoft pragmas. diff --git a/clang/test/CXX/lex/lex.pptoken/p3-2a.cpp b/clang/test/CXX/lex/lex.pptoken/p3-2a.cpp --- a/clang/test/CXX/lex/lex.pptoken/p3-2a.cpp +++ b/clang/test/CXX/lex/lex.pptoken/p3-2a.cpp @@ -57,14 +57,16 @@ // CHECK: import ; import HEADER; -// CHECK: import ; +// CHECK: import +// CHECK: ; import < foo bar >; // CHECK: import{{$}} -// CHECK: {{^}}; +// CHECK: {{^}} +// CHECK: ; import < foo diff --git a/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/i386-unknown-linux-gnu-ld b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/i386-unknown-linux-gnu-ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/i386-unknown-linux-gnu-ld @@ -0,0 +1 @@ +i386-unknown-linux-gnu-ld.gold \ No newline at end of file diff --git a/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/x86_64-unknown-linux-gnu-ld b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/x86_64-unknown-linux-gnu-ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/bin/x86_64-unknown-linux-gnu-ld @@ -0,0 +1 @@ +x86_64-unknown-linux-gnu-ld.gold \ No newline at end of file diff --git a/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/i386-unknown-linux-gnu/bin/ld b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/i386-unknown-linux-gnu/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/i386-unknown-linux-gnu/bin/ld @@ -0,0 +1 @@ +ld.gold \ No newline at end of file diff --git a/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/x86_64-unknown-linux-gnu/bin/ld b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/x86_64-unknown-linux-gnu/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/basic_cross_linux_tree/usr/x86_64-unknown-linux-gnu/bin/ld @@ -0,0 +1 @@ +ld.gold \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/as b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/as old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/as @@ -0,0 +1 @@ +i386-unknown-linux-gnu-as \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/ld b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/bin/ld @@ -0,0 +1 @@ +i386-unknown-linux-gnu-ld \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/as b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/as old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/as @@ -0,0 +1 @@ +../../bin/i386-unknown-linux-gnu-as \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/ld b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_32bit_linux_tree/usr/i386-unknown-linux/bin/ld @@ -0,0 +1 @@ +../../bin/i386-unknown-linux-gnu-ld \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/as b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/as old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/as @@ -0,0 +1 @@ +x86_64-unknown-linux-gnu-as \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/ld b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/bin/ld @@ -0,0 +1 @@ +x86_64-unknown-linux-gnu-ld \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/as b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/as old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/as @@ -0,0 +1 @@ +../../bin/x86_64-unknown-linux-gnu-as \ No newline at end of file diff --git a/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/ld b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/ld old mode 120000 new mode 100644 --- /dev/null +++ b/clang/test/Driver/Inputs/multilib_64bit_linux_tree/usr/x86_64-unknown-linux/bin/ld @@ -0,0 +1 @@ +../../bin/x86_64-unknown-linux-gnu-ld \ No newline at end of file diff --git a/clang/test/Preprocessor/c99-6_10_3_4_p5.c b/clang/test/Preprocessor/c99-6_10_3_4_p5.c --- a/clang/test/Preprocessor/c99-6_10_3_4_p5.c +++ b/clang/test/Preprocessor/c99-6_10_3_4_p5.c @@ -22,7 +22,8 @@ char c[2][6] = { str(hello), str() }; // CHECK: f(2 * (y+1)) + f(2 * (f(2 * (z[0])))) % f(2 * (0)) + t(1); -// CHECK: f(2 * (2 +(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1))^m(0,1); +// CHECK: f(2 * (2 +(3,4)-0,1)) | f(2 * (~ 5)) & f(2 * (0,1)) +// CHECK: ^m(0,1); // CHECK: int i[] = { 1, 23, 4, 5, }; // CHECK: char c[2][6] = { "hello", "" }; diff --git a/clang/test/Preprocessor/c99-6_10_3_4_p6.c b/clang/test/Preprocessor/c99-6_10_3_4_p6.c --- a/clang/test/Preprocessor/c99-6_10_3_4_p6.c +++ b/clang/test/Preprocessor/c99-6_10_3_4_p6.c @@ -20,7 +20,8 @@ // CHECK: printf("x" "1" "= %d, x" "2" "= s" x1, x2); -// CHECK: fputs("strncmp(\"abc\\0d\" \"abc\", '\\4') == 0" ": @\n", s); +// CHECK: fputs("strncmp(\"abc\\0d\" \"abc\", '\\4') == 0" +// CHECK: ": @\n", s); // CHECK: include "vers2.h" // CHECK: "hello"; // CHECK: "hello" ", world" diff --git a/clang/test/Preprocessor/comment_save.c b/clang/test/Preprocessor/comment_save.c --- a/clang/test/Preprocessor/comment_save.c +++ b/clang/test/Preprocessor/comment_save.c @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -E -C %s | FileCheck -strict-whitespace %s +// RUN: %clang_cc1 -E -C -fnormalize-whitespace %s | FileCheck -strict-whitespace %s // foo // CHECK: // foo diff --git a/clang/test/Preprocessor/first-line-indent.c b/clang/test/Preprocessor/first-line-indent.c --- a/clang/test/Preprocessor/first-line-indent.c +++ b/clang/test/Preprocessor/first-line-indent.c @@ -1,7 +1,14 @@ foo // RUN: %clang_cc1 -E %s | FileCheck -strict-whitespace %s +// RUN: %clang_cc1 -E -fnormalize-whitespace %s | FileCheck -strict-whitespace %s --check-prefix=NORMCOL +// RUN: %clang_cc1 -E -fnormalize-whitespace -P %s | FileCheck -strict-whitespace %s --check-prefix=NORMWS bar // CHECK: {{^ }}foo // CHECK: {{^ }}bar +// NORMCOL: {{^}}foo +// NORMCOL: {{^}}bar + +// NORMWS: {{^}}foo bar + diff --git a/clang/test/Preprocessor/hash_line.c b/clang/test/Preprocessor/hash_line.c --- a/clang/test/Preprocessor/hash_line.c +++ b/clang/test/Preprocessor/hash_line.c @@ -4,6 +4,10 @@ // CHECK-NEXT: {{^ #$}} // CHECK-NEXT: {{^2$}} // CHECK-NEXT: {{^ #$}} + +// RUN: %clang_cc1 -E -P -fnormalize-whitespace %s | FileCheck --strict-whitespace %s --check-prefix=NORMWS +// NORMWS: {{^}}1#2#{{$}} + #define EMPTY #define IDENTITY(X) X 1 diff --git a/clang/test/Preprocessor/line-directive-output-normcol.c b/clang/test/Preprocessor/line-directive-output-normcol.c new file mode 100644 --- /dev/null +++ b/clang/test/Preprocessor/line-directive-output-normcol.c @@ -0,0 +1,10 @@ +// RUN: %clang_cc1 -E -fnormalize-whitespace %s 2>&1 | FileCheck %s -strict-whitespace + +// CHECK: # 6 "{{.*}}line-directive-output-normcol.c" +// CHECK-NEXT: int x; +// CHECK-NEXT: int y; +int x; +int y; +// CHECK-NEXT: # 10 "{{.*}}line-directive-output-normcol.c" +// CHECK-NEXT: int z; +int z; diff --git a/clang/test/Preprocessor/line-directive-output.c b/clang/test/Preprocessor/line-directive-output.c --- a/clang/test/Preprocessor/line-directive-output.c +++ b/clang/test/Preprocessor/line-directive-output.c @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -E %s 2>&1 | FileCheck %s -strict-whitespace +// RUN: %clang_cc1 -E -fnormalize-whitespace %s 2>&1 | FileCheck %s -strict-whitespace // PR6101 int a; // CHECK: # 1 "{{.*}}line-directive-output.c" diff --git a/clang/test/Preprocessor/macro_space.c b/clang/test/Preprocessor/macro_space.c --- a/clang/test/Preprocessor/macro_space.c +++ b/clang/test/Preprocessor/macro_space.c @@ -1,4 +1,5 @@ // RUN: %clang_cc1 -E %s | FileCheck --strict-whitespace %s +// RUN: %clang_cc1 -E -P -fnormalize-whitespace %s | FileCheck --strict-whitespace %s --check-prefix=NORMWS #define FOO1() #define FOO2(x)x @@ -13,24 +14,32 @@ TEST(FOO1,) // CHECK: FOO1 <> < > <> <> < > <> < > < > +// NORMWS: FOO1<><><><><><><><> TEST(FOO2,) // CHECK: FOO2 <> < > <> <> < > <> < > < > +// NORMWS-SAME: FOO2<><><><><><><><> TEST(FOO3,) // CHECK: FOO3 <> < > <> <> < > <> < > < > +// NORMWS-SAME: FOO3<><><><><><><><> TEST(FOO4,) // CHECK: FOO4 < > < > < > < > < > < > < > < > +// NORMWS-SAME: FOO4<><><><><><><><> TEST(FOO5,) // CHECK: FOO5 < > < > < > < > < > < > < > < > +// NORMWS-SAME: FOO5<><><><><><><><> TEST(FOO6,) // CHECK: FOO6 <[]> < []> <[]> <[]> <[] > <[]> <[] > < []> +// NORMWS-SAME: FOO6<[]><[]><[]><[]><[]><[]><[]><[]> TEST(FOO7,) // CHECK: FOO7 <[ ]> < [ ]> <[ ]> <[ ]> <[ ] > <[ ]> <[ ] > < [ ]> +// NORMWS-SAME: FOO7<[]><[]><[]><[]><[]><[]><[]><[]> TEST(FOO8,) // CHECK: FOO8 <[ ]> < [ ]> <[ ]> <[ ]> <[ ] > <[ ]> <[ ] > < [ ]> +// NORMWS-SAME: FOO8<[]><[]><[]><[]><[]><[]><[]><[]> diff --git a/clang/test/Preprocessor/normalize-whitespace.c b/clang/test/Preprocessor/normalize-whitespace.c new file mode 100644 --- /dev/null +++ b/clang/test/Preprocessor/normalize-whitespace.c @@ -0,0 +1,54 @@ +// RUN: %clang_cc1 -E -fnormalize-whitespace %s 2>&1 | FileCheck %s --strict-whitespace --check-prefix=NORMCOL +// RUN: %clang_cc1 -E -C -fnormalize-whitespace %s 2>&1 | FileCheck %s --strict-whitespace --check-prefix=NORMCCOL +// RUN: %clang_cc1 -E -P -fnormalize-whitespace %s 2>&1 | FileCheck %s --strict-whitespace --check-prefix=NORMWS +// RUN: %clang_cc1 -E -C -P -fnormalize-whitespace %s 2>&1 | FileCheck %s --strict-whitespace --check-prefix=NORMCWS + +#define NOT_OMP omp something +#define HASH # + + int a; /* span-comment */ + int b ; // line-comment + _Pragma ( "omp barrier" ) x // more line-comments + #pragma omp nothing // another comment +HASH pragma NOT_OMP + int e; // again a line + int f ; + + +// NORMCOL: {{^}}# 9 "{{.*}}normalize-whitespace.c"{{$}} +// NORMCOL: {{^}}int a;{{$}} +// NORMCOL-NEXT: {{^}}int b;{{$}} +// NORMCOL-NEXT: {{^}}#pragma omp barrier{{$}} +// NORMCOL-NEXT: # 11 "{{.*}}normalize-whitespace.c" +// NORMCOL-NEXT: {{^}}x{{$}} +// NORMCOL-NEXT: {{^}}#pragma omp nothing{{$}} +// NORMCOL-NEXT: {{^ }}#pragma omp something{{$}} +// NORMCOL-NEXT: {{^}}int e;{{$}} +// NORMCOL-NEXT: {{^}}int f;{{$}} + +// FIXME: Comments after pragmas disappear, even without -fnormalize-whitespace +// NORMCCOL: {{^}}# 9 "{{.*}}normalize-whitespace.c"{{$}} +// NORMCCOL: {{^}}int a;/* span-comment */{{$}} +// NORMCCOL-NEXT: {{^}}int b;// line-comment{{$}} +// NORMCCOL-NEXT: {{^}}#pragma omp barrier{{$}} +// NORMCCOL-NEXT: # 11 "{{.*}}normalize-whitespace.c" +// NORMCCOL-NEXT: {{^}}x// more line-comments{{$}} +// NORMCCOL-NEXT: {{^}}#pragma omp nothing{{$}} +// NORMCCOL-NEXT: {{^ }}#pragma omp something{{$}} +// NORMCCOL-NEXT: {{^}}int e;// again a line{{$}} +// NORMCCOL-NEXT: {{^}}int f;{{$}} + +// NORMWS: {{^}}int a;int b;{{$}} +// NORMWS-NEXT: {{^}}#pragma omp barrier{{$}} +// NORMWS-NEXT: {{^}}x{{$}} +// NORMWS-NEXT: {{^}}#pragma omp nothing{{$}} +// NORMWS-NEXT: {{^ }}#pragma omp something int e;int f;{{$}} + +// FIXME: Comments after pragmas disappear, even without -fnormalize-whitespace +// NORMCWS: {{^}}int a;/* span-comment */int b;// line-comment{{$}} +// NORMCWS-NEXT: {{^}}#pragma omp barrier{{$}} +// NORMCWS-NEXT: {{^}}x// more line-comments{{$}} +// NORMCWS-NEXT: {{^}}#pragma omp nothing{{$}} +// NORMCWS-NEXT: {{^ }}#pragma omp something int e;// again a line{{$}} +// NORMCWS-NEXT: {{^}}int f; + diff --git a/clang/test/Preprocessor/print_line_include.c b/clang/test/Preprocessor/print_line_include.c --- a/clang/test/Preprocessor/print_line_include.c +++ b/clang/test/Preprocessor/print_line_include.c @@ -2,5 +2,8 @@ // CHECK: int x; // CHECK-NEXT: int x; +// RUN: %clang_cc1 -E -P -fnormalize-whitespace %s | FileCheck %s --check-prefix=NORMWS --strict-whitespace +// NORMWS: {{^}}int x;int x;{{$}} + #include "print_line_include.h" #include "print_line_include.h" diff --git a/clang/test/Preprocessor/stringize_space.c b/clang/test/Preprocessor/stringize_space.c --- a/clang/test/Preprocessor/stringize_space.c +++ b/clang/test/Preprocessor/stringize_space.c @@ -1,16 +1,18 @@ // RUN: %clang_cc1 -E %s | FileCheck --strict-whitespace %s +// RUN: %clang_cc1 -E -P -fnormalize-whitespace %s | FileCheck --strict-whitespace %s --check-prefix=NORMWS #define A(b) -#b , - #b , -# b , - # b A() // CHECK: {{^}}-"" , - "" , -"" , - ""{{$}} - +// NORMWS: {{^}}-"",-"",-"",-"" #define t(x) #x t(a c) // CHECK: {{^}}"a c"{{$}} +// NORMWS-SAME: "a c" #define str(x) #x #define f(x) str(-x) @@ -18,6 +20,7 @@ 1) // CHECK: {{^}}"-1" +// NORMWS-SAME: "-1" #define paste(a,b) str(a