Add -fuse-line-directive flag to control usage of #line with -E

[rnk]

Summary

Currently -fms-extensions controls this behavior, which doesn't make
much sense. It means we can't identify what is and isn't a system header
when compiling our own preprocessed output, because #line doesn't
represent this information.

If someone is feeding Clang's preprocessed output to another compiler,
they can use this flag.

Fixes PR20553.

[rnk]

Ancient, ancient ping

[rsmith]

You seem to be missing code in the driver to handle your two new -f options and to pass -fuse-line-directive to the frontend.

include/clang/Driver/Options.td

592–594

Maybe [...]directives?

[rnk]

• Pluralize the argument everywhere
• Add driver logic. I considered a test, but it's expensive.

[rnk]

Over-the-shoulder approved by Richard

include/clang/Driver/Options.td

 
 def ffor_scope : Flag<["-"], "ffor-scope">, Group<f_Group>;
 def fno_for_scope : Flag<["-"], "fno-for-scope">, Group<f_Group>;
 
 def frewrite_includes : Flag<["-"], "frewrite-includes">, Group<f_Group>,
   Flags<[CC1Option]>;
 def fno_rewrite_includes : Flag<["-"], "fno-rewrite-includes">, Group<f_Group>;
 
 def frewrite_map_file : Separate<["-"], "frewrite-map-file">,
                         Group<f_Group>,
                         Flags<[ DriverOption, CC1Option ]>;
 def frewrite_map_file_EQ : Joined<["-"], "frewrite-map-file=">,
                            Group<f_Group>,
                            Flags<[DriverOption]>;
 
+def fuse_line_directives : Flag<["-"], "fuse-line-directives">, Group<f_Group>,
+  Flags<[CC1Option]>;
+def fno_use_line_directives : Flag<["-"], "fno-use-line-directives">, Group<f_Group>;
+
 def ffreestanding : Flag<["-"], "ffreestanding">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Assert that the compilation takes place in a freestanding environment">;
 def fgnu_keywords : Flag<["-"], "fgnu-keywords">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Allow GNU-extension keywords regardless of language standard">;
 def fgnu89_inline : Flag<["-"], "fgnu89-inline">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Use the gnu89 inline semantics">;
 def fno_gnu89_inline : Flag<["-"], "fno-gnu89-inline">, Group<f_Group>;
 def fgnu_runtime : Flag<["-"], "fgnu-runtime">, Group<f_Group>,

include/clang/Frontend/PreprocessorOutputOptions.h

 
 /// PreprocessorOutputOptions - Options for controlling the C preprocessor
 /// output (e.g., -E).
 class PreprocessorOutputOptions {
 public:
   unsigned ShowCPP : 1;            ///< Print normal preprocessed output.
   unsigned ShowComments : 1;       ///< Show comments.
   unsigned ShowLineMarkers : 1;    ///< Show \#line markers.
+  unsigned UseLineDirectives : 1;   ///< Use \#line instead of GCC-style \# N.
   unsigned ShowMacroComments : 1;  ///< Show comments, even in macros.
   unsigned ShowMacros : 1;         ///< Print macro definitions.
   unsigned RewriteIncludes : 1;    ///< Preprocess include directives only.
 
 public:
   PreprocessorOutputOptions() {
     ShowCPP = 0;
     ShowComments = 0;
     ShowLineMarkers = 1;
+    UseLineDirectives = 0;
     ShowMacroComments = 0;
     ShowMacros = 0;
     RewriteIncludes = 0;
   }
 };
 
 }  // end namespace clang
 
 #endif

lib/Driver/Tools.cpp

if (!Args.hasFlag(options::OPT_fuse_cxa_atexit,
       KernelOrKext)
     CmdArgs.push_back("-fno-use-cxa-atexit");
 
   // -fms-extensions=0 is default.
   if (Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions,
                    IsWindowsMSVC))
     CmdArgs.push_back("-fms-extensions");
 
+  // -fno-use-line-directives is default.
+  if (Args.hasFlag(options::OPT_fuse_line_directives,
+                   options::OPT_fno_use_line_directives, false))
+    CmdArgs.push_back("-fuse-line-directives");
+
   // -fms-compatibility=0 is default.
   if (Args.hasFlag(options::OPT_fms_compatibility, 
                    options::OPT_fno_ms_compatibility,
                    (IsWindowsMSVC && Args.hasFlag(options::OPT_fms_extensions,
                                                   options::OPT_fno_ms_extensions,
                                                   true))))
     CmdArgs.push_back("-fms-compatibility");
 

lib/Frontend/CompilerInvocation.cpp

case frontend::RunPreprocessorOnly:
     break;
   }
 
   Opts.ShowComments = Args.hasArg(OPT_C);
   Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
   Opts.ShowMacroComments = Args.hasArg(OPT_CC);
   Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
   Opts.RewriteIncludes = Args.hasArg(OPT_frewrite_includes);
+  Opts.UseLineDirectives = Args.hasArg(OPT_fuse_line_directives);
 }
 
 static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
   using namespace options;
   Opts.ABI = Args.getLastArgValue(OPT_target_abi);
   Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
   Opts.FPMath = Args.getLastArgValue(OPT_mfpmath);
   Opts.FeaturesAsWritten = Args.getAllArgValues(OPT_target_feature);

lib/Frontend/PrintPreprocessedOutput.cpp

private:
 
   bool EmittedTokensOnThisLine;
   bool EmittedDirectiveOnThisLine;
   SrcMgr::CharacteristicKind FileType;
   SmallString<512> CurFilename;
   bool Initialized;
   bool DisableLineMarkers;
   bool DumpDefines;
-  bool UseLineDirective;
+  bool UseLineDirectives;
   bool IsFirstFileEntered;
 public:
-  PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream &os,
-                           bool lineMarkers, bool defines)
-     : PP(pp), SM(PP.getSourceManager()),
-       ConcatInfo(PP), OS(os), DisableLineMarkers(lineMarkers),
-       DumpDefines(defines) {
+  PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream &os, bool lineMarkers,
+                           bool defines, bool UseLineDirectives)
+      : PP(pp), SM(PP.getSourceManager()), ConcatInfo(PP), OS(os),
+        DisableLineMarkers(lineMarkers), DumpDefines(defines),
+        UseLineDirectives(UseLineDirectives) {
     CurLine = 0;
     CurFilename += "<uninit>";
     EmittedTokensOnThisLine = false;
     EmittedDirectiveOnThisLine = false;
     FileType = SrcMgr::C_User;
     Initialized = false;
     IsFirstFileEntered = false;
-
-    // If we're in microsoft mode, use normal #line instead of line markers.
-    UseLineDirective = PP.getLangOpts().MicrosoftExt;
   }
 
   void setEmittedTokensOnThisLine() { EmittedTokensOnThisLine = true; }
   bool hasEmittedTokensOnThisLine() const { return EmittedTokensOnThisLine; }
 
   void setEmittedDirectiveOnThisLine() { EmittedDirectiveOnThisLine = true; }
   bool hasEmittedDirectiveOnThisLine() const {
     return EmittedDirectiveOnThisLine;

 }  // end anonymous namespace
 
 void PrintPPOutputPPCallbacks::WriteLineInfo(unsigned LineNo,
                                              const char *Extra,
                                              unsigned ExtraLen) {
   startNewLineIfNeeded(/*ShouldUpdateCurrentLine=*/false);
 
   // Emit #line directives or GNU line markers depending on what mode we're in.
-  if (UseLineDirective) {
+  if (UseLineDirectives) {
     OS << "#line" << ' ' << LineNo << ' ' << '"';
     OS.write_escaped(CurFilename);
     OS << '"';
   } else {
     OS << '#' << ' ' << LineNo << ' ' << '"';
     OS.write_escaped(CurFilename);
     OS << '"';
 
if (!Opts.ShowCPP) {
     DoPrintMacros(PP, OS);
     return;
   }
 
   // Inform the preprocessor whether we want it to retain comments or not, due
   // to -C or -CC.
   PP.SetCommentRetentionState(Opts.ShowComments, Opts.ShowMacroComments);
 
-  PrintPPOutputPPCallbacks *Callbacks =
-      new PrintPPOutputPPCallbacks(PP, *OS, !Opts.ShowLineMarkers,
-                                   Opts.ShowMacros);
+  PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(
+      PP, *OS, !Opts.ShowLineMarkers, Opts.ShowMacros, Opts.UseLineDirectives);
   PP.AddPragmaHandler(new UnknownPragmaHandler("#pragma", Callbacks));
   PP.AddPragmaHandler("GCC", new UnknownPragmaHandler("#pragma GCC",Callbacks));
   PP.AddPragmaHandler("clang",
                       new UnknownPragmaHandler("#pragma clang", Callbacks));
 
   PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
 
   // After we have configured the preprocessor, enter the main file.

lib/Frontend/Rewrite/InclusionRewriter.cpp

struct FileChange {
     }
   };
   Preprocessor &PP; ///< Used to find inclusion directives.
   SourceManager &SM; ///< Used to read and manage source files.
   raw_ostream &OS; ///< The destination stream for rewritten contents.
   StringRef MainEOL; ///< The line ending marker to use.
   const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
   bool ShowLineMarkers; ///< Show #line markers.
-  bool UseLineDirective; ///< Use of line directives or line markers.
+  bool UseLineDirectives; ///< Use of line directives or line markers.
   typedef std::map<unsigned, FileChange> FileChangeMap;
   FileChangeMap FileChanges; ///< Tracks which files were included where.
   /// Used transitively for building up the FileChanges mapping over the
   /// various \c PPCallbacks callbacks.
   FileChangeMap::iterator LastInsertedFileChange;
 public:
-  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers);
+  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers,
+                    bool UseLineDirectives);
   bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
   void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
     PredefinesBuffer = Buf;
   }
   void detectMainFileEOL();
 private:
   void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                    SrcMgr::CharacteristicKind FileType,
private:
   const FileChange *FindFileChangeLocation(SourceLocation Loc) const;
   StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
 };
 
 }  // end anonymous namespace
 
 /// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
 InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
-                                     bool ShowLineMarkers)
+                                     bool ShowLineMarkers,
+                                     bool UseLineDirectives)
     : PP(PP), SM(PP.getSourceManager()), OS(OS), MainEOL("\n"),
       PredefinesBuffer(nullptr), ShowLineMarkers(ShowLineMarkers),
-      LastInsertedFileChange(FileChanges.end()) {
-  // If we're in microsoft mode, use normal #line instead of line markers.
-  UseLineDirective = PP.getLangOpts().MicrosoftExt;
-}
+      LastInsertedFileChange(FileChanges.end()),
+      UseLineDirectives(UseLineDirectives) {}
 
 /// Write appropriate line information as either #line directives or GNU line
 /// markers depending on what mode we're in, including the \p Filename and
 /// \p Line we are located at, using the specified \p EOL line separator, and
 /// any \p Extra context specifiers in GNU line directives.
 void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
                                       SrcMgr::CharacteristicKind FileType,
                                       StringRef Extra) {
   if (!ShowLineMarkers)
     return;
-  if (UseLineDirective) {
+  if (UseLineDirectives) {
     OS << "#line" << ' ' << Line << ' ' << '"';
     OS.write_escaped(Filename);
     OS << '"';
   } else {
     // Use GNU linemarkers as described here:
     // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
     OS << '#' << ' ' << Line << ' ' << '"';
     OS.write_escaped(Filename);
OutputContentUpTo(FromFile, NextToWrite,
                     Line, /*EnsureNewline=*/true);
   return true;
 }
 
 /// InclusionRewriterInInput - Implement -frewrite-includes mode.
 void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
                                    const PreprocessorOutputOptions &Opts) {
   SourceManager &SM = PP.getSourceManager();
-  InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS,
-                                                     Opts.ShowLineMarkers);
+  InclusionRewriter *Rewrite = new InclusionRewriter(
+      PP, *OS, Opts.ShowLineMarkers, Opts.UseLineDirectives);
   Rewrite->detectMainFileEOL();
 
   PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
   PP.IgnorePragmas();
 
   // First let the preprocessor process the entire file and call callbacks.
   // Callbacks will record which #include's were actually performed.
   PP.EnterMainSourceFile();

test/Frontend/rewrite-includes-line-markers.c

+// RUN: %clang_cc1 -E -frewrite-includes -I %S/Inputs %s | FileCheck %s --check-prefix=GNU
+// RUN: %clang_cc1 -E -frewrite-includes -fuse-line-directives -I %S/Inputs %s | FileCheck %s --check-prefix=LINE
+#include "test.h"
+int f() { return x; }
+
+// GNU: {{^}}# 1 "{{.*}}rewrite-includes-line-markers.c"
+// GNU: {{^}}#include "test.h"
+// GNU: {{^}}# 1 "{{.*}}test.h"
+// GNU: {{^}}#include "test2.h"
+// GNU: {{^}}# 1 "{{.*}}test2.h"
+// GNU: {{^}}int x;
+// GNU: {{^}}# 4 "{{.*}}rewrite-includes-line-markers.c" 2
+// GNU: {{^}}int f() { return x; }
+
+// LINE: {{^}}#line 1 "{{.*}}rewrite-includes-line-markers.c"
+// LINE: {{^}}#include "test.h"
+// LINE: {{^}}#line 1 "{{.*}}test.h"
+// LINE: {{^}}#include "test2.h"
+// LINE: {{^}}#line 1 "{{.*}}test2.h"
+// LINE: {{^}}int x;
+// LINE: {{^}}#line 4 "{{.*}}rewrite-includes-line-markers.c"
+// LINE: {{^}}int f() { return x; }

test/Lexer/pragma-operators.cpp

-// RUN: %clang_cc1 -fms-extensions -std=c++11 -E %s | FileCheck %s
+// RUN: %clang_cc1 -fms-extensions -std=c++11 -E %s -fuse-line-directives | FileCheck %s
 
 // Test that we properly expand the C99 _Pragma and Microsoft __pragma
 // into #pragma directives, with newlines where needed. <rdar://problem/8412013>
 
 // CHECK: #line
 // CHECK: #pragma warning(push)
 // CHECK: extern "C" {
 // CHECK: #line

test/Preprocessor/_Pragma-location.c

 // RUN: %clang_cc1 %s -fms-extensions -E | FileCheck %s
 // We use -fms-extensions to test both _Pragma and __pragma.
 
 // A long time ago the pragma lexer's buffer showed through in -E output.
 // CHECK-NOT: scratch space
 
 #define push_p _Pragma ("pack(push)")
 push_p
 // CHECK: #pragma pack(push)
 
 push_p _Pragma("pack(push)") __pragma(pack(push))
 // CHECK: #pragma pack(push)
-// CHECK-NEXT: #line 11 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 11 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma pack(push)
-// CHECK-NEXT: #line 11 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 11 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma pack(push)
 
 
 #define __PRAGMA_PUSH_NO_EXTRA_ARG_WARNINGS _Pragma("clang diagnostic push") \
 _Pragma("clang diagnostic ignored \"-Wformat-extra-args\"")
 #define __PRAGMA_POP_NO_EXTRA_ARG_WARNINGS _Pragma("clang diagnostic pop")
 
 void test () {
   1;_Pragma("clang diagnostic push") \
   _Pragma("clang diagnostic ignored \"-Wformat-extra-args\"")
   _Pragma("clang diagnostic pop")
 
   2;__PRAGMA_PUSH_NO_EXTRA_ARG_WARNINGS
   3;__PRAGMA_POP_NO_EXTRA_ARG_WARNINGS
 }
 
 // CHECK: void test () {
 // CHECK-NEXT:   1;
-// CHECK-NEXT: #line 24 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 24 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma clang diagnostic push
 // CHECK-NEXT: #pragma clang diagnostic ignored "-Wformat-extra-args"
 // CHECK-NEXT: #pragma clang diagnostic pop
 
 // CHECK:   2;
-// CHECK-NEXT: #line 28 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 28 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma clang diagnostic push
-// CHECK-NEXT: #line 28 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 28 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma clang diagnostic ignored "-Wformat-extra-args"
 // CHECK-NEXT:   3;
-// CHECK-NEXT: #line 29 "{{.*}}_Pragma-location.c"
+// CHECK-NEXT: # 29 "{{.*}}_Pragma-location.c"
 // CHECK-NEXT: #pragma clang diagnostic pop
 // CHECK-NEXT: }