[C++20][Modules][HU 5/5] Add fdirectives-only mode for preprocessing output.

[iains]

Summary

When the -fdirectives-only option is used together with -E, the preprocessor
output reflects evaluation of if/then/else directives.

As such, it preserves defines and undefs of macros that are still live after
such processing. The intent is that this output could be consumed as input
to generate considered a C++20 header unit.

We strip out any (unused) defines that come from built-in, built-in-file or
command line; these are re-added when the preprocessed source is consumed.

[iains]

rebased.

[iains]

rebased.

[urnathan]

Kind of surprised this wasn't already a thing :)

[iains]

rebased

clang/include/clang/Driver/Options.td

 
 defm rewrite_imports : BoolFOption<"rewrite-imports",
   PreprocessorOutputOpts<"RewriteImports">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option]>, NegFlag<SetFalse>>;
 defm rewrite_includes : BoolFOption<"rewrite-includes",
   PreprocessorOutputOpts<"RewriteIncludes">, DefaultFalse,
   PosFlag<SetTrue, [CC1Option]>, NegFlag<SetFalse>>;
 
+defm directives_only : OptInCC1FFlag<"directives-only", "">;
+
 defm delete_null_pointer_checks : BoolFOption<"delete-null-pointer-checks",
   CodeGenOpts<"NullPointerIsValid">, DefaultFalse,
   NegFlag<SetTrue, [CC1Option], "Do not treat usage of null pointers as undefined behavior">,
   PosFlag<SetFalse, [], "Treat usage of null pointers as undefined behavior (default)">,
   BothFlags<[CoreOption]>>;
 
 def frewrite_map_file_EQ : Joined<["-"], "frewrite-map-file=">,
                            Group<f_Group>,

clang/include/clang/Frontend/PreprocessorOutputOptions.h

public:
   unsigned ShowLineMarkers : 1;    ///< Show \#line markers.
   unsigned UseLineDirectives : 1;   ///< Use \#line instead of GCC-style \# N.
   unsigned ShowMacroComments : 1;  ///< Show comments, even in macros.
   unsigned ShowMacros : 1;         ///< Print macro definitions.
   unsigned ShowIncludeDirectives : 1;  ///< Print includes, imports etc. within preprocessed output.
   unsigned RewriteIncludes : 1;    ///< Preprocess include directives only.
   unsigned RewriteImports  : 1;    ///< Include contents of transitively-imported modules.
   unsigned MinimizeWhitespace : 1; ///< Ignore whitespace from input.
+  unsigned DirectivesOnly : 1; ///< Process directives but do not expand macros.
 
 public:
   PreprocessorOutputOptions() {
     ShowCPP = 0;
     ShowComments = 0;
     ShowLineMarkers = 1;
     UseLineDirectives = 0;
     ShowMacroComments = 0;
     ShowMacros = 0;
     ShowIncludeDirectives = 0;
     RewriteIncludes = 0;
     RewriteImports = 0;
     MinimizeWhitespace = 0;
+    DirectivesOnly = 0;
   }
 };
 
 }  // end namespace clang
 
 #endif

clang/lib/Frontend/CompilerInvocation.cpp

 #include "clang/Driver/Options.inc"
 #undef PREPROCESSOR_OUTPUT_OPTION_WITH_MARSHALLING
 
   bool Generate_dM = isStrictlyPreprocessorAction(Action) && !Opts.ShowCPP;
   if (Generate_dM)
     GenerateArg(Args, OPT_dM, SA);
   if (!Generate_dM && Opts.ShowMacros)
     GenerateArg(Args, OPT_dD, SA);
+  if (Opts.DirectivesOnly)
+    GenerateArg(Args, OPT_fdirectives_only, SA);
 }
 
 static bool ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
                                         ArgList &Args, DiagnosticsEngine &Diags,
                                         frontend::ActionKind Action) {
   unsigned NumErrorsBefore = Diags.getNumErrors();
 
   PreprocessorOutputOptions &PreprocessorOutputOpts = Opts;
 
 #define PREPROCESSOR_OUTPUT_OPTION_WITH_MARSHALLING(                           \
     PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
     HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
     DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
     MERGER, EXTRACTOR, TABLE_INDEX)                                            \
   PARSE_OPTION_WITH_MARSHALLING(                                               \
       Args, Diags, ID, FLAGS, PARAM, SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,     \
       IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, MERGER, TABLE_INDEX)
 #include "clang/Driver/Options.inc"
 #undef PREPROCESSOR_OUTPUT_OPTION_WITH_MARSHALLING
 
   Opts.ShowCPP = isStrictlyPreprocessorAction(Action) && !Args.hasArg(OPT_dM);
   Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
+  Opts.DirectivesOnly = Args.hasArg(OPT_fdirectives_only);
 
   return Diags.getNumErrors() == NumErrorsBefore;
 }
 
 static void GenerateTargetArgs(const TargetOptions &Opts,
                                SmallVectorImpl<const char *> &Args,
                                CompilerInvocation::StringAllocator SA) {
   const TargetOptions *TargetOpts = &Opts;

clang/lib/Frontend/PrintPreprocessedOutput.cpp

private:
   SmallString<512> CurFilename;
   bool Initialized;
   bool DisableLineMarkers;
   bool DumpDefines;
   bool DumpIncludeDirectives;
   bool UseLineDirectives;
   bool IsFirstFileEntered;
   bool MinimizeWhitespace;
+  bool DirectivesOnly;
 
   Token PrevTok;
   Token PrevPrevTok;
 
 public:
   PrintPPOutputPPCallbacks(Preprocessor &pp, raw_ostream &os, bool lineMarkers,
                            bool defines, bool DumpIncludeDirectives,
-                           bool UseLineDirectives, bool MinimizeWhitespace)
+                           bool UseLineDirectives, bool MinimizeWhitespace,
+                           bool DirectivesOnly)
       : PP(pp), SM(PP.getSourceManager()), ConcatInfo(PP), OS(os),
         DisableLineMarkers(lineMarkers), DumpDefines(defines),
         DumpIncludeDirectives(DumpIncludeDirectives),
         UseLineDirectives(UseLineDirectives),
-        MinimizeWhitespace(MinimizeWhitespace) {
+        MinimizeWhitespace(MinimizeWhitespace), DirectivesOnly(DirectivesOnly) {
     CurLine = 0;
     CurFilename += "<uninit>";
     EmittedTokensOnThisLine = false;
     EmittedDirectiveOnThisLine = false;
     FileType = SrcMgr::C_User;
     Initialized = false;
     IsFirstFileEntered = false;
 
void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, StringRef S) {
   OS.write(S.begin(), S.size());
   setEmittedTokensOnThisLine();
 }
 
 /// MacroDefined - This hook is called whenever a macro definition is seen.
 void PrintPPOutputPPCallbacks::MacroDefined(const Token &MacroNameTok,
                                             const MacroDirective *MD) {
   const MacroInfo *MI = MD->getMacroInfo();
-  // Only print out macro definitions in -dD mode.
-  if (!DumpDefines ||
+  // Print out macro definitions in -dD mode and when we have -fdirectives-only
+  // for C++20 header units.
+  if ((!DumpDefines && !DirectivesOnly) ||
       // Ignore __FILE__ etc.
-      MI->isBuiltinMacro()) return;
+      MI->isBuiltinMacro())
+    return;
 
-  MoveToLine(MI->getDefinitionLoc(), /*RequireStartOfLine=*/true);
+  SourceLocation DefLoc = MI->getDefinitionLoc();
+  if (DirectivesOnly && !MI->isUsed()) {
+    SourceManager &SM = PP.getSourceManager();
+    if (SM.isWrittenInBuiltinFile(DefLoc) ||
+        SM.isWrittenInCommandLineFile(DefLoc))
+      return;
+  }
+  MoveToLine(DefLoc, /*RequireStartOfLine=*/true);
   PrintMacroDefinition(*MacroNameTok.getIdentifierInfo(), *MI, PP, OS);
   setEmittedDirectiveOnThisLine();
 }
 
 void PrintPPOutputPPCallbacks::MacroUndefined(const Token &MacroNameTok,
                                               const MacroDefinition &MD,
                                               const MacroDirective *Undef) {
-  // Only print out macro definitions in -dD mode.
-  if (!DumpDefines) return;
+  // Print out macro definitions in -dD mode and when we have -fdirectives-only
+  // for C++20 header units.
+  if (!DumpDefines && !DirectivesOnly)
+    return;
 
   MoveToLine(MacroNameTok.getLocation(), /*RequireStartOfLine=*/true);
   OS << "#undef " << MacroNameTok.getIdentifierInfo()->getName();
   setEmittedDirectiveOnThisLine();
 }
 
 static void outputPrintable(raw_ostream &OS, StringRef Str) {
   for (unsigned char Char : Str) {
void clang::DoPrintPreprocessedInput(Preprocessor &PP, raw_ostream *OS,
 
   // Inform the preprocessor whether we want it to retain comments or not, due
   // to -C or -CC.
   PP.SetCommentRetentionState(Opts.ShowComments, Opts.ShowMacroComments);
 
   PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(
       PP, *OS, !Opts.ShowLineMarkers, Opts.ShowMacros,
       Opts.ShowIncludeDirectives, Opts.UseLineDirectives,
-      Opts.MinimizeWhitespace);
+      Opts.MinimizeWhitespace, Opts.DirectivesOnly);
 
   // Expand macros in pragmas with -fms-extensions.  The assumption is that
   // the majority of pragmas in such a file will be Microsoft pragmas.
   // Remember the handlers we will add so that we can remove them later.
   std::unique_ptr<UnknownPragmaHandler> MicrosoftExtHandler(
       new UnknownPragmaHandler(
           "#pragma", Callbacks,
           /*RequireTokenExpansion=*/PP.getLangOpts().MicrosoftExt));
std::unique_ptr<UnknownPragmaHandler> OpenMPHandler(
       new UnknownPragmaHandler("#pragma omp", Callbacks,
                                /*RequireTokenExpansion=*/true));
   PP.AddPragmaHandler("omp", OpenMPHandler.get());
 
   PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Callbacks));
 
   // After we have configured the preprocessor, enter the main file.
   PP.EnterMainSourceFile();
+  if (Opts.DirectivesOnly)
+    PP.SetMacroExpansionOnlyInDirectives();
 
   // Consume all of the tokens that come from the predefines buffer.  Those
   // should not be emitted into the output and are guaranteed to be at the
   // start.
   const SourceManager &SourceMgr = PP.getSourceManager();
   Token Tok;
   do {
     PP.Lex(Tok);

clang/test/Modules/cxx20-hu-05.cpp

+// Test check that consuming -E -fdirectives-only output produces the expected
+// header unit.
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+// RUN: cd %t
+
+// RUN: %clang_cc1 -std=c++20 -E -fdirectives-only -xc++-user-header hu-01.h \
+// RUN: -o hu-01.iih
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit \
+// RUN: -xc++-user-header-cpp-output hu-01.iih -o hu-01.pcm
+
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header hu-02.h \
+// RUN: -DFOO -fmodule-file=hu-01.pcm -o hu-02.pcm -Rmodule-import 2>&1 | \
+// RUN: FileCheck --check-prefix=CHECK-IMP %s -DTDIR=%t
+
+//--- hu-01.h
+#ifndef __GUARD
+#define __GUARD
+
+int baz(int);
+#define FORTYTWO 42
+
+#define SHOULD_NOT_BE_DEFINED -1
+#undef SHOULD_NOT_BE_DEFINED
+
+#endif // __GUARD
+// expected-no-diagnostics
+
+//--- hu-02.h
+export import "hu-01.h";
+#if !defined(FORTYTWO) || FORTYTWO != 42
+#error FORTYTWO missing in hu-02
+#endif
+
+#ifndef __GUARD
+#error __GUARD missing in hu-02
+#endif
+
+#ifdef SHOULD_NOT_BE_DEFINED
+#error SHOULD_NOT_BE_DEFINED is visible
+#endif
+
+// Make sure that we have not discarded macros from the builtin file.
+#ifndef __cplusplus
+#error we dropped a defined macro
+#endif
+
+#define KAP 6174
+
+#ifdef FOO
+#define FOO_BRANCH(X) (X) + 1
+inline int foo(int x) {
+  if (x == FORTYTWO)
+    return FOO_BRANCH(x);
+  return FORTYTWO;
+}
+#else
+#define BAR_BRANCH(X) (X) + 2
+inline int bar(int x) {
+  if (x == FORTYTWO)
+    return BAR_BRANCH(x);
+  return FORTYTWO;
+}
+#endif
+// CHECK-IMP: remark: importing module './hu-01.h' from 'hu-01.pcm'