[clang][cli] Generate and round-trip DependencyOutput options

[jansvoboda11]

Summary

This patch implements generation of remaining dependency output options and tests it by performing parse-generate-parse round trip.

[dexonsmith]

LGTM.

clang/include/clang/Driver/Options.td

class LangOpts<string base>
   : KeyPathAndMacro<"LangOpts->", base, "LANG_"> {}
 class TargetOpts<string base>
   : KeyPathAndMacro<"TargetOpts->", base, "TARGET_"> {}
 class FrontendOpts<string base>
   : KeyPathAndMacro<"FrontendOpts.", base, "FRONTEND_"> {}
 class PreprocessorOutputOpts<string base>
   : KeyPathAndMacro<"PreprocessorOutputOpts.", base, "PREPROCESSOR_OUTPUT_"> {}
 class DependencyOutputOpts<string base>
-  : KeyPathAndMacro<"DependencyOutputOpts.", base> {}
+  : KeyPathAndMacro<"DependencyOutputOpts.", base, "DEPENDENCY_OUTPUT_"> {}
 class CodeGenOpts<string base>
   : KeyPathAndMacro<"CodeGenOpts.", base, "CODEGEN_"> {}
 class HeaderSearchOpts<string base>
   : KeyPathAndMacro<"HeaderSearchOpts->", base, "HEADER_SEARCH_"> {}
 class PreprocessorOpts<string base>
   : KeyPathAndMacro<"PreprocessorOpts->", base, "PREPROCESSOR_"> {}
 class FileSystemOpts<string base>
   : KeyPathAndMacro<"FileSystemOpts.", base> {}

clang/include/clang/Frontend/DependencyOutputOptions.h

 namespace clang {
 
 /// ShowIncludesDestination - Destination for /showIncludes output.
 enum class ShowIncludesDestination { None, Stdout, Stderr };
 
 /// DependencyOutputFormat - Format for the compiler dependency file.
 enum class DependencyOutputFormat { Make, NMake };
 
+/// ExtraDepKind - The kind of extra dependency file.
+enum ExtraDepKind {
+  EDK_SanitizeBlacklist,
+  EDK_ProfileList,
+  EDK_ModuleFile,
+  EDK_DepFileEntry,
+};
+
 /// DependencyOutputOptions - Options for controlling the compiler dependency
 /// file generation.
 class DependencyOutputOptions {
 public:
   unsigned IncludeSystemHeaders : 1; ///< Include system header dependencies.
   unsigned ShowHeaderIncludes : 1;   ///< Show header inclusions (-H).
   unsigned UsePhonyTargets : 1;      ///< Include phony targets for each
                                      /// dependency, which can avoid some 'make'
public:
   /// predefines buffer. If the output file is "-", output will be sent to
   /// stderr.
   std::string HeaderIncludeOutputFile;
 
   /// A list of names to use as the targets in the dependency file; this list
   /// must contain at least one entry.
   std::vector<std::string> Targets;
 
-  /// A list of filenames to be used as extra dependencies for every target.
-  std::vector<std::string> ExtraDeps;
+  /// A list of extra dependencies (filename and kind) to be used for every
+  /// target.
+  std::vector<std::pair<std::string, ExtraDepKind>> ExtraDeps;
 
   /// In /showIncludes mode, pretend the main TU is a header with this name.
   std::string ShowIncludesPretendHeader;
 
   /// The file to write GraphViz-formatted header dependencies to.
   std::string DOTOutputFile;
 
   /// The directory to copy module dependencies to when collecting them.

clang/lib/Frontend/CompilerInvocation.cpp

return RoundTrip(
                             OutputFile, &LangOptsRef);
       },
       [&DummyOpts](CompilerInvocation &Res) {
         std::swap(Res.CodeGenOpts, DummyOpts);
       },
       Res, Args, Diags, "CodeGenOptions");
 }
 
-static void ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
-                                      ArgList &Args) {
+static void
+GenerateDependencyOutputArgs(const DependencyOutputOptions &Opts,
+                             SmallVectorImpl<const char *> &Args,
+                             CompilerInvocation::StringAllocator SA) {
+  const DependencyOutputOptions &DependencyOutputOpts = Opts;
+#define DEPENDENCY_OUTPUT_OPTION_WITH_MARSHALLING(                             \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  GENERATE_OPTION_WITH_MARSHALLING(                                            \
+      Args, SA, KIND, FLAGS, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,    \
+      IMPLIED_CHECK, IMPLIED_VALUE, DENORMALIZER, EXTRACTOR, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef DEPENDENCY_OUTPUT_OPTION_WITH_MARSHALLING
+
+  if (Opts.ShowIncludesDest != ShowIncludesDestination::None)
+    GenerateArg(Args, OPT_show_includes, SA);
+
+  for (const auto &Dep : Opts.ExtraDeps) {
+    switch (Dep.second) {
+    case EDK_SanitizeBlacklist:
+      // Sanitizer blacklist arguments are generated from LanguageOptions.
+      continue;
+    case EDK_ModuleFile:
+      // Module file arguments are generated from FrontendOptions and
+      // HeaderSearchOptions.
+      continue;
+    case EDK_ProfileList:
+      GenerateArg(Args, OPT_fprofile_list_EQ, Dep.first, SA);
+      break;
+    case EDK_DepFileEntry:
+      GenerateArg(Args, OPT_fdepfile_entry, Dep.first, SA);
+      break;
+    }
+  }
+}
+
+static bool ParseDependencyOutputArgsImpl(
+    DependencyOutputOptions &Opts, ArgList &Args,
+    DiagnosticsEngine &Diags,
+    frontend::ActionKind Action, bool ShowLineMarkers) {
+  unsigned NumErrorsBefore = Diags.getNumErrors();
+  bool Success = true;
+
+  DependencyOutputOptions &DependencyOutputOpts = Opts;
+#define DEPENDENCY_OUTPUT_OPTION_WITH_MARSHALLING(                             \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  PARSE_OPTION_WITH_MARSHALLING(Args, Diags, Success, ID, FLAGS, PARAM,        \
+                                SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,          \
+                                IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER,      \
+                                MERGER, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef DEPENDENCY_OUTPUT_OPTION_WITH_MARSHALLING
+
   if (Args.hasArg(OPT_show_includes)) {
     // Writing both /showIncludes and preprocessor output to stdout
     // would produce interleaved output, so use stderr for /showIncludes.
     // This behaves the same as cl.exe, when /E, /EP or /P are passed.
-    if (Args.hasArg(options::OPT_E) || Args.hasArg(options::OPT_P))
+    if (Action == frontend::PrintPreprocessedInput || !ShowLineMarkers)
       Opts.ShowIncludesDest = ShowIncludesDestination::Stderr;
     else
       Opts.ShowIncludesDest = ShowIncludesDestination::Stdout;
   } else {
     Opts.ShowIncludesDest = ShowIncludesDestination::None;
   }
+
   // Add sanitizer blacklists as extra dependencies.
   // They won't be discovered by the regular preprocessor, so
   // we let make / ninja to know about this implicit dependency.
   if (!Args.hasArg(OPT_fno_sanitize_blacklist)) {
     for (const auto *A : Args.filtered(OPT_fsanitize_blacklist)) {
       StringRef Val = A->getValue();
       if (Val.find('=') == StringRef::npos)
-        Opts.ExtraDeps.push_back(std::string(Val));
+        Opts.ExtraDeps.emplace_back(std::string(Val), EDK_SanitizeBlacklist);
     }
     if (Opts.IncludeSystemHeaders) {
       for (const auto *A : Args.filtered(OPT_fsanitize_system_blacklist)) {
         StringRef Val = A->getValue();
         if (Val.find('=') == StringRef::npos)
-          Opts.ExtraDeps.push_back(std::string(Val));
+          Opts.ExtraDeps.emplace_back(std::string(Val), EDK_SanitizeBlacklist);
       }
     }
   }
 
   // -fprofile-list= dependencies.
   for (const auto &Filename : Args.getAllArgValues(OPT_fprofile_list_EQ))
-    Opts.ExtraDeps.push_back(Filename);
+    Opts.ExtraDeps.emplace_back(Filename, EDK_ProfileList);
 
   // Propagate the extra dependencies.
-  for (const auto *A : Args.filtered(OPT_fdepfile_entry)) {
-    Opts.ExtraDeps.push_back(A->getValue());
-  }
+  for (const auto *A : Args.filtered(OPT_fdepfile_entry))
+    Opts.ExtraDeps.emplace_back(A->getValue(), EDK_DepFileEntry);
 
   // Only the -fmodule-file=<file> form.
   for (const auto *A : Args.filtered(OPT_fmodule_file)) {
     StringRef Val = A->getValue();
     if (Val.find('=') == StringRef::npos)
-      Opts.ExtraDeps.push_back(std::string(Val));
+      Opts.ExtraDeps.emplace_back(std::string(Val), EDK_ModuleFile);
   }
+
+  return Success && Diags.getNumErrors() == NumErrorsBefore;
+}
+
+static bool ParseDependencyOutputArgs(CompilerInvocation &Res,
+                                      DependencyOutputOptions &Opts,
+                                      ArgList &Args, DiagnosticsEngine &Diags,
+                                      frontend::ActionKind Action,
+                                      bool ShowLineMarkers) {
+  DependencyOutputOptions DummyOpts;
+
+  return RoundTrip(
+      [Action, ShowLineMarkers](CompilerInvocation &Res, ArgList &Args,
+                                DiagnosticsEngine &Diags) {
+        return ParseDependencyOutputArgsImpl(Res.getDependencyOutputOpts(),
+                                             Args, Diags, Action,
+                                             ShowLineMarkers);
+      },
+      [&Args](CompilerInvocation &Res,
+              SmallVectorImpl<const char *> &GeneratedArgs,
+              CompilerInvocation::StringAllocator SA) {
+        GenerateDependencyOutputArgs(Res.getDependencyOutputOpts(),
+                                     GeneratedArgs, SA);
+        // We're querying sanitizer blacklist and module file arguments, but
+        // they are generated from LanguageOptions and HeaderSearchOptions.
+        // Let's satisfy RoundTrip by generating them ourselves for now.
+        if (!Args.hasArg(OPT_fno_sanitize_blacklist)) {
+          for (const auto *A : Args.filtered(OPT_fsanitize_blacklist)) {
+            StringRef Val = A->getValue();
+            if (Val.find('=') == StringRef::npos)
+              GenerateArg(GeneratedArgs, OPT_fsanitize_blacklist, Val, SA);
+          }
+          if (Res.getDependencyOutputOpts().IncludeSystemHeaders) {
+            for (const auto *A :
+                 Args.filtered(OPT_fsanitize_system_blacklist)) {
+              StringRef Val = A->getValue();
+              if (Val.find('=') == StringRef::npos)
+                GenerateArg(GeneratedArgs, OPT_fsanitize_system_blacklist, Val,
+                            SA);
+            }
+          }
+        }
+
+        for (const auto *A : Args.filtered(OPT_fmodule_file)) {
+          StringRef Val = A->getValue();
+          if (Val.find('=') == StringRef::npos)
+            GenerateArg(GeneratedArgs, OPT_fmodule_file, Val, SA);
+        }
+      },
+      [&DummyOpts](CompilerInvocation &Res) {
+        std::swap(Res.getDependencyOutputOpts(), DummyOpts);
+      },
+      Res, Args, Diags, "DependencyOutputOptions");
 }
 
 static bool parseShowColorsArgs(const ArgList &Args, bool DefaultColor) {
   // Color diagnostics default to auto ("on" if terminal supports) in the driver
   // but default to off in cc1, needing an explicit OPT_fdiagnostics_color.
   // Support both clang's -f[no-]color-diagnostics and gcc's
   // -f[no-]diagnostics-colors[=never|always|auto].
   enum {
else
       Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
           << ArgString << Nearest;
     Success = false;
   }
 
   Success &= Res.parseSimpleArgs(Args, Diags);
 
   Success &= ParseAnalyzerArgs(Res, *Res.getAnalyzerOpts(), Args, Diags);
-  ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), Args);
-  if (!Res.getDependencyOutputOpts().OutputFile.empty() &&
-      Res.getDependencyOutputOpts().Targets.empty()) {
-    Diags.Report(diag::err_fe_dependency_file_requires_MT);
-    Success = false;
-  }
   Success &= ParseDiagnosticArgs(Res.getDiagnosticOpts(), Args, &Diags,
                                  /*DefaultDiagColor=*/false);
   Success &= ParseFrontendArgs(Res, Res.getFrontendOpts(), Args, Diags,
                                LangOpts.IsHeaderFile);
   // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = Res.getFrontendOpts().DashX;
   ParseTargetArgs(Res, Res.getTargetOpts(), Args, Diags);
   llvm::Triple T(Res.getTargetOpts().Triple);
Res.getCodeGenOpts().DiscardValueNames &=
       !LangOpts.Sanitize.has(SanitizerKind::KernelMemory);
 
   ParsePreprocessorArgs(Res, Res.getPreprocessorOpts(), Args, Diags,
                         Res.getFrontendOpts().ProgramAction,
                         Res.getFrontendOpts());
   ParsePreprocessorOutputArgs(Res, Res.getPreprocessorOutputOpts(), Args, Diags,
                               Res.getFrontendOpts().ProgramAction);
 
+  ParseDependencyOutputArgs(Res, Res.getDependencyOutputOpts(), Args, Diags,
+                            Res.getFrontendOpts().ProgramAction,
+                            Res.getPreprocessorOutputOpts().ShowLineMarkers);
+  if (!Res.getDependencyOutputOpts().OutputFile.empty() &&
+      Res.getDependencyOutputOpts().Targets.empty()) {
+    Diags.Report(diag::err_fe_dependency_file_requires_MT);
+    Success = false;
+  }
+
   // Turn on -Wspir-compat for SPIR target.
   if (T.isSPIR())
     Res.getDiagnosticOpts().Warnings.push_back("spir-compat");
 
   // If sanitizer is enabled, disable OPT_ffine_grained_bitfield_accesses.
   if (Res.getCodeGenOpts().FineGrainedBitfieldAccesses &&
       !Res.getLangOpts()->Sanitize.empty()) {
     Res.getCodeGenOpts().FineGrainedBitfieldAccesses = false;
#undef OPTION_WITH_MARSHALLING
   GenerateHeaderSearchArgs(*HeaderSearchOpts, Args, SA);
   GenerateLangArgs(*LangOpts, Args, SA, T);
   GenerateCodeGenArgs(CodeGenOpts, Args, SA, T, FrontendOpts.OutputFile,
                       &*LangOpts);
   GeneratePreprocessorArgs(*PreprocessorOpts, Args, SA, *LangOpts, FrontendOpts,
                            CodeGenOpts);
   GeneratePreprocessorOutputArgs(PreprocessorOutputOpts, Args, SA,
                                  FrontendOpts.ProgramAction);
+  GenerateDependencyOutputArgs(DependencyOutputOpts, Args, SA);
 }
 
 IntrusiveRefCntPtr<llvm::vfs::FileSystem>
 clang::createVFSFromCompilerInvocation(const CompilerInvocation &CI,
                                        DiagnosticsEngine &Diags) {
   return createVFSFromCompilerInvocation(CI, Diags,
                                          llvm::vfs::getRealFileSystem());
 }

clang/lib/Frontend/DependencyFile.cpp

DependencyFileGenerator::DependencyFileGenerator(
     const DependencyOutputOptions &Opts)
     : OutputFile(Opts.OutputFile), Targets(Opts.Targets),
       IncludeSystemHeaders(Opts.IncludeSystemHeaders),
       PhonyTarget(Opts.UsePhonyTargets),
       AddMissingHeaderDeps(Opts.AddMissingHeaderDeps), SeenMissingHeader(false),
       IncludeModuleFiles(Opts.IncludeModuleFiles),
       OutputFormat(Opts.OutputFormat), InputFileIndex(0) {
   for (const auto &ExtraDep : Opts.ExtraDeps) {
-    if (addDependency(ExtraDep))
+    if (addDependency(ExtraDep.first))
       ++InputFileIndex;
   }
 }
 
 void DependencyFileGenerator::attachToPreprocessor(Preprocessor &PP) {
   // Disable the "file not found" diagnostic if the -MG option was given.
   if (AddMissingHeaderDeps)
     PP.SetSuppressIncludeNotFoundError(true);

clang/lib/Frontend/HeaderIncludeGen.cpp

void clang::AttachHeaderIncludeGen(Preprocessor &PP,
   }
 
   // Print header info for extra headers, pretending they were discovered by
   // the regular preprocessor. The primary use case is to support proper
   // generation of Make / Ninja file dependencies for implicit includes, such
   // as sanitizer blacklists. It's only important for cl.exe compatibility,
   // the GNU way to generate rules is -M / -MM / -MD / -MMD.
   for (const auto &Header : DepOpts.ExtraDeps)
-    PrintHeaderInfo(OutputFile, Header, ShowDepth, 2, MSStyle);
+    PrintHeaderInfo(OutputFile, Header.first, ShowDepth, 2, MSStyle);
   PP.addPPCallbacks(std::make_unique<HeaderIncludesCallback>(
       &PP, ShowAllHeaders, OutputFile, DepOpts, OwnsOutputFile, ShowDepth,
       MSStyle));
 }
 
 void HeaderIncludesCallback::FileChanged(SourceLocation Loc,
                                          FileChangeReason Reason,
                                          SrcMgr::CharacteristicKind NewFileType,