[clang][cli] Extract FileSystem and Migrator options parsing/generation

[jansvoboda11]

Summary

This patch splits out the last two option groups (Filesystem and Migrator) into their own Parse/Generate functions.

This effectively removes the need for parseSimpleArgs and marshalling block in CompilerInvocation::generateCC1CommandLine.

The two new Parse/Generate functions are not part of the round-trip, because they contain no custom code and the very next patch starts round-tripping the whole CompilerInvocation.

[dexonsmith]

LGTM.

[dmajor]

This broke the windows bot: http://lab.llvm.org:8011/#/builders/86/builds/7540/steps/6/logs/stdio

FAILED: tools/clang/lib/Frontend/CMakeFiles/obj.clangFrontend.dir/CompilerInvocation.cpp.obj 
C:\PROGRA~2\MICROS~1\2017\COMMUN~1\VC\Tools\MSVC\1416~1.270\bin\Hostx64\x64\cl.exe  /nologo /TP -DGTEST_HAS_RTTI=0 -DUNICODE -D_CRT_NONSTDC_NO_DEPRECATE -D_CRT_NONSTDC_NO_WARNINGS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_SECURE_NO_WARNINGS -D_GNU_SOURCE -D_HAS_EXCEPTIONS=0 -D_SCL_SECURE_NO_DEPRECATE -D_SCL_SECURE_NO_WARNINGS -D_UNICODE -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Itools\clang\lib\Frontend -IC:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\clang\lib\Frontend -IC:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\clang\include -Itools\clang\include -Iinclude -IC:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\llvm\include /DWIN32 /D_WINDOWS   /Zc:inline /Zc:__cplusplus /Zc:strictStrings /Oi /Zc:rvalueCast /bigobj /W4 -wd4141 -wd4146 -wd4244 -wd4267 -wd4291 -wd4351 -wd4456 -wd4457 -wd4458 -wd4459 -wd4503 -wd4624 -wd4722 -wd4100 -wd4127 -wd4512 -wd4505 -wd4610 -wd4510 -wd4702 -wd4245 -wd4706 -wd4310 -wd4701 -wd4703 -wd4389 -wd4611 -wd4805 -wd4204 -wd4577 -wd4091 -wd4592 -wd4319 -wd4709 -wd4324 -w14062 -we4238 /Gw /MD /O2 /Ob2 /DNDEBUG    /EHs-c- /GR- -std:c++14 /showIncludes /Fotools\clang\lib\Frontend\CMakeFiles\obj.clangFrontend.dir\CompilerInvocation.cpp.obj /Fdtools\clang\lib\Frontend\CMakeFiles\obj.clangFrontend.dir\ /FS -c C:\buildbot\as-builder-3\llvm-clang-x86_64-win-fast\llvm-project\clang\lib\Frontend\CompilerInvocation.cpp
tools\clang\include\clang/Driver/Options.inc(6004): error C2065: 'FileSystemOpts': undeclared identifier
tools\clang\include\clang/Driver/Options.inc(6005): note: see reference to function template instantiation 'auto GenerateFileSystemArgs::<lambda_2f26d53f66ebc0b2209fc818ab99b309>::operator ()<T>(const T &) const' being compiled

[jansvoboda11]

In D96278#2553807, @dmajor wrote:

This broke the windows bot: http://lab.llvm.org:8011/#/builders/86/builds/7540/steps/6/logs/stdio

Thanks, I'm pushing a fix. If it doesn't fix the build, I'll revert.

[jansvoboda11]

@dmajor a7150415 turned the build green: http://lab.llvm.org:8011/#/builders/86/builds/7541

[dmajor]

Thank you for the quick followup!

clang/include/clang/Driver/Options.td

class DependencyOutputOpts<string base>
   : KeyPathAndMacro<"DependencyOutputOpts.", base, "DEPENDENCY_OUTPUT_"> {}
 class CodeGenOpts<string base>
   : KeyPathAndMacro<"CodeGenOpts.", base, "CODEGEN_"> {}
 class HeaderSearchOpts<string base>
   : KeyPathAndMacro<"HeaderSearchOpts->", base, "HEADER_SEARCH_"> {}
 class PreprocessorOpts<string base>
   : KeyPathAndMacro<"PreprocessorOpts->", base, "PREPROCESSOR_"> {}
 class FileSystemOpts<string base>
-  : KeyPathAndMacro<"FileSystemOpts.", base> {}
+  : KeyPathAndMacro<"FileSystemOpts.", base, "FILE_SYSTEM_"> {}
 class AnalyzerOpts<string base>
   : KeyPathAndMacro<"AnalyzerOpts->", base, "ANALYZER_"> {}
 class MigratorOpts<string base>
-  : KeyPathAndMacro<"MigratorOpts.", base> {}
+  : KeyPathAndMacro<"MigratorOpts.", base, "MIGRATOR_"> {}
 
 // A boolean option which is opt-in in CC1. The positive option exists in CC1 and
 // Args.hasArg(OPT_ffoo) is used to check that the flag is enabled.
 // This is useful if the option is usually disabled.
 multiclass OptInFFlag<string name, string pos_prefix, string neg_prefix="",
                       string help="", list<OptionFlag> flags=[],
                       KeyPathAndMacro kpm = EmptyKPM,
                       list<string> enablers = []> {

clang/include/clang/Frontend/CompilerInvocation.h

public:
 
   const PreprocessorOutputOptions &getPreprocessorOutputOpts() const {
     return PreprocessorOutputOpts;
   }
 
   /// @}
 
 private:
-  /// Parse options for flags that expose marshalling information in their
-  /// table-gen definition
-  ///
-  /// \param Args - The argument list containing the arguments to parse
-  /// \param Diags - The DiagnosticsEngine associated with CreateFromArgs
-  /// \returns - True if parsing was successful, false otherwise
-  bool parseSimpleArgs(const llvm::opt::ArgList &Args,
-                       DiagnosticsEngine &Diags);
-
   /// Parse command line options from DiagnosticOptions.
   static bool ParseDiagnosticArgsRoundTrip(CompilerInvocation &Res,
                                            DiagnosticOptions &Opts,
                                            llvm::opt::ArgList &Args,
                                            DiagnosticsEngine *Diags = nullptr,
                                            bool DefaultDiagColor = true);
 
   /// Generate command line options from DiagnosticOptions.

clang/lib/Frontend/CompilerInvocation.cpp

if (BadChar != Prefix.end() || !isLetter(Prefix[0])) {
       Success = false;
       Diags.Report(diag::err_drv_invalid_value) << "-verify=" << Prefix;
       Diags.Report(diag::note_drv_verify_prefix_spelling);
     }
   }
   return Success;
 }
 
-bool CompilerInvocation::parseSimpleArgs(const ArgList &Args,
-                                         DiagnosticsEngine &Diags) {
+static void GenerateFileSystemArgs(const FileSystemOptions &FileSystemOpts,
+                                   SmallVectorImpl<const char *> &Args,
+                                   CompilerInvocation::StringAllocator SA) {
+#define FILE_SYSTEM_OPTION_WITH_MARSHALLING(                                   \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  GENERATE_OPTION_WITH_MARSHALLING(                                            \
+      Args, SA, KIND, FLAGS, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,    \
+      IMPLIED_CHECK, IMPLIED_VALUE, DENORMALIZER, EXTRACTOR, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef FILE_SYSTEM_OPTION_WITH_MARSHALLING
+}
+
+static bool ParseFileSystemArgs(FileSystemOptions &FileSystemOpts,
+                                const ArgList &Args, DiagnosticsEngine &Diags) {
+  bool Success = true;
+
+#define FILE_SYSTEM_OPTION_WITH_MARSHALLING(                                   \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  PARSE_OPTION_WITH_MARSHALLING(Args, Diags, Success, ID, FLAGS, PARAM,        \
+                                SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,          \
+                                IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER,      \
+                                MERGER, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef FILE_SYSTEM_OPTION_WITH_MARSHALLING
+
+  return Success;
+}
+
+static void GenerateMigratorArgs(const MigratorOptions &MigratorOpts,
+                                 SmallVectorImpl<const char *> &Args,
+                                 CompilerInvocation::StringAllocator SA) {
+#define MIGRATOR_OPTION_WITH_MARSHALLING(                                      \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  GENERATE_OPTION_WITH_MARSHALLING(                                            \
+      Args, SA, KIND, FLAGS, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,    \
+      IMPLIED_CHECK, IMPLIED_VALUE, DENORMALIZER, EXTRACTOR, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef MIGRATOR_OPTION_WITH_MARSHALLING
+}
+
+static bool ParseMigratorArgs(MigratorOptions &MigratorOpts,
+                              const ArgList &Args, DiagnosticsEngine &Diags) {
   bool Success = true;
 
-#define OPTION_WITH_MARSHALLING(                                               \
+#define MIGRATOR_OPTION_WITH_MARSHALLING(                                      \
     PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
     HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
     DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
     MERGER, EXTRACTOR, TABLE_INDEX)                                            \
   PARSE_OPTION_WITH_MARSHALLING(Args, Diags, Success, ID, FLAGS, PARAM,        \
-                                SHOULD_PARSE, this->KEYPATH, DEFAULT_VALUE,    \
+                                SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,          \
                                 IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER,      \
                                 MERGER, TABLE_INDEX)
 #include "clang/Driver/Options.inc"
-#undef OPTION_WITH_MARSHALLING
+#undef MIGRATOR_OPTION_WITH_MARSHALLING
 
   return Success;
 }
 
 void CompilerInvocation::GenerateDiagnosticArgs(
     const DiagnosticOptions &Opts, SmallVectorImpl<const char *> &Args,
     StringAllocator SA, bool DefaultDiagColor) {
   const DiagnosticOptions *DiagnosticOpts = &Opts;
for (const auto *A : Args.filtered(OPT_UNKNOWN)) {
     if (Opts.findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1)
       Diags.Report(diag::err_drv_unknown_argument) << ArgString;
     else
       Diags.Report(diag::err_drv_unknown_argument_with_suggestion)
           << ArgString << Nearest;
     Success = false;
   }
 
-  Success &= Res.parseSimpleArgs(Args, Diags);
-
+  Success &= ParseFileSystemArgs(Res.getFileSystemOpts(), Args, Diags);
+  Success &= ParseMigratorArgs(Res.getMigratorOpts(), Args, Diags);
   Success &= ParseAnalyzerArgs(Res, *Res.getAnalyzerOpts(), Args, Diags);
   Success &=
       ParseDiagnosticArgsRoundTrip(Res, Res.getDiagnosticOpts(), Args, &Diags,
                                    /*DefaultDiagColor=*/false);
   Success &= ParseFrontendArgs(Res, Res.getFrontendOpts(), Args, Diags,
                                LangOpts.IsHeaderFile);
   // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = Res.getFrontendOpts().DashX;
#include "clang/Basic/LangOptions.def"
   if (!SanHash.empty())
     code = hash_combine(code, SanHash.Mask);
 
   return llvm::APInt(64, code).toString(36, /*Signed=*/false);
 }
 
 void CompilerInvocation::generateCC1CommandLine(
     SmallVectorImpl<const char *> &Args, StringAllocator SA) const {
-#define OPTION_WITH_MARSHALLING(                                               \
-    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
-    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
-    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
-    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
-  GENERATE_OPTION_WITH_MARSHALLING(Args, SA, KIND, FLAGS, SPELLING,            \
-                                   ALWAYS_EMIT, this->KEYPATH, DEFAULT_VALUE,  \
-                                   IMPLIED_CHECK, IMPLIED_VALUE, DENORMALIZER, \
-                                   EXTRACTOR, TABLE_INDEX)
-#include "clang/Driver/Options.inc"
-#undef OPTION_WITH_MARSHALLING
-
   llvm::Triple T(TargetOpts->Triple);
 
+  GenerateFileSystemArgs(FileSystemOpts, Args, SA);
+  GenerateMigratorArgs(MigratorOpts, Args, SA);
   GenerateAnalyzerArgs(*AnalyzerOpts, Args, SA);
   GenerateDiagnosticArgs(*DiagnosticOpts, Args, SA, false);
   GenerateFrontendArgs(FrontendOpts, Args, SA, LangOpts->IsHeaderFile);
   GenerateTargetArgs(*TargetOpts, Args, SA);
   GenerateHeaderSearchArgs(*HeaderSearchOpts, Args, SA);
   GenerateLangArgs(*LangOpts, Args, SA, T);
   GenerateCodeGenArgs(CodeGenOpts, Args, SA, T, FrontendOpts.OutputFile,
                       &*LangOpts);