[clang][cli] Generate and round-trip Frontend options

[jansvoboda11]

Summary

This patch implements generation of remaining frontend options and tests it by performing parse-generate-parse round trip.

Depends on D96269.

[jansvoboda11]

Document getProgramActionOpt

[jansvoboda11]

Fix whitespace

[jansvoboda11]

clang/include/clang/Serialization/ModuleFileExtension.h

75

I'm not sure what the best approach is here.

ModuleFileExtension is an abstract class. When parsing command line arguments, we may instantiate its (only) subclass TestModuleFileExtension.
To generate the argument, we can:

• Add a pure virtual function (e.g. serialize) to ModuleFileExtension, implement it in TestModuleFileExtension and use that in the generator. I'm not sure if introducing the concept of command line and serialization into argument makes sense. For clients subclassing ModuleFileExtension, serialization might not make sense.
• (The current approach): Add LLVM-style RTTI to the whole hierarchy, and only handle TestModuleFileExtension in the generator by invoking its (non-virtual) serialize.

WDYT would be the best approach here?

[dexonsmith]

LGTM, although the description seems incorrect:

This patch implements generation of remaining preprocessor options and tests it by performing parse-generate-parse round trip.

In fact, this patch is working with frontend options. Please fix for the commit message.

[jansvoboda11]

Thanks, a copy-paste mistake.

clang/include/clang/Driver/Options.td

 
 class DiagnosticOpts<string base>
   : KeyPathAndMacro<"DiagnosticOpts->", base, "DIAG_"> {}
 class LangOpts<string base>
   : KeyPathAndMacro<"LangOpts->", base, "LANG_"> {}
 class TargetOpts<string base>
   : KeyPathAndMacro<"TargetOpts->", base> {}
 class FrontendOpts<string base>
-  : KeyPathAndMacro<"FrontendOpts.", base> {}
+  : KeyPathAndMacro<"FrontendOpts.", base, "FRONTEND_"> {}
 class PreprocessorOutputOpts<string base>
   : KeyPathAndMacro<"PreprocessorOutputOpts.", base> {}
 class DependencyOutputOpts<string base>
   : KeyPathAndMacro<"DependencyOutputOpts.", base> {}
 class CodeGenOpts<string base>
   : KeyPathAndMacro<"CodeGenOpts.", base, "CODEGEN_"> {}
 class HeaderSearchOpts<string base>
   : KeyPathAndMacro<"HeaderSearchOpts->", base, "HEADER_SEARCH_"> {}

clang/include/clang/Frontend/CommandLineSourceLoc.h

if (!ColSplit.second.getAsInteger(10, PSL.Column) &&
       // On the command-line, stdin may be specified via "-". Inside the
       // compiler, stdin is called "<stdin>".
       if (PSL.FileName == "-")
         PSL.FileName = "<stdin>";
     }
 
     return PSL;
   }
+
+  /// Serialize ParsedSourceLocation back to a string.
+  std::string ToString() const {
+    return (llvm::Twine(FileName == "<stdin>" ? "-" : FileName) + ":" +
+            Twine(Line) + ":" + Twine(Column))
+        .str();
+  }
 };
 
 /// A source range that has been parsed on the command line.
 struct ParsedSourceRange {
   std::string FileName;
   /// The starting location of the range. The first element is the line and
   /// the second element is the column.
   std::pair<unsigned, unsigned> Begin;

clang/include/clang/Serialization/ModuleFileExtension.h

 /// An abstract superclass that describes a custom extension to the
 /// module/precompiled header file format.
 ///
 /// A module file extension can introduce additional information into
 /// compiled module files (.pcm) and precompiled headers (.pch) via a
 /// custom writer that can then be accessed via a custom reader when
 /// the module file or precompiled header is loaded.
 class ModuleFileExtension {
+protected:
+  /// Discriminator for LLVM-style RTTI.
+  enum ModuleFileExtensionKind {
+    MFEK_Test,
+  };
+
+  const ModuleFileExtensionKind Kind;
 public:
+  ModuleFileExtension(ModuleFileExtensionKind Kind) : Kind(Kind) {}
+
   virtual ~ModuleFileExtension();
 
+  ModuleFileExtensionKind getKind() const { return Kind; }
+
   /// Retrieves the metadata for this module file extension.
   virtual ModuleFileExtensionMetadata getExtensionMetadata() const = 0;
 
   /// Hash information about the presence of this extension into the
   /// module hash code.
   ///
   /// The module hash code is used to distinguish different variants
   /// of a module that are incompatible. If the presence, absence, or

clang/lib/Frontend/CompilerInvocation.cpp

 }
 
 template <typename T> static T extractForwardValue(T KeyPath) {
   return KeyPath;
 }
 
 template <typename T, typename U, U Value>
 static T extractMaskValue(T KeyPath) {
-  return KeyPath & Value;
+  return ((KeyPath & Value) == Value) ? Value : T();
 }
 
 #define PARSE_OPTION_WITH_MARSHALLING(ARGS, DIAGS, SUCCESS, ID, FLAGS, PARAM,  \
                                       SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,    \
                                       IMPLIED_CHECK, IMPLIED_VALUE,            \
                                       NORMALIZER, MERGER, TABLE_INDEX)         \
   if ((FLAGS)&options::CC1Option) {                                            \
     KEYPATH = MERGER(KEYPATH, DEFAULT_VALUE);                                  \

 static Optional<frontend::ActionKind> getFrontendAction(OptSpecifier &Opt) {
   for (const auto &ActionOpt : getFrontendActionTable())
     if (ActionOpt.second == Opt.getID())
       return ActionOpt.first;
 
   return None;
 }
 
-static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
-                              DiagnosticsEngine &Diags, bool &IsHeaderFile) {
+/// Maps frontend action to command line option.
+static Optional<OptSpecifier>
+getProgramActionOpt(frontend::ActionKind ProgramAction) {
+  for (const auto &ActionOpt : getFrontendActionTable())
+    if (ActionOpt.first == ProgramAction)
+      return OptSpecifier(ActionOpt.second);
+
+  return None;
+}
+
+static void GenerateFrontendArgs(const FrontendOptions &Opts,
+                                 SmallVectorImpl<const char *> &Args,
+                                 CompilerInvocation::StringAllocator SA,
+                                 bool IsHeader) {
+  const FrontendOptions &FrontendOpts = Opts;
+#define FRONTEND_OPTION_WITH_MARSHALLING(                                      \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  GENERATE_OPTION_WITH_MARSHALLING(                                            \
+      Args, SA, KIND, FLAGS, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,    \
+      IMPLIED_CHECK, IMPLIED_VALUE, DENORMALIZER, EXTRACTOR, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef FRONTEND_OPTION_WITH_MARSHALLING
+
+  Optional<OptSpecifier> ProgramActionOpt =
+      getProgramActionOpt(Opts.ProgramAction);
+
+  // Generating a simple flag covers most frontend actions.
+  std::function<void()> GenerateProgramAction = [&]() {
+    GenerateArg(Args, *ProgramActionOpt, SA);
+  };
+
+  if (!ProgramActionOpt) {
+    // PluginAction is the only program action handled separately.
+    assert(Opts.ProgramAction == frontend::PluginAction &&
+           "Frontend action without option.");
+    GenerateProgramAction = [&]() {
+      GenerateArg(Args, OPT_plugin, Opts.ActionName, SA);
+    };
+  }
+
+  // FIXME: Simplify the complex 'AST dump' command line.
+  if (Opts.ProgramAction == frontend::ASTDump) {
+    GenerateProgramAction = [&]() {
+      // ASTDumpLookups, ASTDumpDeclTypes and ASTDumpFilter are generated via
+      // marshalling infrastructure.
+
+      if (Opts.ASTDumpFormat != ADOF_Default) {
+        StringRef Format;
+        switch (Opts.ASTDumpFormat) {
+        case ADOF_Default:
+          llvm_unreachable("Default AST dump format.");
+        case ADOF_JSON:
+          Format = "json";
+          break;
+        }
+
+        if (Opts.ASTDumpAll)
+          GenerateArg(Args, OPT_ast_dump_all_EQ, Format, SA);
+        if (Opts.ASTDumpDecls)
+          GenerateArg(Args, OPT_ast_dump_EQ, Format, SA);
+      } else {
+        if (Opts.ASTDumpAll)
+          GenerateArg(Args, OPT_ast_dump_all, SA);
+        if (Opts.ASTDumpDecls)
+          GenerateArg(Args, OPT_ast_dump, SA);
+      }
+    };
+  }
+
+  if (Opts.ProgramAction == frontend::FixIt && !Opts.FixItSuffix.empty()) {
+    GenerateProgramAction = [&]() {
+      GenerateArg(Args, OPT_fixit_EQ, Opts.FixItSuffix, SA);
+    };
+  }
+
+  GenerateProgramAction();
+
+  for (const auto &PluginArgs : Opts.PluginArgs)
+    for (const auto &PluginArg : PluginArgs.second)
+      GenerateArg(Args, OPT_plugin_arg, PluginArgs.first + PluginArg, SA);
+
+  for (const auto &Ext : Opts.ModuleFileExtensions) {
+    if (auto *TestExt = dyn_cast_or_null<TestModuleFileExtension>(Ext.get())) {
+      std::string Buffer;
+      llvm::raw_string_ostream OS(Buffer);
+      OS << *TestExt;
+      GenerateArg(Args, OPT_ftest_module_file_extension_EQ, OS.str(), SA);
+    }
+  }
+
+  if (!Opts.CodeCompletionAt.FileName.empty())
+    GenerateArg(Args, OPT_code_completion_at, Opts.CodeCompletionAt.ToString(),
+                SA);
+
+  for (const auto &Plugin : Opts.Plugins)
+    GenerateArg(Args, OPT_load, Plugin, SA);
+
+  // ASTDumpDecls and ASTDumpAll already handled with ProgramAction.
+
+  for (const auto &ModuleFile : Opts.ModuleFiles)
+    GenerateArg(Args, OPT_fmodule_file, ModuleFile, SA);
+
+  if (Opts.AuxTargetCPU.hasValue())
+    GenerateArg(Args, OPT_aux_target_cpu, *Opts.AuxTargetCPU, SA);
+
+  if (Opts.AuxTargetFeatures.hasValue())
+    for (const auto &Feature : *Opts.AuxTargetFeatures)
+      GenerateArg(Args, OPT_aux_target_feature, Feature, SA);
+
+  {
+    StringRef Preprocessed = Opts.DashX.isPreprocessed() ? "-cpp-output" : "";
+    StringRef ModuleMap =
+        Opts.DashX.getFormat() == InputKind::ModuleMap ? "-module-map" : "";
+    StringRef Header = IsHeader ? "-header" : "";
+
+    StringRef Lang;
+    switch (Opts.DashX.getLanguage()) {
+    case Language::C:
+      Lang = "c";
+      break;
+    case Language::OpenCL:
+      Lang = "cl";
+      break;
+    case Language::CUDA:
+      Lang = "cuda";
+      break;
+    case Language::HIP:
+      Lang = "hip";
+      break;
+    case Language::CXX:
+      Lang = "c++";
+      break;
+    case Language::ObjC:
+      Lang = "objective-c";
+      break;
+    case Language::ObjCXX:
+      Lang = "objective-c++";
+      break;
+    case Language::RenderScript:
+      Lang = "renderscript";
+      break;
+    case Language::Asm:
+      Lang = "assembler-with-cpp";
+      break;
+    case Language::Unknown:
+      assert(Opts.DashX.getFormat() == InputKind::Precompiled &&
+             "Generating -x argument for unknown language (not precompiled).");
+      Lang = "ast";
+      break;
+    case Language::LLVM_IR:
+      Lang = "ir";
+      break;
+    }
+
+    GenerateArg(Args, OPT_x, Lang + Header + ModuleMap + Preprocessed, SA);
+  }
+
+  // OPT_INPUT has a unique class, generate it directly.
+  for (const auto &Input : Opts.Inputs)
+    Args.push_back(SA(Input.getFile()));
+}
+
+static bool ParseFrontendArgsImpl(FrontendOptions &Opts, ArgList &Args,
+                                  DiagnosticsEngine &Diags,
+                                  bool &IsHeaderFile) {
+  FrontendOptions &FrontendOpts = Opts;
+  bool Success = true;
   unsigned NumErrorsBefore = Diags.getNumErrors();
+#define FRONTEND_OPTION_WITH_MARSHALLING(                                      \
+    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
+    HELPTEXT, METAVAR, VALUES, SPELLING, SHOULD_PARSE, ALWAYS_EMIT, KEYPATH,   \
+    DEFAULT_VALUE, IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER,     \
+    MERGER, EXTRACTOR, TABLE_INDEX)                                            \
+  PARSE_OPTION_WITH_MARSHALLING(Args, Diags, Success, ID, FLAGS, PARAM,        \
+                                SHOULD_PARSE, KEYPATH, DEFAULT_VALUE,          \
+                                IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER,      \
+                                MERGER, TABLE_INDEX)
+#include "clang/Driver/Options.inc"
+#undef FRONTEND_OPTION_WITH_MARSHALLING
 
   Opts.ProgramAction = frontend::ParseSyntaxOnly;
   if (const Arg *A = Args.getLastArg(OPT_Action_Group)) {
     OptSpecifier Opt = OptSpecifier(A->getOption().getID());
     Optional<frontend::ActionKind> ProgramAction = getFrontendAction(Opt);
     assert(ProgramAction && "Option specifier not in Action_Group.");
 
     if (ProgramAction == frontend::ASTDump &&
for (unsigned i = 0, e = Inputs.size(); i != e; ++i) {
     Opts.Inputs.emplace_back(std::move(Inputs[i]), IK, IsSystem);
   }
 
   Opts.DashX = DashX;
 
   return Diags.getNumErrors() == NumErrorsBefore;
 }
 
+static bool ParseFrontendArgs(CompilerInvocation &Res, FrontendOptions &Opts,
+                              ArgList &Args, DiagnosticsEngine &Diags,
+                              bool &IsHeaderFile) {
+  FrontendOptions DummyOpts;
+
+  return RoundTrip(
+      [&IsHeaderFile](CompilerInvocation &Res, ArgList &Args,
+                      DiagnosticsEngine &Diags) {
+        // ParseFrontendArgsImpl handles frontend action without querying the
+        // options. Let's do it now so RoundTrip considers us responsible for
+        // generating it.
+        for (const auto &Pair : getFrontendActionTable())
+          Args.hasArg(Pair.second);
+
+        return ParseFrontendArgsImpl(Res.getFrontendOpts(), Args, Diags,
+                                     IsHeaderFile);
+      },
+      [&IsHeaderFile](CompilerInvocation &Res,
+                      SmallVectorImpl<const char *> &Args,
+                      CompilerInvocation::StringAllocator SA) {
+        GenerateFrontendArgs(Res.getFrontendOpts(), Args, SA, IsHeaderFile);
+      },
+      [&DummyOpts](CompilerInvocation &Res) {
+        std::swap(Res.getFrontendOpts(), DummyOpts);
+      },
+      Res, Args, Diags, "FrontendOptions");
+}
+
 std::string CompilerInvocation::GetResourcesPath(const char *Argv0,
                                                  void *MainAddr) {
   std::string ClangExecutable =
       llvm::sys::fs::getMainExecutable(Argv0, MainAddr);
   return Driver::GetResourcesPath(ClangExecutable, CLANG_RESOURCE_DIR);
 }
 
 void CompilerInvocation::GenerateHeaderSearchArgs(
bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), Args);
   if (!Res.getDependencyOutputOpts().OutputFile.empty() &&
       Res.getDependencyOutputOpts().Targets.empty()) {
     Diags.Report(diag::err_fe_dependency_file_requires_MT);
     Success = false;
   }
   Success &= ParseDiagnosticArgs(Res.getDiagnosticOpts(), Args, &Diags,
                                  /*DefaultDiagColor=*/false);
-  Success &= ParseFrontendArgs(Res.getFrontendOpts(), Args, Diags,
+  Success &= ParseFrontendArgs(Res, Res.getFrontendOpts(), Args, Diags,
                                LangOpts.IsHeaderFile);
   // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = Res.getFrontendOpts().DashX;
   ParseTargetArgs(Res.getTargetOpts(), Args, Diags);
   llvm::Triple T(Res.getTargetOpts().Triple);
   ParseHeaderSearchArgs(Res, Res.getHeaderSearchOpts(), Args, Diags,
                         Res.getFileSystemOpts().WorkingDir);
   if (DashX.getFormat() == InputKind::Precompiled ||

 #include "clang/Driver/Options.inc"
 
 #undef DIAG_OPTION_WITH_MARSHALLING
 #undef OPTION_WITH_MARSHALLING
 
   llvm::Triple T(TargetOpts->Triple);
 
   GenerateAnalyzerArgs(*AnalyzerOpts, Args, SA);
+  GenerateFrontendArgs(FrontendOpts, Args, SA, LangOpts->IsHeaderFile);
   GenerateHeaderSearchArgs(*HeaderSearchOpts, Args, SA);
   GenerateLangArgs(*LangOpts, Args, SA, T);
   GenerateCodeGenArgs(CodeGenOpts, Args, SA, T, FrontendOpts.OutputFile,
                       &*LangOpts);
   GeneratePreprocessorArgs(*PreprocessorOpts, Args, SA, *LangOpts,
                            FrontendOpts, CodeGenOpts);
 }
 

clang/lib/Frontend/TestModuleFileExtension.h

class TestModuleFileExtension : public ModuleFileExtension {
   };
 
 public:
   TestModuleFileExtension(StringRef BlockName,
                           unsigned MajorVersion,
                           unsigned MinorVersion,
                           bool Hashed,
                           StringRef UserInfo)
-    : BlockName(BlockName),
+    : ModuleFileExtension(ModuleFileExtensionKind::MFEK_Test),
+      BlockName(BlockName),
       MajorVersion(MajorVersion), MinorVersion(MinorVersion),
       Hashed(Hashed), UserInfo(UserInfo) { }
   ~TestModuleFileExtension() override;
 
   ModuleFileExtensionMetadata getExtensionMetadata() const override;
 
   llvm::hash_code hashExtension(llvm::hash_code Code) const override;
 
   std::unique_ptr<ModuleFileExtensionWriter>
   createExtensionWriter(ASTWriter &Writer) override;
 
   std::unique_ptr<ModuleFileExtensionReader>
   createExtensionReader(const ModuleFileExtensionMetadata &Metadata,
                         ASTReader &Reader, serialization::ModuleFile &Mod,
                         const llvm::BitstreamCursor &Stream) override;
+
+  static bool classof(const ModuleFileExtension *E) {
+    return E->getKind() == MFEK_Test;
+  }
+
+  /// Serialize the extension.
+  friend llvm::raw_ostream &
+  operator<<(llvm::raw_ostream &OS, const TestModuleFileExtension &Extension);
 };
 
 } // end namespace clang
 
 #endif // LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H

clang/lib/Frontend/TestModuleFileExtension.cpp

Reader.getDiags().Report(Mod.ImportLoc,
       << BlockName << Metadata.MajorVersion << Metadata.MinorVersion
       << MajorVersion << MinorVersion;
     return nullptr;
   }
 
   return std::unique_ptr<ModuleFileExtensionReader>(
                                                     new TestModuleFileExtension::Reader(this, Stream));
 }
+
+llvm::raw_ostream &clang::operator<<(llvm::raw_ostream &OS,
+                                     const TestModuleFileExtension &Extension) {
+  return OS << Extension.BlockName << ":" << Extension.MajorVersion << ":"
+            << Extension.MinorVersion << ":" << Extension.Hashed << ":"
+            << Extension.UserInfo;
+}