[Modules] Allow generation of BMIs in the same job as compiled output.

[iains]

Summary

This adds an AST consumer to the pipelines for 'regular' compilation jobs
which can generate a binary module interface (BMI) when one is required
by the source, rather than making this a separate step.

We only write output if a module interface is required for the source (e.g.
it contains an exported named module).

This initial version has a simplistic approach to determining a default
path and name for the binary module interface.

The patch is preparatory work for other opportunites - for example that
with distinct channels for codegen and BMI output, we can now consider
actions to prune unnecessary content from the BMIs.

[iains]

rebased, edited descriptions and text to use BMI instead of CMI.

[iains]

this will be updated and moved to a pull request

clang/lib/Frontend/FrontendAction.cpp

 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclGroup.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileEntry.h"
 #include "clang/Basic/LangStandard.h"
 #include "clang/Basic/Sarif.h"
+#include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendPluginRegistry.h"
 #include "clang/Frontend/LayoutOverrideSource.h"
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Frontend/SARIFDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Parse/ParseAST.h"
 #include "clang/Sema/HLSLExternalSemaSource.h"
 #include "clang/Sema/MultiplexExternalSemaSource.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "clang/Serialization/GlobalModuleIndex.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/BuryPointer.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
for (const std::string &Arg : CI.getFrontendOpts().AddPluginActions) {
     if (!Found) {
       CI.getDiagnostics().Report(diag::err_fe_invalid_plugin_name) << Arg;
       FoundAllPlugins = false;
     }
   }
   if (!FoundAllPlugins)
     return nullptr;
 
-  // If there are no registered plugins we don't need to wrap the consumer
-  if (FrontendPluginRegistry::begin() == FrontendPluginRegistry::end())
-    return Consumer;
-
   // If this is a code completion run, avoid invoking the plugin consumers
   if (CI.hasCodeCompletionConsumer())
     return Consumer;
 
+  // Do we want to emit a BMI as a second artefact of the compile, on demand if
+  // the source generates an interface?
+  // We do this for C++20 modules, if the input is source, we do not stop after
+  // the Preprocessor and we intend to emit an output.  Note that we do not need
+  // to consider the case in which a BMI is explicitly the main output of the
+  // compilation.
+  InputKind IK = getCurrentFileKind();
+  bool EmitBMI = CI.getLangOpts().CPlusPlusModules &&
+                 IK.getFormat() == InputKind::Format::Source &&
+                 !this->usesPreprocessorOnly() &&
+                (CI.getFrontendOpts().ProgramAction == frontend::EmitObj ||
+                 CI.getFrontendOpts().ProgramAction == frontend::EmitAssembly ||
+                 CI.getFrontendOpts().ProgramAction == frontend::EmitBC ||
+                 CI.getFrontendOpts().ProgramAction == frontend::EmitLLVM ||
+                 CI.getFrontendOpts().ProgramAction == frontend::EmitLLVMOnly ||
+                 CI.getFrontendOpts().ProgramAction == frontend::EmitCodeGenOnly);
+
+  // If there are no registered plugins and we do not need to emit a BMI, we
+  // do not need to wrap the consumer in a MultiplexConsumer.
+  if (!EmitBMI &&
+      FrontendPluginRegistry::begin() == FrontendPluginRegistry::end())
+    return Consumer;
+
+  // List of AST consumers for this source.
+  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+
+  // First, add a pair of consumers that will write the AST as a CMI for this
+  // module.  ??? : Should any plugins that precede the main consumer also be
+  // run before this.
+  if (EmitBMI) {
+    // Make a default output filename (this would be overwritten by the one
+    // derived from the module to BMI name-mapping determined from any export
+    // statement).  We do not open this on the output stream, but provide it
+    // as a fallback.
+    // The default here is to output the pcm alongside the main output.
+    std::string XOut
+      = llvm::sys::path::parent_path(CI.getFrontendOpts().OutputFile).str();
+    std::string XIn = llvm::sys::path::filename(InFile).str();
+    if (!XOut.empty()) {
+      XOut += llvm::sys::path::get_separator();
+      XOut += XIn;
+    } else
+      XOut = XIn;
+
+    SmallString<128> Path(XOut);
+    llvm::sys::path::replace_extension(Path, "pcm");
+    XOut = std::string(Path.str());
+
+    std::unique_ptr<raw_pwrite_stream> OS;
+    std::string Sysroot;
+    auto Buffer = std::make_shared<PCHBuffer>();
+
+    // Add a job to build the CMI from the AST.
+    // ??? : change the CTOR flags to note that this is a CXX20 module?
+    Consumers.push_back(std::make_unique<PCHGenerator>(
+      CI.getPreprocessor(), CI.getModuleCache(), XOut, Sysroot, Buffer,
+      CI.getFrontendOpts().ModuleFileExtensions,
+      /*AllowASTWithErrors=*/false,
+      /*IncludeTimestamps=*/false,
+      /*ShouldCacheASTInMemory=*/false,
+      /*IsForBMI=*/true));
+
+    // This writes the CMI (if one is needed), but does not open the output
+    // file unless/until it is required.
+    Consumers.push_back(CI.getPCHContainerWriter()
+                        .CreatePCHDeferredContainerGenerator(
+                        CI, std::string(InFile), XOut, std::move(OS), Buffer));
+  }
+
   // Collect the list of plugins that go before the main action (in Consumers)
   // or after it (in AfterConsumers)
-  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+
   std::vector<std::unique_ptr<ASTConsumer>> AfterConsumers;
   for (const FrontendPluginRegistry::entry &Plugin :
        FrontendPluginRegistry::entries()) {
     std::unique_ptr<PluginASTAction> P = Plugin.instantiate();
     PluginASTAction::ActionType ActionType = P->getActionType();
     if (ActionType == PluginASTAction::CmdlineAfterMainAction ||
         ActionType == PluginASTAction::CmdlineBeforeMainAction) {
       // This is O(|plugins| * |add_plugins|), but since both numbers are