[Modules] Allow outputing BMIs in the same job as other artefacts.

[iains]

Summary

This provides a base implementation for the ability to emit binary
module interfaces in the same compilation as generating object or
other output.

This part defers opening the BMI PCM file until it is actually needed.
At this point we will have parsed the TU and therefore can determine
if this job should emit both the BMI and some other artefact (in many
cases an object file).

This initial implementation assumes that the output filename for the
potential BMI will be known when the job is created.

[iains]

rebased and updated to change 'CMI' to 'BMI' in the text.

[iains]

this will be updated and moved to a pull request

clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h

class ObjectFilePCHContainerWriter : public PCHContainerWriter {
   /// PCHGenerator that produces a wrapper file format
   /// that also contains full debug info for the module.
   std::unique_ptr<ASTConsumer>
   CreatePCHContainerGenerator(CompilerInstance &CI,
                               const std::string &MainFileName,
                               const std::string &OutputFileName,
                               std::unique_ptr<llvm::raw_pwrite_stream> OS,
                               std::shared_ptr<PCHBuffer> Buffer) const override;
+
+  std::unique_ptr<ASTConsumer> CreatePCHDeferredContainerGenerator(
+      CompilerInstance &CI, const std::string &MainFileName,
+      const std::string &OutputFileName,
+      std::unique_ptr<llvm::raw_pwrite_stream> OS,
+      std::shared_ptr<PCHBuffer> Buffer) const override;
 };
 
 /// A PCHContainerReader implementation that uses LLVM to
 /// wraps Clang modules inside a COFF, ELF, or Mach-O container.
 class ObjectFilePCHContainerReader : public PCHContainerReader {
   ArrayRef<StringRef> getFormats() const override;
 
   /// Returns the serialized AST inside the PCH container Buffer.
   StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
 };
 }
 
 #endif

clang/include/clang/Serialization/ASTWriter.h

class PCHGenerator : public SemaConsumer {
   std::string OutputFile;
   std::string isysroot;
   Sema *SemaPtr;
   std::shared_ptr<PCHBuffer> Buffer;
   llvm::BitstreamWriter Stream;
   ASTWriter Writer;
   bool AllowASTWithErrors;
   bool ShouldCacheASTInMemory;
+  bool IsForBMI;
 
 protected:
   ASTWriter &getWriter() { return Writer; }
   const ASTWriter &getWriter() const { return Writer; }
   SmallVectorImpl<char> &getPCH() const { return Buffer->Data; }
 
 public:
   PCHGenerator(const Preprocessor &PP, InMemoryModuleCache &ModuleCache,
                StringRef OutputFile, StringRef isysroot,
                std::shared_ptr<PCHBuffer> Buffer,
                ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
                bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
-               bool ShouldCacheASTInMemory = false);
+               bool ShouldCacheASTInMemory = false, bool IsForBMI = false);
   ~PCHGenerator() override;
 
   void InitializeSema(Sema &S) override { SemaPtr = &S; }
   void HandleTranslationUnit(ASTContext &Ctx) override;
   ASTMutationListener *GetASTMutationListener() override;
   ASTDeserializationListener *GetASTDeserializationListener() override;
   bool hasEmittedPCH() const { return Buffer->IsComplete; }
 };
 
 } // namespace clang
 
 #endif // LLVM_CLANG_SERIALIZATION_ASTWRITER_H

clang/include/clang/Serialization/PCHContainerOperations.h

 
 namespace clang {
 
 class ASTConsumer;
 class CompilerInstance;
 
 struct PCHBuffer {
   ASTFileSignature Signature;
+  std::string PresumedFileName;
   llvm::SmallVector<char, 0> Data;
   bool IsComplete;
 };
 
 /// This abstract interface provides operations for creating
 /// containers for serialized ASTs (precompiled headers and clang
 /// modules).
 class PCHContainerWriter {
 public:
   virtual ~PCHContainerWriter() = 0;
   virtual llvm::StringRef getFormat() const = 0;
 
   /// Return an ASTConsumer that can be chained with a
   /// PCHGenerator that produces a wrapper file format containing a
   /// serialized AST bitstream.
   virtual std::unique_ptr<ASTConsumer>
   CreatePCHContainerGenerator(CompilerInstance &CI,
                               const std::string &MainFileName,
                               const std::string &OutputFileName,
                               std::unique_ptr<llvm::raw_pwrite_stream> OS,
                               std::shared_ptr<PCHBuffer> Buffer) const = 0;
+
+  /// Return an ASTConsumer that can be chained with a
+  /// PCHGenerator that produces a wrapper file format containing a
+  /// serialized AST bitstream.
+  virtual std::unique_ptr<ASTConsumer> CreatePCHDeferredContainerGenerator(
+      CompilerInstance &CI, const std::string &MainFileName,
+      const std::string &OutputFileName,
+      std::unique_ptr<llvm::raw_pwrite_stream> OS,
+      std::shared_ptr<PCHBuffer> Buffer) const = 0;
 };
 
 /// This abstract interface provides operations for unwrapping
 /// containers for serialized ASTs (precompiled headers and clang
 /// modules).
 class PCHContainerReader {
 public:
   virtual ~PCHContainerReader() = 0;
class RawPCHContainerWriter : public PCHContainerWriter {
   /// Return an ASTConsumer that can be chained with a
   /// PCHGenerator that writes the module to a flat file.
   std::unique_ptr<ASTConsumer>
   CreatePCHContainerGenerator(CompilerInstance &CI,
                               const std::string &MainFileName,
                               const std::string &OutputFileName,
                               std::unique_ptr<llvm::raw_pwrite_stream> OS,
                               std::shared_ptr<PCHBuffer> Buffer) const override;
+
+  std::unique_ptr<ASTConsumer> CreatePCHDeferredContainerGenerator(
+      CompilerInstance &CI, const std::string &MainFileName,
+      const std::string &OutputFileName,
+      std::unique_ptr<llvm::raw_pwrite_stream> OS,
+      std::shared_ptr<PCHBuffer> Buffer) const override;
 };
 
 /// Implements read operations for a raw pass-through PCH container.
 class RawPCHContainerReader : public PCHContainerReader {
   llvm::ArrayRef<llvm::StringRef> getFormats() const override;
   /// Simply returns the buffer contained in Buffer.
   llvm::StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const override;
 };

clang/lib/CodeGen/ObjectFilePCHContainerOperations.cpp

return std::make_unique<PCHContainerGenerator>(
       CI, MainFileName, OutputFileName, std::move(OS), Buffer);
 }
 
 ArrayRef<StringRef> ObjectFilePCHContainerReader::getFormats() const {
   static StringRef Formats[] = {"obj", "raw"};
   return Formats;
 }
 
+std::unique_ptr<ASTConsumer>
+ObjectFilePCHContainerWriter::CreatePCHDeferredContainerGenerator(
+    CompilerInstance &CI, const std::string &MainFileName,
+    const std::string &OutputFileName,
+    std::unique_ptr<llvm::raw_pwrite_stream> OS,
+    std::shared_ptr<PCHBuffer> Buffer) const {
+  assert(0 && "Did not mean to arrive here");
+  return nullptr;
+}
+
 StringRef
 ObjectFilePCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   StringRef PCH;
   auto OFOrErr = llvm::object::ObjectFile::createObjectFile(Buffer);
   if (OFOrErr) {
     auto &OF = OFOrErr.get();
     bool IsCOFF = isa<llvm::object::COFFObjectFile>(*OF);
     // Find the clang AST section in the container.

clang/lib/Serialization/GeneratePCH.cpp

 //===--- GeneratePCH.cpp - Sema Consumer for PCH Generation -----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file defines the PCHGenerator, which as a SemaConsumer that generates
 //  a PCH file.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 
 using namespace clang;
 
 PCHGenerator::PCHGenerator(
     const Preprocessor &PP, InMemoryModuleCache &ModuleCache,
     StringRef OutputFile, StringRef isysroot, std::shared_ptr<PCHBuffer> Buffer,
     ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
     bool AllowASTWithErrors, bool IncludeTimestamps,
-    bool ShouldCacheASTInMemory)
+    bool ShouldCacheASTInMemory, bool IsForBMI)
     : PP(PP), OutputFile(OutputFile), isysroot(isysroot.str()),
       SemaPtr(nullptr), Buffer(std::move(Buffer)), Stream(this->Buffer->Data),
       Writer(Stream, this->Buffer->Data, ModuleCache, Extensions,
              IncludeTimestamps),
       AllowASTWithErrors(AllowASTWithErrors),
-      ShouldCacheASTInMemory(ShouldCacheASTInMemory) {
+      ShouldCacheASTInMemory(ShouldCacheASTInMemory), IsForBMI(IsForBMI) {
   this->Buffer->IsComplete = false;
 }
 
 PCHGenerator::~PCHGenerator() {
 }
 
 void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   // Don't create a PCH if there were fatal failures during module loading.
   if (PP.getModuleLoader().HadFatalFailure)
     return;
 
   bool hasErrors = PP.getDiagnostics().hasErrorOccurred();
   if (hasErrors && !AllowASTWithErrors)
     return;
 
   Module *Module = nullptr;
-  if (PP.getLangOpts().isCompilingModule()) {
+  if (PP.getLangOpts().isCompilingModule() || IsForBMI) {
     Module = PP.getHeaderSearchInfo().lookupModule(
         PP.getLangOpts().CurrentModule, SourceLocation(),
         /*AllowSearch*/ false);
     if (!Module) {
-      assert(hasErrors && "emitting module but current module doesn't exist");
+      // If we have errors, then that might have prevented the creation of the
+      // module - otherwise, for the case we are compiling a module, it must be
+      // present.
+      // Conversely, IsForBMI output is speculative and only produced for TUs
+      // in which module interfaces are discovered, thus it is not an error to
+      // find that there is no module in this case.
+      assert((hasErrors || IsForBMI) &&
+             "emitting module but current module doesn't exist");
       return;
     }
-  }
+  } // else, non-modular PCH.
 
   // Errors that do not prevent the PCH from being written should not cause the
   // overall compilation to fail either.
   if (AllowASTWithErrors)
     PP.getDiagnostics().getClient()->clear();
 
-  // Emit the PCH file to the Buffer.
   assert(SemaPtr && "No Sema?");
+
+  // A module implementation implicitly pulls in its interface module.
+  // Since it has the same name as the implementation, it will be found
+  // by the lookup above.  Fortunately, Sema records the difference in
+  // the ModuleScopes; We do not need to output the BMI in that case.
+  if (IsForBMI && SemaPtr->isModuleImplementation())
+    return;
+
+  if (IsForBMI) {
+
+    assert(Module && !Module->IsFromModuleFile &&
+           "trying to re-write a module?");
+
+    // So now attach that name to the buffer we are about to create.
+    Buffer->PresumedFileName = OutputFile;
+  }
+
   Buffer->Signature =
       Writer.WriteAST(*SemaPtr, OutputFile, Module, isysroot,
                       // For serialization we are lenient if the errors were
                       // only warn-as-error kind.
                       PP.getDiagnostics().hasUncompilableErrorOccurred(),
                       ShouldCacheASTInMemory);
 
   Buffer->IsComplete = true;

clang/lib/Serialization/PCHContainerOperations.cpp

 //=== Serialization/PCHContainerOperations.cpp - PCH Containers -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file defines PCHContainerOperations and RawPCHContainerOperation.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Serialization/PCHContainerOperations.h"
 #include "clang/AST/ASTConsumer.h"
+#include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "llvm/Bitstream/BitstreamReader.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
 
 using namespace clang;
 
 PCHContainerWriter::~PCHContainerWriter() {}
 PCHContainerReader::~PCHContainerReader() {}
 
if (Buffer->IsComplete) {
       OS->flush();
     }
     // Free the space of the temporary buffer.
     llvm::SmallVector<char, 0> Empty;
     Buffer->Data = std::move(Empty);
   }
 };
 
+/// A PCHContainerGenerator that writes out the PCH to a flat file if the
+/// action is needed (and the filename is determined at the time the output
+/// is done).
+class RawPCHDeferredContainerGenerator : public ASTConsumer {
+  std::shared_ptr<PCHBuffer> Buffer;
+
+public:
+  RawPCHDeferredContainerGenerator(std::shared_ptr<PCHBuffer> Buffer)
+      : Buffer(std::move(Buffer)) {}
+
+  ~RawPCHDeferredContainerGenerator() override = default;
+
+  void HandleTranslationUnit(ASTContext &Ctx) override {
+    if (Buffer->IsComplete && !Buffer->PresumedFileName.empty()) {
+      std::error_code EC;
+      StringRef Parent = llvm::sys::path::parent_path(Buffer->PresumedFileName);
+      if (!Parent.empty())
+        EC = llvm::sys::fs::create_directory(Parent);
+      if (!EC) {
+        int FD;
+        EC = llvm::sys::fs::openFileForWrite(Buffer->PresumedFileName, FD);
+        if (!EC) {
+          std::unique_ptr<raw_pwrite_stream> OS;
+          OS.reset(new llvm::raw_fd_ostream(FD, /*shouldClose=*/true));
+          *OS << Buffer->Data;
+          OS->flush(); // Make sure it hits disk now.
+        } else
+          llvm::dbgs() << " Problem creating : " << Buffer->PresumedFileName
+                       << "\n";
+      } else
+        llvm::dbgs() << " Problem creating dir : " << Parent << "\n";
+    }
+
+    // Free the space of the temporary buffer.
+    llvm::SmallVector<char, 0> Empty;
+    Buffer->Data = std::move(Empty);
+  }
+};
+
 } // anonymous namespace
 
 std::unique_ptr<ASTConsumer> RawPCHContainerWriter::CreatePCHContainerGenerator(
     CompilerInstance &CI, const std::string &MainFileName,
     const std::string &OutputFileName, std::unique_ptr<llvm::raw_pwrite_stream> OS,
     std::shared_ptr<PCHBuffer> Buffer) const {
   return std::make_unique<RawPCHContainerGenerator>(std::move(OS), Buffer);
 }
 
 ArrayRef<llvm::StringRef> RawPCHContainerReader::getFormats() const {
   static StringRef Raw("raw");
   return ArrayRef(Raw);
 }
 
+std::unique_ptr<ASTConsumer>
+RawPCHContainerWriter::CreatePCHDeferredContainerGenerator(
+    CompilerInstance &CI, const std::string &MainFileName,
+    const std::string &OutputFileName,
+    std::unique_ptr<llvm::raw_pwrite_stream> OS,
+    std::shared_ptr<PCHBuffer> Buffer) const {
+  return std::make_unique<RawPCHDeferredContainerGenerator>(Buffer);
+}
+
 StringRef
 RawPCHContainerReader::ExtractPCH(llvm::MemoryBufferRef Buffer) const {
   return Buffer.getBuffer();
 }
 
 PCHContainerOperations::PCHContainerOperations() {
   registerWriter(std::make_unique<RawPCHContainerWriter>());
   registerReader(std::make_unique<RawPCHContainerReader>());
 }