[clang][modules][deps] WIP: In-memory module transfer

[jansvoboda11]

Summary

Let's not create PCM files in the dependency scanner.

[jansvoboda11]

Improve SourceLocation handling: invalid & inside predefines buffer.

[jansvoboda11]

Handle macros

[jansvoboda11]

Kinda rebase

[jansvoboda11]

Fixed two tests:

• modules-pch.c by always marking identifier info as not out-of-date when getting leaf module macros,
• modules-module-map-order.m by updating the sorting logic for gathering SortedFiles.

[jansvoboda11]

Using CompilerInstance as the module storage dramatically increases memory usage (peak ~30GB) of clang-scan-deps when scanning LLVM. (Memory mapping serialized files peaks at ~300MB.)

clang/include/clang/Frontend/CompilerInstance.h

 //===-- CompilerInstance.h - Clang Compiler Instance ------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
class CompilerInstance : public ModuleLoader {
   std::list<OutputFile> OutputFiles;
 
   /// Force an output buffer.
   std::unique_ptr<llvm::raw_pwrite_stream> OutputStream;
 
   CompilerInstance(const CompilerInstance &) = delete;
   void operator=(const CompilerInstance &) = delete;
 public:
+  struct EntryStruct {
+    Optional<FileEntryRef> FE;
+    bool IsSystemFile;
+    bool IsTopLevelModuleMap;
+  };
+  llvm::StringMap<std::deque<EntryStruct>> SortedFiles;
+
   explicit CompilerInstance(
       std::shared_ptr<PCHContainerOperations> PCHContainerOps =
           std::make_shared<PCHContainerOperations>(),
       InMemoryModuleCache *SharedModuleCache = nullptr);
   ~CompilerInstance() override;
 
   /// @name High-Level Operations
   /// {

clang/lib/Frontend/CompilerInstance.cpp

 //===--- CompilerInstance.cpp ---------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
if (LangOpts.OpenCL)
     return Language::OpenCL;
   if (LangOpts.CUDA)
     return Language::CUDA;
   if (LangOpts.ObjC)
     return LangOpts.CPlusPlus ? Language::ObjCXX : Language::ObjC;
   return LangOpts.CPlusPlus ? Language::CXX : Language::C;
 }
 
+class Translator {
+  CompilerInstance &A;
+  const CompilerInstance &B;
+
+  llvm::StringSet<> TranslatedModules;
+
+public:
+  Translator(CompilerInstance &A, const CompilerInstance &B) : A(A), B(B) {}
+
+  template <class T> Optional<T> translate(const Optional<T> &BO) {
+    if (!BO)
+      return None;
+    return translate(*BO);
+  }
+
+  template <class T> const T *translate(const T *BP) {
+    if (!BP)
+      return nullptr;
+    return &translate(*BP);
+  }
+
+  template <class T> T *translate(T *BP) {
+    if (!BP)
+      return nullptr;
+    return &translate(*BP);
+  }
+
+  template <class T, class U> llvm::PointerUnion<T, U> translate(llvm::PointerUnion<T, U> BPU) {
+    if (!BPU)
+      return nullptr;
+    if (BPU.template is<T>())
+      return translate(BPU.template get<T>());
+    return translate(BPU.template get<U>());
+  }
+
+  template <class T, unsigned N>
+  SmallVector<T, N> translate(const SmallVector<T, N> &BV) {
+    SmallVector<T, N> AV;
+    AV.reserve(BV.size());
+    for (const T &Entry : BV)
+      AV.push_back(translate(Entry));
+    return AV;
+  }
+
+  template <class T, unsigned N>
+  llvm::SmallSetVector<T, N> translate(const llvm::SmallSetVector<T, N>& BSSV) {
+    llvm::SmallSetVector<T, N> ASSV;
+    for (const auto &Entry : BSSV)
+      ASSV.insert(translate(Entry));
+    return ASSV;
+  }
+
+  const FileEntry &translate(const FileEntry &BFE) {
+    return **A.getFileManager().getFile(BFE.getName());
+  }
+
+  FileEntryRef translate(FileEntryRef BFE) {
+    return *A.getFileManager().getOptionalFileRef(BFE.getName());
+  }
+
+  const DirectoryEntry &translate(const DirectoryEntry &BDE) {
+    return **A.getFileManager().getDirectory(BDE.getName());
+  }
+
+  SourceLocation translate(SourceLocation BLoc) {
+    auto &ASM = A.getSourceManager();
+    auto &BSM = B.getSourceManager();
+
+    auto BFileID = BSM.getFileID(BLoc);
+    auto BFileCharacteristic = BSM.getFileCharacteristic(BLoc);
+    auto *AFileEntry = translate(BSM.getFileEntryForID(BFileID));
+    auto AFileID = ASM.getOrCreateFileID(AFileEntry, BFileCharacteristic);
+
+    auto AOffset = BSM.getFileOffset(BLoc);
+
+    return ASM.getComposedLoc(AFileID, AOffset);
+  }
+
+  Module::Header translate(const Module::Header &BHeader) {
+    return Module::Header{BHeader.NameAsWritten,
+                          BHeader.PathRelativeToRootModuleDirectory,
+                          translate(BHeader.Entry)};
+  }
+
+  std::vector<Module *> translate(Module::submodule_iterator Begin,
+                                  Module::submodule_iterator End) {
+    std::vector<Module *> ASubModules;
+    for (auto It = Begin; It != End; ++It)
+      ASubModules.push_back(translate(*It));
+    return ASubModules;
+  }
+
+  Module::UnresolvedHeaderDirective
+  translate(const Module::UnresolvedHeaderDirective &BD) {
+    return {BD.Kind, //
+            translate(BD.FileNameLoc),
+            BD.FileName,
+            BD.IsUmbrella,
+            BD.HasBuiltinHeader,
+            BD.Size,
+            BD.ModTime};
+  }
+
+  Module::ExportDecl translate(const Module::ExportDecl &BED) {
+    return {translate(BED.getPointer()), BED.getInt()};
+  }
+
+  ModuleId translate(const ModuleId &BId) {
+    ModuleId Res;
+    for (const auto &Element : BId)
+      Res.push_back({Element.first, translate(Element.second)});
+    return Res;
+  }
+
+  Module::UnresolvedExportDecl
+  translate(const Module::UnresolvedExportDecl &BUED) {
+    return {translate(BUED.ExportLoc), translate(BUED.Id), BUED.Wildcard};
+  }
+
+  Module::LinkLibrary translate(const Module::LinkLibrary &X) {
+    return X;
+  }
+
+  Module::UnresolvedConflict translate(const Module::UnresolvedConflict &X) {
+    return {translate(X.Id), X.Message};
+  }
+
+  Module::Conflict translate(const Module::Conflict &X) {
+    return {translate(X.Other), X.Message};
+  }
+
+  Module &translate(const Module &BMod) {
+    return translate(const_cast<Module &>(BMod));
+  }
+
+  Module &translate(Module &BMod) {
+    auto &AModMap = A.getPreprocessor().getHeaderSearchInfo().getModuleMap();
+    auto &BHS = B.getPreprocessor().getHeaderSearchInfo();
+
+    auto [AMod, New] = AModMap.findOrCreateModule(
+        BMod.Name, translate(BMod.Parent),
+        BMod.IsFramework, BMod.IsExplicit);
+
+    if (!TranslatedModules.insert(BMod.Name).second)
+      return *AMod;
+
+    // Even if instance A already knows about module, it might not know
+    // information that's figured during compile of the module (e.g. imports).
+    if (!New) {
+      //      llvm::errs() << "translating existing module " << BMod.Name << "\n";
+    }
+
+    AMod->Kind = BMod.Kind;
+    AMod->Directory = translate(BMod.Directory);
+    AMod->PresumedModuleMapFile = BMod.PresumedModuleMapFile;
+    AMod->DefinitionLoc = translate(AMod->DefinitionLoc);
+    AMod->Umbrella = translate(BMod.Umbrella);
+    AMod->UmbrellaAsWritten = BMod.UmbrellaAsWritten;
+    AMod->UmbrellaRelativeToRootModuleDirectory = BMod.UmbrellaRelativeToRootModuleDirectory;
+    AMod->ExportAsModule = BMod.ExportAsModule;
+
+    for (Module *BSubMod : BMod.submodules())
+      translate(BSubMod);
+
+    for (const FileEntry *BTopHeader : BMod.getTopHeaders(B.getFileManager()))
+      AMod->addTopHeader(translate(BTopHeader));
+
+    // TODO: Propagate VisibilityID to other data structures.
+
+    for (auto Kind : {Module::HK_Normal, Module::HK_Textual, Module::HK_Private,
+                      Module::HK_PrivateTextual, Module::HK_Excluded}) {
+      for (const auto &BH : BMod.Headers[Kind]) {
+        const auto &AH = translate(BH);
+        AModMap.addHeader(AMod, AH, ModuleMap::headerKindToRole(Kind),
+                          BHS.getFileInfo(BH.Entry).isModuleHeader);
+      }
+    }
+
+    AMod->UnresolvedHeaders = translate(BMod.UnresolvedHeaders);
+    AMod->MissingHeaders = translate(BMod.MissingHeaders);
+    AMod->Requirements = BMod.Requirements;
+    AMod->ShadowingModule = translate(BMod.ShadowingModule);
+    AMod->IsUnimportable = BMod.IsUnimportable;
+    AMod->HasIncompatibleModuleFile = BMod.HasIncompatibleModuleFile;
+    AMod->IsAvailable = BMod.IsAvailable;
+    AMod->IsFromModuleFile = BMod.IsFromModuleFile;
+    AMod->IsFramework = BMod.IsFramework;
+    AMod->IsExplicit = BMod.IsExplicit;
+    AMod->IsSystem = BMod.IsSystem;
+    AMod->IsExternC = BMod.IsExternC;
+    AMod->IsInferred = BMod.IsInferred;
+    AMod->InferSubmodules = BMod.InferSubmodules;
+    AMod->InferExplicitSubmodules = BMod.InferExplicitSubmodules;
+    AMod->InferExportWildcard = BMod.InferExportWildcard;
+    AMod->ConfigMacrosExhaustive = BMod.ConfigMacrosExhaustive;
+    AMod->NoUndeclaredIncludes = BMod.NoUndeclaredIncludes;
+    AMod->ModuleMapIsPrivate = BMod.ModuleMapIsPrivate;
+    AMod->NameVisibility = BMod.NameVisibility;
+    AMod->InferredSubmoduleLoc = translate(BMod.InferredSubmoduleLoc);
+    AMod->Imports = translate(BMod.Imports);
+    AMod->Exports = translate(BMod.Exports);
+    AMod->UnresolvedExports = translate(BMod.UnresolvedExports);
+    AMod->DirectUses = translate(BMod.DirectUses);
+    AMod->UnresolvedDirectUses = translate(BMod.UnresolvedDirectUses);
+    AMod->UndeclaredUses = translate(BMod.UndeclaredUses);
+    AMod->LinkLibraries = translate(BMod.LinkLibraries);
+    AMod->UseExportAsModuleLinkName = BMod.UseExportAsModuleLinkName;
+    AMod->ConfigMacros = BMod.ConfigMacros;
+    AMod->UnresolvedConflicts = BMod.UnresolvedConflicts;
+    AMod->Conflicts = BMod.Conflicts;
+
+    return *AMod;
+  }
+
+  Module *translateModule(StringRef BName) {
+    return translate(
+        B.getPreprocessor().getHeaderSearchInfo().lookupModule(BName));
+  }
+};
+
 /// Compile a module file for the given module, using the options
 /// provided by the importing compiler instance. Returns true if the module
 /// was built without errors.
 static bool
 compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
                   StringRef ModuleName, FrontendInputFile Input,
                   StringRef OriginalModuleMapFile, StringRef ModuleFileName,
                   llvm::function_ref<void(CompilerInstance &)> PreBuildStep =
compileModuleImpl(CompilerInstance &ImportingInstance, SourceLocation ImportLoc,
   PreBuildStep(Instance);
 
   // Execute the action to actually build the module in-place. Use a separate
   // thread so that we get a stack large enough.
   bool Crashed = !llvm::CrashRecoveryContext().RunSafelyOnThread(
       [&]() {
         GenerateModuleFromModuleMapAction Action;
         Instance.ExecuteAction(Action);
+
+        Translator T(ImportingInstance, Instance);
+        T.translateModule(ModuleName);
+        //        llvm::errs() << "translated " << ModuleName << "\n";
+
+        std::set<const FileEntry *> AffectingModuleMaps;
+        std::set<const FileEntry *> SkippedModuleMaps;
+
+        auto &SortedFiles = ImportingInstance.SortedFiles[ModuleName];
+        for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size(); I != N; ++I) {
+          // Get this source location entry.
+          const SrcMgr::SLocEntry *SLoc = &SourceMgr.getLocalSLocEntry(I);
+
+          // We only care about file entries that were not overridden.
+          if (!SLoc->isFile())
+            continue;
+          const SrcMgr::FileInfo &File = SLoc->getFile();
+          const SrcMgr::ContentCache *Cache = &File.getContentCache();
+          if (!Cache->OrigEntry)
+            continue;
+
+          if (isModuleMap(File.getFileCharacteristic()) &&
+              !isSystem(File.getFileCharacteristic()) &&
+              !AffectingModuleMaps.empty() &&
+              AffectingModuleMaps.find(Cache->OrigEntry) ==
+                  AffectingModuleMaps.end()) {
+            SkippedModuleMaps.insert(Cache->OrigEntry);
+            // Do not emit modulemaps that do not affect current module.
+            continue;
+          }
+
+          CompilerInstance::EntryStruct Entry{
+              T.translate(Cache->OrigEntry),
+              isSystem(File.getFileCharacteristic()),
+              isModuleMap(File.getFileCharacteristic()) &&
+                  File.getIncludeLoc().isInvalid()};
+
+          if (Entry.IsSystemFile)
+            SortedFiles.push_back(Entry);
+          else
+            SortedFiles.push_front(Entry);
+        }
       },
       DesiredStackSize);
 
   PostBuildStep(Instance);
 
   ImportingInstance.getDiagnostics().Report(ImportLoc,
                                             diag::remark_module_build_done)
     << ModuleName;
static bool readASTAfterCompileModule(CompilerInstance &ImportingInstance,
                                       Module *Module, StringRef ModuleFileName,
                                       bool *OutOfDate) {
   DiagnosticsEngine &Diags = ImportingInstance.getDiagnostics();
 
   unsigned ModuleLoadCapabilities = ASTReader::ARR_Missing;
   if (OutOfDate)
     ModuleLoadCapabilities |= ASTReader::ARR_OutOfDate;
 
+  llvm::errs() << "readASTAfterCompileModule " << Module << " " << Module->getFullModuleName() << " " << Module->Imports.size() << "\n";
+
   // Try to read the module file, now that we've compiled it.
   ASTReader::ASTReadResult ReadResult =
       ImportingInstance.getASTReader()->ReadAST(
           ModuleFileName, serialization::MK_ImplicitModule, ImportLoc,
           ModuleLoadCapabilities);
+
+  llvm::errs() << "readASTAfterCompileModule " << Module << " " << Module->getFullModuleName() << " " << Module->Imports.size() << "\n";
+
   if (ReadResult == ASTReader::Success)
     return true;
 
   // The caller wants to handle out-of-date failures.
   if (OutOfDate && ReadResult == ASTReader::OutOfDate) {
     *OutOfDate = true;
     return false;
   }

 ModuleLoadResult CompilerInstance::findOrCompileModuleAndReadAST(
     StringRef ModuleName, SourceLocation ImportLoc,
     SourceLocation ModuleNameLoc, bool IsInclusionDirective) {
   // Search for a module with the given name.
   HeaderSearch &HS = PP->getHeaderSearchInfo();
   Module *M =
       HS.lookupModule(ModuleName, ImportLoc, true, !IsInclusionDirective);
 
+  assert(M && "no module in findOrCompileModuleAndReadAST");
+
   // Select the source and filename for loading the named module.
   std::string ModuleFilename;
   ModuleSource Source =
       selectModuleSource(M, ModuleName, ModuleFilename, BuiltModules, HS);
   if (Source == MS_ModuleNotFound) {
     // We can't find a module, error out here.
     getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_found)
         << ModuleName << SourceRange(ImportLoc, ModuleNameLoc);
if (getPreprocessorOpts().FailedModules &&
       getPreprocessorOpts().FailedModules->hasAlreadyFailed(ModuleName)) {
     getDiagnostics().Report(ModuleNameLoc, diag::err_module_not_built)
         << ModuleName << SourceRange(ImportLoc, ModuleNameLoc);
     return nullptr;
   }
 
   // Try to compile and then read the AST.
   if (!compileModuleAndReadAST(*this, ImportLoc, ModuleNameLoc, M,
-                               ModuleFilename)) {
+                               ModuleFilename)) { //
     assert(getDiagnostics().hasErrorOccurred() &&
            "undiagnosed error in compileModuleAndReadAST");
     if (getPreprocessorOpts().FailedModules)
       getPreprocessorOpts().FailedModules->addFailed(ModuleName);
     return nullptr;
   }
 
   // Okay, we've rebuilt and now loaded the module.

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

 //===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   Optional<FileEntryRef> ModuleMap = ModMapInfo.getModuleMapFileForUniquing(M);
 
   if (ModuleMap) {
     SmallString<128> Path = ModuleMap->getNameAsRequested();
     ModMapInfo.canonicalizeModuleMapPath(Path);
     MD.ClangModuleMapFile = std::string(Path);
   }
 
+  if (MDC.ScanInstance.SortedFiles.find(M->getFullModuleName()) == MDC.ScanInstance.SortedFiles.end()) {
   serialization::ModuleFile *MF =
       MDC.ScanInstance.getASTReader()->getModuleManager().lookup(
           M->getASTFile());
   MDC.ScanInstance.getASTReader()->visitInputFiles(
       *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
         // __inferred_module.map is the result of the way in which an implicit
         // module build handles inferred modules. It adds an overlay VFS with
         // this file in the proper directory and relies on the rest of Clang to
ModuleID ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
   addAllAffectingClangModules(M, MD, SeenDeps);
 
   MDC.ScanInstance.getASTReader()->visitTopLevelModuleMaps(
       *MF, [&](const FileEntry *FE) {
         if (FE->getName().endswith("__inferred_module.map"))
           return;
         MD.ModuleMapFileDeps.emplace_back(FE->getName());
       });
+  } else {
+    for (const auto &E : MDC.ScanInstance.SortedFiles[M->getFullModuleName()]) {
+      if (E.FE->getName().endswith("__inferred_module.map")) {
+        MDC.addFileDep(MD, ModuleMap->getName());
+        continue;
+      }
+      MDC.addFileDep(MD, E.FE->getName());
+    }
+    if (M->NoUndeclaredIncludes) {
+      for (const auto &E : MDC.ScanInstance.SortedFiles[M->getFullModuleName()]) {
+        if (E.FE->getName().endswith("__inferred_module.map"))
+          continue;
+        // The top-level modulemap of this module will be the input file. We
+        // don't need to specify it as a module map.
+        if (E.FE == ModuleMap)
+          continue;
+        MD.ModuleMapFileDeps.push_back(E.FE->getName().str());
+      }
+    }
+  }
 
   CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs(
       MD, [&](CompilerInvocation &BuildInvocation) {
-        if (MDC.OptimizeArgs)
-          optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
-                                   *MDC.ScanInstance.getASTReader(), *MF);
+//        if (MDC.OptimizeArgs)
+//          optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
+//                                   *MDC.ScanInstance.getASTReader(), *MF);
       });
 
   MDC.associateWithContextHash(CI, MD);
 
   // Finish the compiler invocation. Requires dependencies and the context hash.
   MDC.addOutputPaths(CI, MD);
 
   MD.BuildArguments = CI.getCC1CommandLine();