[clang][modules][deps] Report modulemaps describing excluded headers

[jansvoboda11]

Summary

Module map files describing excluded headers do affect compilation. Track them in the compiler, serialize them into the PCM file and report them in the scanner.

Depends on D134222.

[jansvoboda11]

Fix failing test by undoing some API changes

[Bigcheese]

I think this is fine, but I need to think more about the ModuleDepCollector.cpp change.

clang/lib/Lex/ModuleMap.cpp

1275

Cast to void no longer needed.

[Bigcheese]

lgtm with formatting fixed.

[srj]

This change has broken x86-32 builds, at least on Linux.

/usr/local/google/home/srj/GitHub/llvm-project/16/llvm/include/llvm/ADT/PointerIntPair.h:136:25: error: static assertion failed: PointerIntPair with integer size too large for pointer
  136 |   static_assert(IntBits <= PtrTraits::NumLowBitsAvailable,
      |                 ~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
/usr/local/google/home/srj/GitHub/llvm-project/16/llvm/include/llvm/ADT/PointerIntPair.h:136:25: note: the comparison reduces to ‘(3 <= 2)’
/usr/local/google/home/srj/GitHub/llvm-project/16/llvm/include/llvm/ADT/PointerIntPair.h:151:42: warning: left shift count >= width of type [-Wshift-count-overflow]
  151 |     ShiftedIntMask = (uintptr_t)(IntMask << IntShift)
      |                                 ~~~~~~~~~^~~~~~~~~~~~
/usr/local/google/home/srj/GitHub/llvm-project/16/llvm/include/llvm/ADT/PointerIntPair.h:151:42: error: right operand of shift expression ‘(7 << 4294967295)’ is greater than or equal to the precision 32 of the left operand [-fpermissive]
/usr/local/google/home/srj/GitHub/llvm-project/16/llvm/include/llvm/ADT/PointerIntPair.h:138:8: error: enumerator value for ‘ShiftedIntMask’ is not an integer constant
  138 |   enum MaskAndShiftConstants : uintptr_t {

Please fix or revert immediately.

[jansvoboda11]

Fixed in D134653.

clang/include/clang/Lex/HeaderSearch.h

public:
   bool hasModuleMap(StringRef Filename, const DirectoryEntry *Root,
                     bool IsSystem);
 
   /// Retrieve the module that corresponds to the given file, if any.
   ///
   /// \param File The header that we wish to map to a module.
   /// \param AllowTextual Whether we want to find textual headers too.
   ModuleMap::KnownHeader findModuleForHeader(const FileEntry *File,
-                                             bool AllowTextual = false) const;
+                                             bool AllowTextual = false,
+                                             bool AllowExcluded = false) const;
 
   /// Retrieve all the modules corresponding to the given file.
   ///
   /// \ref findModuleForHeader should typically be used instead of this.
   ArrayRef<ModuleMap::KnownHeader>
   findAllModulesForHeader(const FileEntry *File) const;
 
   /// Read the contents of the given module map file.

clang/include/clang/Lex/ModuleMap.h

enum ModuleHeaderRole {
 
     /// This header is included but private.
     PrivateHeader = 0x1,
 
     /// This header is part of the module (for layering purposes) but
     /// should be textually included.
     TextualHeader = 0x2,
 
+    /// This header is explicitly excluded from the module.
+    ExcludedHeader = 0x4,
+
     // Caution: Adding an enumerator needs other changes.
     // Adjust the number of bits for KnownHeader::Storage.
-    // Adjust the bitfield HeaderFileInfo::HeaderRole size.
     // Adjust the HeaderFileInfoTrait::ReadData streaming.
     // Adjust the HeaderFileInfoTrait::EmitData streaming.
     // Adjust ModuleMap::addHeader.
   };
 
   /// Convert a header kind to a role. Requires Kind to not be HK_Excluded.
   static ModuleHeaderRole headerKindToRole(Module::HeaderKind Kind);
 
   /// Convert a header role to a kind.
   static Module::HeaderKind headerRoleToKind(ModuleHeaderRole Role);
 
   /// A header that is known to reside within a given module,
   /// whether it was included or excluded.
   class KnownHeader {
-    llvm::PointerIntPair<Module *, 2, ModuleHeaderRole> Storage;
+    llvm::PointerIntPair<Module *, 3, ModuleHeaderRole> Storage;
 
   public:
     KnownHeader() : Storage(nullptr, NormalHeader) {}
     KnownHeader(Module *M, ModuleHeaderRole Role) : Storage(M, Role) {}
 
     friend bool operator==(const KnownHeader &A, const KnownHeader &B) {
       return A.Storage == B.Storage;
     }
public:
   /// \param AllowTextual If \c true and \p File is a textual header, return
   /// its owning module. Otherwise, no KnownHeader will be returned if the
   /// file is only known as a textual header.
   ///
   /// \returns The module KnownHeader, which provides the module that owns the
   /// given header file.  The KnownHeader is default constructed to indicate
   /// that no module owns this header file.
   KnownHeader findModuleForHeader(const FileEntry *File,
-                                  bool AllowTextual = false);
+                                  bool AllowTextual = false,
+                                  bool AllowExcluded = false);
 
   /// Retrieve all the modules that contain the given header file. Note that
   /// this does not implicitly load module maps, except for builtin headers,
   /// and does not consult the external source. (Those checks are the
   /// responsibility of \ref HeaderSearch.)
   ///
   /// Typically, \ref findModuleForHeader should be used instead, as it picks
   /// the preferred module for the header.

clang/lib/Lex/HeaderSearch.cpp

do {
 
     // Keep track of all of the directories we checked, so we can mark them as
     // having module maps if we eventually do find a module map.
     FixUpDirectories.push_back(*Dir);
   } while (true);
 }
 
 ModuleMap::KnownHeader
-HeaderSearch::findModuleForHeader(const FileEntry *File,
-                                  bool AllowTextual) const {
+HeaderSearch::findModuleForHeader(const FileEntry *File, bool AllowTextual,
+                                  bool AllowExcluded) const {
   if (ExternalSource) {
     // Make sure the external source has handled header info about this file,
     // which includes whether the file is part of a module.
     (void)getExistingFileInfo(File);
   }
-  return ModMap.findModuleForHeader(File, AllowTextual);
+  return ModMap.findModuleForHeader(File, AllowTextual, AllowExcluded);
 }
 
 ArrayRef<ModuleMap::KnownHeader>
 HeaderSearch::findAllModulesForHeader(const FileEntry *File) const {
   if (ExternalSource) {
     // Make sure the external source has handled header info about this file,
     // which includes whether the file is part of a module.
     (void)getExistingFileInfo(File);

clang/lib/Lex/ModuleMap.cpp

if ((New.getRole() & ModuleMap::TextualHeader) !=
       (Old.getRole() & ModuleMap::TextualHeader))
     return !(New.getRole() & ModuleMap::TextualHeader);
 
   // Don't have a reason to choose between these. Just keep the first one.
   return false;
 }
 
 ModuleMap::KnownHeader ModuleMap::findModuleForHeader(const FileEntry *File,
-                                                      bool AllowTextual) {
+                                                      bool AllowTextual,
+                                                      bool AllowExcluded) {
   auto MakeResult = [&](ModuleMap::KnownHeader R) -> ModuleMap::KnownHeader {
-    if (!AllowTextual && R.getRole() & ModuleMap::TextualHeader)
+    if ((!AllowTextual && R.getRole() & ModuleMap::TextualHeader) ||
+        (!AllowExcluded && R.getRole() & ModuleMap::ExcludedHeader))
       return {};
     return R;
   };
 
   HeadersMap::iterator Known = findKnownHeader(File);
   if (Known != Headers.end()) {
     ModuleMap::KnownHeader Result;
     // Iterate over all modules that 'File' is part of to find the best fit.
ModuleMap::isHeaderUnavailableInModule(const FileEntry *Header,
   resolveHeaderDirectives(Header);
   HeadersMap::const_iterator Known = Headers.find(Header);
   if (Known != Headers.end()) {
     for (SmallVectorImpl<KnownHeader>::const_iterator
              I = Known->second.begin(),
              E = Known->second.end();
          I != E; ++I) {
 
+      if (I->getRole() == ModuleMap::ExcludedHeader)
+        continue;
+
       if (I->isAvailable() &&
           (!RequestingModule ||
            I->getModule()->isSubModuleOf(RequestingModule))) {
         // When no requesting module is available, the caller is looking if a
         // header is part a module by only looking into the module map. This is
         // done by warn_uncovered_module_header checks; don't consider textual
         // headers part of it in this mode, otherwise we get misleading warnings
         // that a umbrella header is not including a textual header.
void ModuleMap::addHeader(Module *Mod, Module::Header Header,
   }
 
   // Notify callbacks that we just added a new header.
   for (const auto &Cb : Callbacks)
     Cb->moduleMapAddHeader(Header.Entry->getName());
 }
 
 void ModuleMap::excludeHeader(Module *Mod, Module::Header Header) {
+  KnownHeader KH(Mod, ModuleHeaderRole::ExcludedHeader);
+
   // Add this as a known header so we won't implicitly add it to any
   // umbrella directory module.
   // FIXME: Should we only exclude it from umbrella modules within the
   // specified module?
-  (void) Headers[Header.Entry];
+  Headers[Header.Entry].push_back(KH);
 
   Mod->Headers[Module::HK_Excluded].push_back(std::move(Header));
 }
 
 const FileEntry *
 ModuleMap::getContainingModuleMapFile(const Module *Module) const {
   if (Module->DefinitionLoc.isInvalid())
     return nullptr;

clang/lib/Serialization/ASTReader.cpp

if (unsigned FrameworkOffset =
     StringRef FrameworkName(FrameworkStrings + FrameworkOffset - 1);
     HFI.Framework = HS->getUniqueFrameworkName(FrameworkName);
   }
 
   assert((End - d) % 4 == 0 &&
          "Wrong data length in HeaderFileInfo deserialization");
   while (d != End) {
     uint32_t LocalSMID = endian::readNext<uint32_t, little, unaligned>(d);
-    auto HeaderRole = static_cast<ModuleMap::ModuleHeaderRole>(LocalSMID & 3);
-    LocalSMID >>= 2;
+    auto HeaderRole = static_cast<ModuleMap::ModuleHeaderRole>(LocalSMID & 7);
+    LocalSMID >>= 3;
 
     // This header is part of a module. Associate it with the module to enable
     // implicit module import.
     SubmoduleID GlobalSMID = Reader.getGlobalSubmoduleID(M, LocalSMID);
     Module *Mod = Reader.getSubmodule(GlobalSMID);
     FileManager &FileMgr = Reader.getFileManager();
     ModuleMap &ModMap =
         Reader.getPreprocessor().getHeaderSearchInfo().getModuleMap();

clang/lib/Serialization/ASTWriter.cpp

void EmitData(raw_ostream &Out, key_type_ref key,
           FrameworkNameOffset[Data.HFI.Framework] = Offset;
         } else
           Offset = Pos->second;
       }
       LE.write<uint32_t>(Offset);
 
       auto EmitModule = [&](Module *M, ModuleMap::ModuleHeaderRole Role) {
         if (uint32_t ModID = Writer.getLocalOrImportedSubmoduleID(M)) {
-          uint32_t Value = (ModID << 2) | (unsigned)Role;
-          assert((Value >> 2) == ModID && "overflow in header module info");
+          uint32_t Value = (ModID << 3) | (unsigned)Role;
+          assert((Value >> 3) == ModID && "overflow in header module info");
           LE.write<uint32_t>(Value);
         }
       };
 
       // FIXME: If the header is excluded, we should write out some
       // record of that fact.
       for (auto ModInfo : Data.KnownHeaders)
         EmitModule(ModInfo.getModule(), ModInfo.getRole());

clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp

MDC.ScanInstance.getASTReader()->visitInputFiles(
         // to play VFS tricks, so replace it with the correct module map.
         if (IF.getFile()->getName().endswith("__inferred_module.map")) {
           MDC.addFileDep(MD, ModuleMap->getName());
           return;
         }
         MDC.addFileDep(MD, IF.getFile()->getName());
       });
 
-  // We usually don't need to list the module map files of our dependencies when
-  // building a module explicitly: their semantics will be deserialized from PCM
-  // files.
-  //
-  // However, some module maps loaded implicitly during the dependency scan can
-  // describe anti-dependencies. That happens when this module, let's call it
-  // M1, is marked as '[no_undeclared_includes]' and tries to access a header
-  // "M2/M2.h" from another module, M2, but doesn't have a 'use M2;'
-  // declaration. The explicit build needs the module map for M2 so that it
-  // knows that textually including "M2/M2.h" is not allowed.
-  // E.g., '__has_include("M2/M2.h")' should return false, but without M2's
-  // module map the explicit build would return true.
-  //
-  // An alternative approach would be to tell the explicit build what its
-  // textual dependencies are, instead of having it re-discover its
-  // anti-dependencies. For example, we could create and use an `-ivfs-overlay`
-  // with `fall-through: false` that explicitly listed the dependencies.
-  // However, that's more complicated to implement and harder to reason about.
-  if (M->NoUndeclaredIncludes) {
-    // We don't have a good way to determine which module map described the
-    // anti-dependency (let alone what's the corresponding top-level module
-    // map). We simply specify all the module maps in the order they were loaded
-    // during the implicit build during scan.
-    // TODO: Resolve this by serializing and only using Module::UndeclaredUses.
-    MDC.ScanInstance.getASTReader()->visitTopLevelModuleMaps(
-        *MF, [&](const FileEntry *FE) {
-          if (FE->getName().endswith("__inferred_module.map"))
-            return;
-          // The top-level modulemap of this module will be the input file. We
-          // don't need to specify it as a module map.
-          if (FE == ModuleMap)
-            return;
-          MD.ModuleMapFileDeps.push_back(FE->getName().str());
-        });
-  }
-
-  // Add direct prebuilt module dependencies now, so that we can use them when
-  // creating a CompilerInvocation and computing context hash for this
-  // ModuleDeps instance.
-  // TODO: Squash these.
-  llvm::DenseSet<const Module *> SeenModules;
-  addAllSubmodulePrebuiltDeps(M, MD, SeenModules);
-  llvm::DenseSet<const Module *> AddedModules;
-  addAllSubmoduleDeps(M, MD, AddedModules);
-  llvm::DenseSet<const Module *> ProcessedModules;
-  addAllAffectingModules(M, MD, ProcessedModules);
+  llvm::DenseSet<const Module *> SeenDeps;
+  addAllSubmodulePrebuiltDeps(M, MD, SeenDeps);
+  addAllSubmoduleDeps(M, MD, SeenDeps);
+  addAllAffectingModules(M, MD, SeenDeps);
+
+  llvm::DenseSet<const FileEntry *> SeenModuleMaps;
+  for (const Module *SM : SeenDeps)
+    if (const FileEntry *SMM = MDC.ScanInstance.getPreprocessor()
+                                   .getHeaderSearchInfo()
+                                   .getModuleMap()
+                                   .getModuleMapFileForUniquing(SM))
+      SeenModuleMaps.insert(SMM);
+
+        // don't need to specify it via "-fmodule-map-file=".
+        // The top-level modulemap of dependencies will be serialized in the PCM
+        // file (eager loading mode) or passed on the command-line through a
+        // different mechanism (lazy loading mode).
+        if (SeenModuleMaps.contains(FE))
+          return;
+        MD.ModuleMapFileDeps.emplace_back(FE->getName());
 
   CompilerInvocation CI = MDC.makeInvocationForModuleBuildWithoutOutputs(
       MD, [&](CompilerInvocation &BuildInvocation) {
         if (MDC.OptimizeArgs)
           optimizeHeaderSearchOpts(BuildInvocation.getHeaderSearchOpts(),
                                    *MDC.ScanInstance.getASTReader(), *MF);
       });
 

clang/test/ClangScanDeps/modules-excluded-header.m

+// This test checks that we're reporting module maps affecting the compilation by describing excluded headers.
+
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+
+//--- frameworks/X.framework/Modules/module.modulemap
+framework module X {
+  umbrella header "X.h"
+  exclude header "Excluded.h"
+}
+//--- frameworks/X.framework/Headers/X.h
+//--- frameworks/X.framework/Headers/Excluded.h
+
+//--- mod/module.modulemap
+module Mod { header "Mod.h" }
+//--- mod/Mod.h
+#include <X/Excluded.h>
+
+//--- tu.m
+@import Mod;
+
+//--- cdb.json.template
+[{
+  "file": "DIR/tu.m",
+  "directory": "DIR",
+  "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR/mod -F DIR/frameworks -Werror=non-modular-include-in-module -c DIR/tu.m -o DIR/tu.m"
+}]
+
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full > %t/result.json
+// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t
+
+// CHECK:      {
+// CHECK-NEXT:   "modules": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:       "clang-module-deps": [],
+// CHECK-NEXT:       "clang-modulemap-file": "[[PREFIX]]/mod/module.modulemap",
+// CHECK-NEXT:       "command-line": [
+// CHECK:              "-fmodule-map-file=[[PREFIX]]/frameworks/X.framework/Modules/module.modulemap"
+// CHECK-NOT:          "-fmodule-file={{.*}}"
+// CHECK:            ],
+// CHECK-NEXT:       "context-hash": "{{.*}}",
+// CHECK-NEXT:       "file-deps": [
+// CHECK:            ],
+// CHECK-NEXT:       "name": "Mod"
+// CHECK-NEXT:     }
+// CHECK-NEXT:   ]
+// CHECK:      }
+
+// RUN: %deps-to-rsp %t/result.json --module-name=Mod > %t/Mod.cc1.rsp
+// RUN: %deps-to-rsp %t/result.json --tu-index=0 > %t/tu.rsp
+
+// RUN: %clang @%t/Mod.cc1.rsp
+// RUN: %clang @%t/tu.rsp