[CrashReproducer] Add a module map callback for added headers

[bruno]

Summary

The current ModuleDependencyCollector has a AST listener to collect
header files present in loaded modules, but this isn't enough to collect
all headers needed in the crash reproducer. One of the reasons is that
the AST writer doesn't write symbolic link header paths in the pcm modules,
this makes the listeners on the reader only able to collect the real files.

Since the module maps could contain submodules that use headers which
are symbolic links, not collecting those forbid the reproducer scripts
to regen the modules.

For instance:

usr/include/module.map:
  ...
  module pthread {
    header "pthread.h"
    export *

    module impl {
      header "pthread_impl.h"
      export *
    }
  }
  ...

usr/include/pthread/pthread_impl.h
usr/include/pthread_impl.h -> pthread/pthread_impl.h

The AST dump for the module above:

    
<SUBMODULE_HEADER abbrevid=6/> blob data = 'pthread_impl.h'
<SUBMODULE_TOPHEADER abbrevid=7/> blob data = '/<path_to_sdk>/usr/include/pthread/pthread_impl.h'

Note that we don't have "usr/include/pthread_impl.h" which is requested
by the module.map in case we want to reconstruct the module in the
reproducer. The reason the original symbolic link path isn't used is
because the headers are kept by name and requested through the
FileManager, which unique files and returns the real path only.

To fix that, add a callback to be invoked everytime a header is added
while parsing module maps and hook that up to the module dependecy
collector. This callback is only registered when generating the
reproducer.

[benlangmuir]

LGTM

[bruno]

Committed in r264971

include/clang/Frontend/Utils.h

 
 /// An interface for collecting the dependencies of a compilation. Users should
 /// use \c attachToPreprocessor and \c attachToASTReader to get all of the
 /// dependencies.
 /// FIXME: Migrate DependencyFileGen and DependencyGraphGen to use this
 /// interface.
 class DependencyCollector {
 public:
-  void attachToPreprocessor(Preprocessor &PP);
+  virtual void attachToPreprocessor(Preprocessor &PP);
   virtual void attachToASTReader(ASTReader &R);
   llvm::ArrayRef<std::string> getDependencies() const { return Dependencies; }
 
   /// Called when a new file is seen. Return true if \p Filename should be added
   /// to the list of dependencies.
   ///
   /// The default implementation ignores <built-in> and system files.
   virtual bool sawDependency(StringRef Filename, bool FromModule,

 public:
   StringRef getDest() { return DestDir; }
   bool insertSeen(StringRef Filename) { return Seen.insert(Filename).second; }
   void addFile(StringRef Filename);
   void addFileMapping(StringRef VPath, StringRef RPath) {
     VFSWriter.addFileMapping(VPath, RPath);
   }
 
+  void attachToPreprocessor(Preprocessor &PP) override;
   void attachToASTReader(ASTReader &R) override;
 
   void writeFileMap();
   bool hasErrors() { return HasErrors; }
   ModuleDependencyCollector(std::string DestDir) : DestDir(DestDir) {}
   ~ModuleDependencyCollector() { writeFileMap(); }
 };
 

include/clang/Lex/ModuleMap.h

public:
   /// \brief Called when a module map file has been read.
   ///
   /// \param FileStart A SourceLocation referring to the start of the file's
   /// contents.
   /// \param File The file itself.
   /// \param IsSystem Whether this is a module map from a system include path.
   virtual void moduleMapFileRead(SourceLocation FileStart,
                                  const FileEntry &File, bool IsSystem) {}
+
+  /// \brief Called when a header is added during module map parsing.
+  ///
+  /// \param File The header file itself.
+  virtual void moduleMapAddHeader(const FileEntry &File) {}
 };
   
 class ModuleMap {
   SourceManager &SourceMgr;
   DiagnosticsEngine &Diags;
   const LangOptions &LangOpts;
   const TargetInfo *Target;
   HeaderSearch &HeaderInfo;

lib/Frontend/ModuleDependencyCollector.cpp

 //===--- ModuleDependencyCollector.cpp - Collect module dependencies ------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // Collect the dependencies of a set of modules.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/Utils.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Serialization/ASTReader.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang;
 
 namespace {
-/// Private implementation for ModuleDependencyCollector
+/// Private implementations for ModuleDependencyCollector
 class ModuleDependencyListener : public ASTReaderListener {
   ModuleDependencyCollector &Collector;
 public:
   ModuleDependencyListener(ModuleDependencyCollector &Collector)
       : Collector(Collector) {}
   bool needsInputFileVisitation() override { return true; }
   bool needsSystemInputFileVisitation() override { return true; }
   bool visitInputFile(StringRef Filename, bool IsSystem, bool IsOverridden,
                       bool IsExplicitModule) override {
     Collector.addFile(Filename);
     return true;
   }
 };
+
+struct ModuleDependencyMMCallbacks : public ModuleMapCallbacks {
+  ModuleDependencyCollector &Collector;
+  ModuleDependencyMMCallbacks(ModuleDependencyCollector &Collector)
+      : Collector(Collector) {}
+
+  void moduleMapAddHeader(const FileEntry &File) override {
+    StringRef HeaderPath = File.getName();
+    if (llvm::sys::path::is_absolute(HeaderPath))
+      Collector.addFile(HeaderPath);
+  }
+};
+
 }
 
 void ModuleDependencyCollector::attachToASTReader(ASTReader &R) {
   R.addListener(llvm::make_unique<ModuleDependencyListener>(*this));
 }
 
+void ModuleDependencyCollector::attachToPreprocessor(Preprocessor &PP) {
+  PP.getHeaderSearchInfo().getModuleMap().addModuleMapCallbacks(
+      llvm::make_unique<ModuleDependencyMMCallbacks>(*this));
+}
+
 void ModuleDependencyCollector::writeFileMap() {
   if (Seen.empty())
     return;
 
   SmallString<256> Dest = getDest();
   llvm::sys::path::append(Dest, "vfs.yaml");
 
   // Default to use relative overlay directories in the VFS yaml file. This

lib/Lex/ModuleMap.cpp

void ModuleMap::addHeader(Module *Mod, Module::Header Header,
   bool isCompilingModuleHeader =
       LangOpts.CompilingModule && Mod->getTopLevelModule() == SourceModule;
   if (!Imported || isCompilingModuleHeader) {
     // When we import HeaderFileInfo, the external source is expected to
     // set the isModuleHeader flag itself.
     HeaderInfo.MarkFileModuleHeader(Header.Entry, Role,
                                     isCompilingModuleHeader);
   }
+
+  // Notify callbacks that we just added a new header.
+  for (const auto &Cb : Callbacks)
+    Cb->moduleMapAddHeader(*Header.Entry);
 }
 
 void ModuleMap::excludeHeader(Module *Mod, Module::Header Header) {
   // Add this as a known header so we won't implicitly add it to any
   // umbrella directory module.
   // FIXME: Should we only exclude it from umbrella modules within the
   // specified module?
   (void) Headers[Header.Entry];

test/Modules/Inputs/crash-recovery/usr/include/module.map

 module cstd [system] {
   // Only in system headers directory
   module stdio {
     header "stdio.h"
   }
+
+  module pthread {
+    header "pthread.h"
+    export *
+
+    module impl {
+      header "pthread_impl.h"
+      export *
+    }
+  }
 }

test/Modules/Inputs/crash-recovery/usr/include/pthread.h

+#include "pthread/pthread_impl.h"

test/Modules/Inputs/crash-recovery/usr/include/pthread/pthread_impl.h

+#define _PTHREAD_MUTEX_SIG_init   0x32AAABA7

test/Modules/Inputs/crash-recovery/usr/include/pthread_impl.h

+#define _PTHREAD_MUTEX_SIG_init   0x32AAABA7

test/Modules/crash-vfs-path-symlink-topheader.m

+// REQUIRES: crash-recovery, shell
+
+// FIXME: This XFAIL is cargo-culted from crash-report.c. Do we need it?
+// XFAIL: mingw32
+
+// Test clang can collect symbolic link headers used in modules.
+// crash reproducer if there's a symbolic link header file used in a module.
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/i %t/m %t %t/sysroot
+// RUN: cp -a %S/Inputs/crash-recovery/usr %t/i/
+// RUN: rm -f %t/i/usr/include/pthread_impl.h
+// RUN: ln -s pthread/pthread_impl.h %t/i/usr/include/pthread_impl.h
+
+// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \
+// RUN: %clang -fsyntax-only %s -I %/t/i -isysroot %/t/sysroot/ \
+// RUN:     -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s
+
+// RUN: FileCheck --check-prefix=CHECKSRC %s -input-file %t/crash-vfs-*.m
+// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh
+// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \
+// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml
+// RUN: find %t/crash-vfs-*.cache/vfs | \
+// RUN:   grep "usr/include/pthread_impl.h" | count 1
+
+#include "usr/include/stdio.h"
+
+// CHECK: Preprocessed source(s) and associated run script(s) are located at:
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.m
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache
+
+// CHECKSRC: @import cstd.stdio;
+
+// CHECKSH: # Crash reproducer
+// CHECKSH-NEXT: # Driver args: "-fsyntax-only"
+// CHECKSH-NEXT: # Original command: {{.*$}}
+// CHECKSH-NEXT: "-cc1"
+// CHECKSH: "-isysroot" "{{[^"]*}}/sysroot/"
+// CHECKSH-NOT: "-fmodules-cache-path="
+// CHECKSH: "crash-vfs-{{[^ ]*}}.m"
+// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
+
+// CHECKYAML: 'type': 'directory',
+// CHECKYAML: 'name': "",
+// CHECKYAML-NEXT: 'contents': [
+// CHECKYAML-NEXT:   {
+// CHECKYAML-NEXT:     'type': 'file',
+// CHECKYAML-NEXT:     'name': "pthread_impl.h",
+// CHECKYAML-NEXT:     'external-contents': "/{{.*}}/i/usr/include/pthread_impl.h"
+// CHECKYAML-NEXT:   },