[FileCollector] Add a VFS that records FS accesses using the FileCollector

[arphaman]

Summary

This patch adds a VFS that can be overlaid on top of another VFS to record file system accesses using the FileCollector. This can help to gather files that are needed for reproducers.

[JDevlieghere]

Interesting approach!

LGTM modulo my comment.

llvm/unittests/Support/FileCollectorTest.cpp

212

Maybe add one more test that ensures we don't record files that don't exist?

llvm/include/llvm/Support/FileCollector.h

public:
 
   /// Copy the files into the root directory.
   ///
   /// When StopOnError is true (the default) we abort as soon as one file
   /// cannot be copied. This is relatively common, for example when a file was
   /// removed after it was added to the mapping.
   std::error_code copyFiles(bool StopOnError = true);
 
+  /// Create a VFS that collects all the paths that might be looked at by the
+  /// file system accesses.
+  static IntrusiveRefCntPtr<vfs::FileSystem>
+  createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
+                     std::shared_ptr<FileCollector> Collector);
+
 private:
   void addFileImpl(StringRef SrcPath);
 
   bool markAsSeen(StringRef Path) { return Seen.insert(Path).second; }
 
   bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result);
 
   void addFileToMapping(StringRef VirtualPath, StringRef RealPath) {

llvm/lib/Support/FileCollector.cpp

std::error_code FileCollector::writeMapping(StringRef mapping_file) {
   raw_fd_ostream os(mapping_file, EC, sys::fs::F_Text);
   if (EC)
     return EC;
 
   VFSWriter.write(os);
 
   return {};
 }
+
+namespace {
+
+class FileCollectorFileSystem : public vfs::FileSystem {
+public:
+  explicit FileCollectorFileSystem(IntrusiveRefCntPtr<vfs::FileSystem> FS,
+                                   std::shared_ptr<FileCollector> Collector)
+      : FS(std::move(FS)), Collector(std::move(Collector)) {}
+
+  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override {
+    auto Result = FS->status(Path);
+    if (Result && Result->exists())
+      Collector->addFile(Path);
+    return Result;
+  }
+
+  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
+  openFileForRead(const Twine &Path) override {
+    auto Result = FS->openFileForRead(Path);
+    if (Result && *Result)
+      Collector->addFile(Path);
+    return Result;
+  }
+
+  llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
+                                          std::error_code &EC) override {
+    auto It = FS->dir_begin(Dir, EC);
+    if (EC)
+      return It;
+    // Collect everything that's listed in case the user needs it.
+    Collector->addFile(Dir);
+    for (; !EC && It != llvm::vfs::directory_iterator(); It.increment(EC)) {
+      if (It->type() == sys::fs::file_type::regular_file ||
+          It->type() == sys::fs::file_type::directory_file ||
+          It->type() == sys::fs::file_type::symlink_file) {
+        Collector->addFile(It->path());
+      }
+    }
+    if (EC)
+      return It;
+    // Return a new iterator.
+    return FS->dir_begin(Dir, EC);
+  }
+
+  std::error_code getRealPath(const Twine &Path,
+                              SmallVectorImpl<char> &Output) const override {
+    auto EC = FS->getRealPath(Path, Output);
+    if (!EC) {
+      Collector->addFile(Path);
+      if (Output.size() > 0)
+        Collector->addFile(Output);
+    }
+    return EC;
+  }
+
+  std::error_code isLocal(const Twine &Path, bool &Result) override {
+    return FS->isLocal(Path, Result);
+  }
+
+  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+    return FS->getCurrentWorkingDirectory();
+  }
+
+  std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
+    return FS->setCurrentWorkingDirectory(Path);
+  }
+
+private:
+  IntrusiveRefCntPtr<vfs::FileSystem> FS;
+  std::shared_ptr<FileCollector> Collector;
+};
+
+} // end anonymous namespace
+
+IntrusiveRefCntPtr<vfs::FileSystem>
+FileCollector::createCollectorVFS(IntrusiveRefCntPtr<vfs::FileSystem> BaseFS,
+                                  std::shared_ptr<FileCollector> Collector) {
+  return new FileCollectorFileSystem(std::move(BaseFS), std::move(Collector));
+}

llvm/unittests/Support/FileCollectorTest.cpp

TEST(FileCollectorTest, recordAndConstructDirectory) {
   llvm::SmallString<128> SubdirInRoot = root.Path;
   llvm::sys::path::append(SubdirInRoot,
                           llvm::sys::path::relative_path(subdir.Path));
   ec = sys::fs::is_directory(SubdirInRoot, IsDirectory);
   EXPECT_FALSE(ec);
   ASSERT_TRUE(IsDirectory);
 }
 
+TEST(FileCollectorTest, recordVFSAccesses) {
+  ScopedDir file_root("dir_root", true);
+  ScopedDir subdir(file_root + "/subdir");
+  ScopedDir subdir2(file_root + "/subdir2");
+  ScopedFile a(subdir2 + "/a");
+  ScopedFile b(file_root + "/b");
+  ScopedDir subdir3(file_root + "/subdir3");
+  ScopedFile subdir3a(subdir3 + "/aa");
+  ScopedDir subdir3b(subdir3 + "/subdirb");
+
+  // Create file collector and add files.
+  ScopedDir root("copy_files_root", true);
+  std::string root_fs = root.Path.str();
+  auto Collector = std::make_shared<TestingFileCollector>(root_fs, root_fs);
+  auto VFS =
+      FileCollector::createCollectorVFS(vfs::getRealFileSystem(), Collector);
+  VFS->status(a.Path);
+  EXPECT_TRUE(Collector->hasSeen(a.Path));
+
+  VFS->openFileForRead(b.Path);
+  EXPECT_TRUE(Collector->hasSeen(b.Path));
+
+  VFS->status(subdir.Path);
+  EXPECT_TRUE(Collector->hasSeen(subdir.Path));
+
+  std::error_code EC;
+  auto It = VFS->dir_begin(subdir3.Path, EC);
+  EXPECT_FALSE(EC);
+  EXPECT_TRUE(Collector->hasSeen(subdir3.Path));
+  EXPECT_TRUE(Collector->hasSeen(subdir3a.Path));
+  EXPECT_TRUE(Collector->hasSeen(subdir3b.Path));
+}
+
 #ifndef _WIN32
 TEST(FileCollectorTest, Symlinks) {
   // Root where the original files live.
   ScopedDir file_root("file_root", true);
 
   // Create some files in the file root.
   ScopedFile a(file_root + "/aaa");
   ScopedFile b(file_root + "/bbb");
TEST(FileCollectorTest, Symlinks) {
   {
     // Make sure that .. is removed from the source path.
     std::string vpath = (file_root + "/eee").str();
     std::string rpath = (reproducer_root.Path + file_root.Path + "/eee").str();
     printf("%s -> %s\n", vpath.c_str(), rpath.c_str());
     EXPECT_THAT(mapping, testing::Contains(vfs::YAMLVFSEntry(vpath, rpath)));
   }
 }
+
+TEST(FileCollectorTest, recordVFSSymlinkAccesses) {
+  ScopedDir file_root("dir_root", true);
+  ScopedFile a(file_root + "/a");
+  ScopedLink symlink(file_root + "/a", file_root + "/b");
+
+  // Create file collector and add files.
+  ScopedDir root("copy_files_root", true);
+  std::string root_fs = root.Path.str();
+  auto Collector = std::make_shared<TestingFileCollector>(root_fs, root_fs);
+  auto VFS =
+      FileCollector::createCollectorVFS(vfs::getRealFileSystem(), Collector);
+  SmallString<256> Output;
+  VFS->getRealPath(symlink.Path, Output);
+  EXPECT_TRUE(Collector->hasSeen(a.Path));
+  EXPECT_TRUE(Collector->hasSeen(symlink.Path));
+}
 #endif