Index: include/lldb/Host/FileSystem.h =================================================================== --- include/lldb/Host/FileSystem.h +++ include/lldb/Host/FileSystem.h @@ -12,6 +12,7 @@ #include "lldb/Host/File.h" #include "lldb/Utility/DataBufferLLVM.h" +#include "lldb/Utility/FileCollector.h" #include "lldb/Utility/FileSpec.h" #include "lldb/Utility/Status.h" @@ -31,13 +32,16 @@ static const char *DEV_NULL; static const char *PATH_CONVERSION_ERROR; - FileSystem() : m_fs(llvm::vfs::getRealFileSystem()) {} - FileSystem(llvm::IntrusiveRefCntPtr fs) : m_fs(fs) {} + FileSystem() : m_fs(llvm::vfs::getRealFileSystem()), m_collector(nullptr) {} + FileSystem(llvm::IntrusiveRefCntPtr fs) + : m_fs(fs), m_collector(nullptr) {} + FileSystem(llvm::StringRef mapping); static FileSystem &Instance(); static void Initialize(); static void Initialize(llvm::IntrusiveRefCntPtr fs); + static void Initialize(llvm::StringRef mapping); static void Terminate(); Status Symlink(const FileSpec &src, const FileSpec &dst); @@ -151,9 +155,13 @@ std::error_code GetRealPath(const llvm::Twine &path, llvm::SmallVectorImpl &output) const; + void SetCollector(FileCollector *collector) { m_collector = collector; } + private: static llvm::Optional &InstanceImpl(); + llvm::IntrusiveRefCntPtr m_fs; + FileCollector *m_collector; }; } // namespace lldb_private Index: include/lldb/Utility/FileCollector.h =================================================================== --- /dev/null +++ include/lldb/Utility/FileCollector.h @@ -0,0 +1,59 @@ +//===-- FileCollector.h -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_UTILITY_FILE_COLLECTOR_H +#define LLDB_UTILITY_FILE_COLLECTOR_H + +#include "lldb/Utility/FileSpec.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/VirtualFileSystem.h" + +#include + +namespace lldb_private { + +/// Collects files into a directory and generates a mapping that can be used by +/// the VFS. +class FileCollector { +public: + FileCollector(const FileSpec &directory); + + void AddFile(const llvm::Twine &file); + void AddFile(const FileSpec &file) { return AddFile(file.GetPath()); } + + std::error_code WriteMapping(const FileSpec &mapping_file); + std::error_code CopyFiles(); + +private: + void AddFileImpl(llvm::StringRef src_path); + + bool MarkAsSeen(llvm::StringRef path) { return m_seen.insert(path).second; } + + bool GetRealPath(llvm::StringRef src_path, + llvm::SmallVectorImpl &result); + + void AddFileToMapping(llvm::StringRef virtual_path, + llvm::StringRef real_path) { + m_vfs_writer.addFileMapping(virtual_path, real_path); + } + + std::mutex m_mutex; + FileSpec m_root; + llvm::StringSet<> m_seen; + llvm::vfs::YAMLVFSWriter m_vfs_writer; + llvm::StringMap m_symlink_map; +}; + +} // namespace lldb_private + +#endif // LLDB_UTILITY_FILE_COLLECTOR_H Index: include/lldb/Utility/Reproducer.h =================================================================== --- include/lldb/Utility/Reproducer.h +++ include/lldb/Utility/Reproducer.h @@ -10,6 +10,7 @@ #ifndef LLDB_UTILITY_REPRODUCER_H #define LLDB_UTILITY_REPRODUCER_H +#include "lldb/Utility/FileCollector.h" #include "lldb/Utility/FileSpec.h" #include "llvm/ADT/StringMap.h" @@ -71,6 +72,29 @@ FileSpec m_root; }; +class FileProvider : public repro::Provider { +public: + static constexpr const char *NAME = "files"; + FileProvider(const FileSpec &directory) + : Provider(directory), + m_collector(directory.CopyByAppendingPathComponent("root")) { + InitializeFileInfo(NAME, {"files.yaml"}); + } + + FileCollector &GetFileCollector() { return m_collector; } + + void Keep() override { + auto mapping = GetRoot().CopyByAppendingPathComponent("files.yaml"); + // Don't write the mapping if we can't copy the files. + if (auto ec = m_collector.CopyFiles()) + return; + m_collector.WriteMapping(mapping); + } + +private: + FileCollector m_collector; +}; + /// The generator is responsible for the logic needed to generate a /// reproducer. For doing so it relies on providers, who serialize data that /// is necessary for reproducing a failure. @@ -141,7 +165,12 @@ public: Loader(const FileSpec &root); - llvm::Optional GetProviderInfo(llvm::StringRef name); + llvm::Optional GetInfo(llvm::StringRef name); + + template llvm::Optional GetInfo() { + return GetInfo(T::NAME); + } + llvm::Error LoadIndex(); const FileSpec &GetRoot() const { return m_root; } Index: source/Core/Debugger.cpp =================================================================== --- source/Core/Debugger.cpp +++ source/Core/Debugger.cpp @@ -422,6 +422,26 @@ auto &r = repro::Reproducer::Instance(); if (auto e = r.SetReplay(true, FileSpec(p))) return e; + + // FIXME: The logic below should live elsewhere. + repro::Loader *loader = r.GetLoader(); + if (!loader) + return llvm::make_error("no loader for replay", + llvm::inconvertibleErrorCode()); + + auto provider_info = loader->GetInfo(); + if (!provider_info) + return llvm::make_error("no file provider for replay", + llvm::inconvertibleErrorCode()); + + if (provider_info->files.empty()) + return llvm::make_error("no vfs mapping for replay", + llvm::inconvertibleErrorCode()); + + FileSpec vfs_mapping(loader->GetRoot()); + vfs_mapping.AppendPathComponent(provider_info->files.front()); + + FileSystem::Initialize(vfs_mapping.GetPath()); return llvm::Error::success(); } @@ -430,6 +450,19 @@ auto root = HostInfo::GetReproducerTempDir(); if (auto e = r.SetCapture(b, root)) return e; + + // FIXME: The logic below should live elsewhere. + if (!b) { + FileSystem::Instance().SetCollector(nullptr); + return llvm::Error::success(); + } + + if (repro::Generator *g = r.GetGenerator()) { + if (repro::FileProvider *fp = g->GetOrCreate()) { + FileSystem::Instance().SetCollector(&(fp->GetFileCollector())); + } + } + return llvm::Error::success(); } Index: source/Host/common/FileSystem.cpp =================================================================== --- source/Host/common/FileSystem.cpp +++ source/Host/common/FileSystem.cpp @@ -49,10 +49,13 @@ } void FileSystem::Initialize(IntrusiveRefCntPtr fs) { - lldbassert(!InstanceImpl() && "Already initialized."); InstanceImpl().emplace(fs); } +void FileSystem::Initialize(llvm::StringRef mapping) { + InstanceImpl().emplace(mapping); +} + void FileSystem::Terminate() { lldbassert(InstanceImpl() && "Already terminated."); InstanceImpl().reset(); @@ -63,6 +66,16 @@ return g_fs; } +FileSystem::FileSystem(llvm::StringRef mapping) + : m_fs(llvm::vfs::getRealFileSystem()), m_collector(nullptr) { + llvm::ErrorOr> buffer = + m_fs->getBufferForFile(mapping); + if (!buffer) + return; + + m_fs = llvm::vfs::getVFSFromYAML(std::move(buffer.get()), nullptr, ""); +} + sys::TimePoint<> FileSystem::GetModificationTime(const FileSpec &file_spec) const { return GetModificationTime(file_spec.GetPath()); @@ -231,6 +244,9 @@ std::shared_ptr FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size, uint64_t offset) { + if (m_collector) + m_collector->AddFile(path); + const bool is_volatile = !IsLocal(path); std::unique_ptr buffer; @@ -362,6 +378,9 @@ Status FileSystem::Open(File &File, const FileSpec &file_spec, uint32_t options, uint32_t permissions) { + if (m_collector) + m_collector->AddFile(file_spec); + if (File.IsValid()) File.Close(); Index: source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp =================================================================== --- source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -3429,7 +3429,7 @@ if (!loader) return Status("No loader provided."); - auto provider_info = loader->GetProviderInfo("gdb-remote"); + auto provider_info = loader->GetInfo("gdb-remote"); if (!provider_info) return Status("No provider for gdb-remote."); Index: source/Utility/CMakeLists.txt =================================================================== --- source/Utility/CMakeLists.txt +++ source/Utility/CMakeLists.txt @@ -53,6 +53,7 @@ DataEncoder.cpp DataExtractor.cpp Environment.cpp + FileCollector.cpp FileSpec.cpp IOObject.cpp JSON.cpp Index: source/Utility/FileCollector.cpp =================================================================== --- /dev/null +++ source/Utility/FileCollector.cpp @@ -0,0 +1,134 @@ +//===-- FileCollector.cpp ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "lldb/Utility/FileCollector.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +using namespace lldb_private; +using namespace llvm; + +static bool IsCaseSensitivePath(StringRef path) { + SmallString<256> tmp_dest = path, upper_dest, real_dest; + + // Remove component traversals, links, etc. + if (!sys::fs::real_path(path, tmp_dest)) + return true; // Current default value in vfs.yaml + path = tmp_dest; + + // Change path to all upper case and ask for its real path, if the latter + // exists and is equal to path, it's not case sensitive. Default to case + // sensitive in the absence of real_path, since this is the YAMLVFSWriter + // default. + for (auto &C : path) + upper_dest.push_back(toUpper(C)); + if (sys::fs::real_path(upper_dest, real_dest) && path.equals(real_dest)) + return false; + return true; +} + +FileCollector::FileCollector(const FileSpec &directory) : m_root(directory) { + sys::fs::create_directories(m_root.GetPath(), true); +} + +bool FileCollector::GetRealPath(StringRef src_path, + SmallVectorImpl &result) { + SmallString<256> real_path; + StringRef FileName = sys::path::filename(src_path); + std::string directory = sys::path::parent_path(src_path).str(); + auto dir_with_symlink = m_symlink_map.find(directory); + + // Use real_path to fix any symbolic link component present in a path. + // Computing the real path is expensive, cache the search through the + // parent path directory. + if (dir_with_symlink == m_symlink_map.end()) { + if (!sys::fs::real_path(directory, real_path)) + return false; + m_symlink_map[directory] = real_path.str(); + } else { + real_path = dir_with_symlink->second; + } + + sys::path::append(real_path, FileName); + result.swap(real_path); + return true; +} + +void FileCollector::AddFile(const Twine &file) { + std::lock_guard lock(m_mutex); + std::string file_str = file.str(); + if (MarkAsSeen(file_str)) + AddFileImpl(file_str); +} + +void FileCollector::AddFileImpl(StringRef src_path) { + std::string root = m_root.GetPath(); + + // We need an absolute src path to append to the root. + SmallString<256> absolute_src = src_path; + sys::fs::make_absolute(absolute_src); + + // Canonicalize src to a native path to avoid mixed separator styles. + sys::path::native(absolute_src); + + // Remove redundant leading "./" pieces and consecutive separators. + absolute_src = sys::path::remove_leading_dotslash(absolute_src); + + // Canonicalize the source path by removing "..", "." components. + SmallString<256> virtual_path = absolute_src; + sys::path::remove_dots(virtual_path, /*remove_dot_dot=*/true); + + // If a ".." component is present after a symlink component, remove_dots may + // lead to the wrong real destination path. Let the source be canonicalized + // like that but make sure we always use the real path for the destination. + SmallString<256> copy_from; + if (!GetRealPath(absolute_src, copy_from)) + copy_from = virtual_path; + + SmallString<256> dst_path = StringRef(root); + sys::path::append(dst_path, sys::path::relative_path(copy_from)); + + // Always map a canonical src path to its real path into the YAML, by doing + // this we map different virtual src paths to the same entry in the VFS + // overlay, which is a way to emulate symlink inside the VFS; this is also + // needed for correctness, not doing that can lead to module redefinition + // errors. + AddFileToMapping(virtual_path, dst_path); +} + +std::error_code FileCollector::CopyFiles() { + for (auto &entry : m_vfs_writer.getMappings()) { + if (std::error_code ec = + sys::fs::create_directories(sys::path::parent_path(entry.RPath), + /*IgnoreExisting=*/true)) + return ec; + if (std::error_code ec = sys::fs::copy_file(entry.VPath, entry.RPath)) + return ec; + } + return {}; +} + +std::error_code FileCollector::WriteMapping(const FileSpec &mapping_file) { + std::lock_guard lock(m_mutex); + + const std::string root = m_root.GetPath(); + m_vfs_writer.setCaseSensitivity(IsCaseSensitivePath(root)); + m_vfs_writer.setUseExternalNames(false); + + std::error_code ec; + raw_fd_ostream os(mapping_file.GetPath(), ec, sys::fs::F_Text); + if (ec) + return ec; + + m_vfs_writer.write(os); + + return {}; +} Index: source/Utility/Reproducer.cpp =================================================================== --- source/Utility/Reproducer.cpp +++ source/Utility/Reproducer.cpp @@ -187,7 +187,7 @@ return llvm::Error::success(); } -llvm::Optional Loader::GetProviderInfo(StringRef name) { +llvm::Optional Loader::GetInfo(StringRef name) { assert(m_loaded); auto it = m_provider_info.find(name);