Index: include/lldb/Host/FileSystem.h =================================================================== --- include/lldb/Host/FileSystem.h +++ include/lldb/Host/FileSystem.h @@ -12,6 +12,7 @@ #include "lldb/Host/File.h" #include "lldb/Utility/DataBufferLLVM.h" +#include "lldb/Utility/FileCollector.h" #include "lldb/Utility/FileSpec.h" #include "lldb/Utility/Status.h" @@ -32,7 +33,10 @@ static const char *PATH_CONVERSION_ERROR; FileSystem() : m_fs(llvm::vfs::getRealFileSystem()) {} + FileSystem(FileCollector *collector) + : m_fs(llvm::vfs::getRealFileSystem()), m_collector(collector) {} FileSystem(llvm::IntrusiveRefCntPtr fs) : m_fs(fs) {} + FileSystem(const FileSpec &mapping); FileSystem(const FileSystem &fs) = delete; FileSystem &operator=(const FileSystem &fs) = delete; @@ -40,6 +44,8 @@ static FileSystem &Instance(); static void Initialize(); + static void Initialize(FileCollector *collector); + static void Initialize(const FileSpec &mapping); static void Initialize(llvm::IntrusiveRefCntPtr fs); static void Terminate(); @@ -168,9 +174,14 @@ std::error_code GetRealPath(const llvm::Twine &path, llvm::SmallVectorImpl &output) const; + llvm::ErrorOr GetExternalPath(const llvm::Twine &path); + llvm::ErrorOr GetExternalPath(const FileSpec &file_spec); + private: static llvm::Optional &InstanceImpl(); llvm::IntrusiveRefCntPtr m_fs; + FileCollector *m_collector = nullptr; + bool m_mapped = false; }; } // namespace lldb_private Index: include/lldb/Utility/FileCollector.h =================================================================== --- /dev/null +++ include/lldb/Utility/FileCollector.h @@ -0,0 +1,59 @@ +//===-- FileCollector.h -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_UTILITY_FILE_COLLECTOR_H +#define LLDB_UTILITY_FILE_COLLECTOR_H + +#include "lldb/Utility/FileSpec.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/VirtualFileSystem.h" + +#include + +namespace lldb_private { + +/// Collects files into a directory and generates a mapping that can be used by +/// the VFS. +class FileCollector { +public: + FileCollector(const FileSpec &directory); + + void AddFile(const llvm::Twine &file); + void AddFile(const FileSpec &file) { return AddFile(file.GetPath()); } + + std::error_code WriteMapping(const FileSpec &mapping_file); + std::error_code CopyFiles(bool stop_on_error = true); + +private: + void AddFileImpl(llvm::StringRef src_path); + + bool MarkAsSeen(llvm::StringRef path) { return m_seen.insert(path).second; } + + bool GetRealPath(llvm::StringRef src_path, + llvm::SmallVectorImpl &result); + + void AddFileToMapping(llvm::StringRef virtual_path, + llvm::StringRef real_path) { + m_vfs_writer.addFileMapping(virtual_path, real_path); + } + + std::mutex m_mutex; + FileSpec m_root; + llvm::StringSet<> m_seen; + llvm::vfs::YAMLVFSWriter m_vfs_writer; + llvm::StringMap m_symlink_map; +}; + +} // namespace lldb_private + +#endif // LLDB_UTILITY_FILE_COLLECTOR_H Index: include/lldb/Utility/Reproducer.h =================================================================== --- include/lldb/Utility/Reproducer.h +++ include/lldb/Utility/Reproducer.h @@ -10,6 +10,7 @@ #ifndef LLDB_UTILITY_REPRODUCER_H #define LLDB_UTILITY_REPRODUCER_H +#include "lldb/Utility/FileCollector.h" #include "lldb/Utility/FileSpec.h" #include "llvm/ADT/DenseMap.h" @@ -88,6 +89,31 @@ using ProviderBase::ProviderBase; // Inherit constructor. }; +class FileProvider : public Provider { +public: + FileProvider(const FileSpec &directory) + : Provider(directory), + m_collector(directory.CopyByAppendingPathComponent("root")) { + m_info.name = "files"; + m_info.files.push_back("files.yaml"); + } + + FileCollector &GetFileCollector() { return m_collector; } + + void Keep() override { + auto mapping = GetRoot().CopyByAppendingPathComponent("files.yaml"); + // Temporary files that are removed during execution can cause copy errors. + if (auto ec = m_collector.CopyFiles(/*stop_on_error=*/false)) + return; + m_collector.WriteMapping(mapping); + } + + static char ID; + +private: + FileCollector m_collector; +}; + /// The generator is responsible for the logic needed to generate a /// reproducer. For doing so it relies on providers, who serialize data that /// is necessary for reproducing a failure. Index: lit/Reproducer/Inputs/FileCapture.in =================================================================== --- /dev/null +++ lit/Reproducer/Inputs/FileCapture.in @@ -0,0 +1,3 @@ +run +reproducer status +reproducer generate Index: lit/Reproducer/Inputs/FileReplay.in =================================================================== --- /dev/null +++ lit/Reproducer/Inputs/FileReplay.in @@ -0,0 +1,2 @@ +reproducer status +run Index: lit/Reproducer/TestFileRepro.test =================================================================== --- /dev/null +++ lit/Reproducer/TestFileRepro.test @@ -0,0 +1,20 @@ +# REQUIRES: system-darwin + +# This tests the replaying of GDB remote packets. +# +# We issue the same commands and ensure the output is identical to the original +# process. To ensure we're not actually running the original binary we check +# that the string "testing" is not printed. + +# RUN: %clang %S/Inputs/simple.c -g -o %t.out +# RUN: %lldb -x -b -s %S/Inputs/FileCapture.in --capture %T/reproducer/files -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix CAPTURE +# RUN: rm %t.out +# RUN: %lldb -x -b -s %S/Inputs/FileReplay.in --replay %T/reproducer/files -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix REPLAY + +# CAPTURE: testing +# REPLAY-NOT: testing + +# CHECK: Process {{.*}} exited + +# CAPTURE: Reproducer is in capture mode. +# CAPTURE: Reproducer written Index: lit/Reproducer/TestGDBRemoteRepro.test =================================================================== --- lit/Reproducer/TestGDBRemoteRepro.test +++ lit/Reproducer/TestGDBRemoteRepro.test @@ -7,8 +7,8 @@ # that the string "testing" is not printed. # RUN: %clang %S/Inputs/simple.c -g -o %t.out -# RUN: %lldb -x -b -s %S/Inputs/GDBRemoteCapture.in --capture %T/reproducer -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix CAPTURE -# RUN: %lldb -x -b -s %S/Inputs/GDBRemoteReplay.in --replay %T/reproducer -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix REPLAY +# RUN: %lldb -x -b -s %S/Inputs/GDBRemoteCapture.in --capture %T/reproducer/gdb -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix CAPTURE +# RUN: %lldb -x -b -s %S/Inputs/GDBRemoteReplay.in --replay %T/reproducer/gdb -- %t.out | FileCheck %s --check-prefix CHECK --check-prefix REPLAY # CHECK: Breakpoint 1 # CHECK: Process {{.*}} stopped Index: source/Host/common/FileSystem.cpp =================================================================== --- source/Host/common/FileSystem.cpp +++ source/Host/common/FileSystem.cpp @@ -12,6 +12,7 @@ #include "lldb/Utility/LLDBAssert.h" #include "lldb/Utility/TildeExpressionResolver.h" +#include "llvm/Support/Errc.h" #include "llvm/Support/Errno.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" @@ -48,6 +49,16 @@ InstanceImpl().emplace(); } +void FileSystem::Initialize(FileCollector *collector) { + lldbassert(!InstanceImpl() && "Already initialized."); + InstanceImpl().emplace(collector); +} + +void FileSystem::Initialize(const FileSpec &mapping) { + lldbassert(!InstanceImpl() && "Already initialized."); + InstanceImpl().emplace(mapping); +} + void FileSystem::Initialize(IntrusiveRefCntPtr fs) { lldbassert(!InstanceImpl() && "Already initialized."); InstanceImpl().emplace(fs); @@ -82,6 +93,15 @@ return m_fs->status(path); } +FileSystem::FileSystem(const FileSpec &mapping) + : m_fs(llvm::vfs::getRealFileSystem()), m_mapped(true) { + llvm::ErrorOr> buffer = + m_fs->getBufferForFile(mapping.GetPath()); + if (!buffer) + return; + m_fs = llvm::vfs::getVFSFromYAML(std::move(buffer.get()), nullptr, ""); +} + sys::TimePoint<> FileSystem::GetModificationTime(const FileSpec &file_spec) const { return GetModificationTime(file_spec.GetPath()); @@ -250,18 +270,25 @@ std::shared_ptr FileSystem::CreateDataBuffer(const llvm::Twine &path, uint64_t size, uint64_t offset) { + if (m_collector) + m_collector->AddFile(path); + const bool is_volatile = !IsLocal(path); + const ErrorOr external_path = GetExternalPath(path); + + if (!external_path) + return nullptr; std::unique_ptr buffer; if (size == 0) { auto buffer_or_error = - llvm::WritableMemoryBuffer::getFile(path, -1, is_volatile); + llvm::WritableMemoryBuffer::getFile(*external_path, -1, is_volatile); if (!buffer_or_error) return nullptr; buffer = std::move(*buffer_or_error); } else { auto buffer_or_error = llvm::WritableMemoryBuffer::getFileSlice( - path, size, offset, is_volatile); + *external_path, size, offset, is_volatile); if (!buffer_or_error) return nullptr; buffer = std::move(*buffer_or_error); @@ -381,16 +408,22 @@ Status FileSystem::Open(File &File, const FileSpec &file_spec, uint32_t options, uint32_t permissions) { + if (m_collector) + m_collector->AddFile(file_spec); + if (File.IsValid()) File.Close(); const int open_flags = GetOpenFlags(options); const mode_t open_mode = (open_flags & O_CREAT) ? GetOpenMode(permissions) : 0; - const std::string path = file_spec.GetPath(); + + auto path = GetExternalPath(file_spec); + if (!path) + return Status(path.getError()); int descriptor = llvm::sys::RetryAfterSignal( - -1, OpenWithFS, *this, path.c_str(), open_flags, open_mode); + -1, OpenWithFS, *this, path->c_str(), open_flags, open_mode); Status error; if (!File::DescriptorIsValid(descriptor)) { @@ -402,3 +435,28 @@ } return error; } + +ErrorOr FileSystem::GetExternalPath(const llvm::Twine &path) { + if (!m_mapped) + return path.str(); + + // If VFS mapped we know the underlying FS is a RedirectingFileSystem. + ErrorOr E = + static_cast(*m_fs).lookupPath(path); + if (!E) { + if (E.getError() == llvm::errc::no_such_file_or_directory) { + return path.str(); + } + return E.getError(); + } + + auto *F = dyn_cast(*E); + if (!F) + return make_error_code(llvm::errc::not_supported); + + return F->getExternalContentsPath().str(); +} + +ErrorOr FileSystem::GetExternalPath(const FileSpec &file_spec) { + return GetExternalPath(file_spec.GetPath()); +} Index: source/Host/macosx/objcxx/Host.mm =================================================================== --- source/Host/macosx/objcxx/Host.mm +++ source/Host/macosx/objcxx/Host.mm @@ -1302,12 +1302,15 @@ lldb::pid_t pid = LLDB_INVALID_PROCESS_ID; - if (ShouldLaunchUsingXPC(launch_info)) { - error = LaunchProcessXPC(exe_spec.GetPath().c_str(), launch_info, pid); - } else { - error = - LaunchProcessPosixSpawn(exe_spec.GetPath().c_str(), launch_info, pid); - } + // From now on we'll deal with the external (devirtualized) path. + auto exe_path = fs.GetExternalPath(exe_spec); + if (!exe_path) + return Status(exe_path.getError()); + + if (ShouldLaunchUsingXPC(launch_info)) + error = LaunchProcessXPC(exe_path->c_str(), launch_info, pid); + else + error = LaunchProcessPosixSpawn(exe_path->c_str(), launch_info, pid); if (pid != LLDB_INVALID_PROCESS_ID) { // If all went well, then set the process ID into the launch info Index: source/Initialization/SystemInitializerCommon.cpp =================================================================== --- source/Initialization/SystemInitializerCommon.cpp +++ source/Initialization/SystemInitializerCommon.cpp @@ -66,6 +66,7 @@ } #endif + // Initialize the reproducer. ReproducerMode mode = ReproducerMode::Off; if (options.reproducer_capture) mode = ReproducerMode::Capture; @@ -75,7 +76,24 @@ if (auto e = Reproducer::Initialize(mode, FileSpec(options.reproducer_path))) return e; - FileSystem::Initialize(); + // Initialize the file system. + auto &r = repro::Reproducer::Instance(); + if (repro::Loader *loader = r.GetLoader()) { + if (auto info = loader->GetProviderInfo("files")) { + FileSpec vfs_mapping(loader->GetRoot()); + vfs_mapping.AppendPathComponent(info->files.front()); + FileSystem::Initialize(vfs_mapping); + } else { + // No file provider, continue with the real filesystem. + FileSystem::Initialize(); + } + } else if (repro::Generator *g = r.GetGenerator()) { + repro::FileProvider &fp = g->GetOrCreate(); + FileSystem::Initialize(&(fp.GetFileCollector())); + } else { + FileSystem::Initialize(); + } + Log::Initialize(); HostInfo::Initialize(); static Timer::Category func_cat(LLVM_PRETTY_FUNCTION); Index: source/Utility/CMakeLists.txt =================================================================== --- source/Utility/CMakeLists.txt +++ source/Utility/CMakeLists.txt @@ -53,6 +53,7 @@ DataEncoder.cpp DataExtractor.cpp Environment.cpp + FileCollector.cpp FileSpec.cpp IOObject.cpp JSON.cpp Index: source/Utility/FileCollector.cpp =================================================================== --- /dev/null +++ source/Utility/FileCollector.cpp @@ -0,0 +1,148 @@ +//===-- FileCollector.cpp ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "lldb/Utility/FileCollector.h" + +#include "llvm/ADT/SmallString.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" + +using namespace lldb_private; +using namespace llvm; + +static bool IsCaseSensitivePath(StringRef path) { + SmallString<256> tmp_dest = path, upper_dest, real_dest; + + // Remove component traversals, links, etc. + if (!sys::fs::real_path(path, tmp_dest)) + return true; // Current default value in vfs.yaml + path = tmp_dest; + + // Change path to all upper case and ask for its real path, if the latter + // exists and is equal to path, it's not case sensitive. Default to case + // sensitive in the absence of real_path, since this is the YAMLVFSWriter + // default. + upper_dest = path.upper(); + if (sys::fs::real_path(upper_dest, real_dest) && path.equals(real_dest)) + return false; + return true; +} + +FileCollector::FileCollector(const FileSpec &directory) : m_root(directory) { + sys::fs::create_directories(m_root.GetPath(), true); +} + +bool FileCollector::GetRealPath(StringRef src_path, + SmallVectorImpl &result) { + SmallString<256> real_path; + StringRef FileName = sys::path::filename(src_path); + std::string directory = sys::path::parent_path(src_path).str(); + auto dir_with_symlink = m_symlink_map.find(directory); + + // Use real_path to fix any symbolic link component present in a path. + // Computing the real path is expensive, cache the search through the + // parent path directory. + if (dir_with_symlink == m_symlink_map.end()) { + if (!sys::fs::real_path(directory, real_path)) + return false; + m_symlink_map[directory] = real_path.str(); + } else { + real_path = dir_with_symlink->second; + } + + sys::path::append(real_path, FileName); + result.swap(real_path); + return true; +} + +void FileCollector::AddFile(const Twine &file) { + std::lock_guard lock(m_mutex); + std::string file_str = file.str(); + if (MarkAsSeen(file_str)) + AddFileImpl(file_str); +} + +void FileCollector::AddFileImpl(StringRef src_path) { + std::string root = m_root.GetPath(); + + // We need an absolute src path to append to the root. + SmallString<256> absolute_src = src_path; + sys::fs::make_absolute(absolute_src); + + // Canonicalize src to a native path to avoid mixed separator styles. + sys::path::native(absolute_src); + + // Remove redundant leading "./" pieces and consecutive separators. + absolute_src = sys::path::remove_leading_dotslash(absolute_src); + + // Canonicalize the source path by removing "..", "." components. + SmallString<256> virtual_path = absolute_src; + sys::path::remove_dots(virtual_path, /*remove_dot_dot=*/true); + + // If a ".." component is present after a symlink component, remove_dots may + // lead to the wrong real destination path. Let the source be canonicalized + // like that but make sure we always use the real path for the destination. + SmallString<256> copy_from; + if (!GetRealPath(absolute_src, copy_from)) + copy_from = virtual_path; + + SmallString<256> dst_path = StringRef(root); + sys::path::append(dst_path, sys::path::relative_path(copy_from)); + + // Always map a canonical src path to its real path into the YAML, by doing + // this we map different virtual src paths to the same entry in the VFS + // overlay, which is a way to emulate symlink inside the VFS; this is also + // needed for correctness, not doing that can lead to module redefinition + // errors. + AddFileToMapping(virtual_path, dst_path); +} + +std::error_code FileCollector::CopyFiles(bool stop_on_error) { + for (auto &entry : m_vfs_writer.getMappings()) { + // Create directory tree. + if (std::error_code ec = + sys::fs::create_directories(sys::path::parent_path(entry.RPath), + /*IgnoreExisting=*/true)) { + if (stop_on_error) + return ec; + } + + // Copy file over. + if (std::error_code ec = sys::fs::copy_file(entry.VPath, entry.RPath)) { + if (stop_on_error) + return ec; + } + + // Copy over permissions. + if (auto perms = sys::fs::getPermissions(entry.VPath)) { + if (std::error_code ec = sys::fs::setPermissions(entry.RPath, *perms)) { + if (stop_on_error) + return ec; + } + } + } + return {}; +} + +std::error_code FileCollector::WriteMapping(const FileSpec &mapping_file) { + std::lock_guard lock(m_mutex); + + const std::string root = m_root.GetPath(); + m_vfs_writer.setCaseSensitivity(IsCaseSensitivePath(root)); + m_vfs_writer.setUseExternalNames(false); + + std::error_code ec; + raw_fd_ostream os(mapping_file.GetPath(), ec, sys::fs::F_Text); + if (ec) + return ec; + + m_vfs_writer.write(os); + + return {}; +} Index: source/Utility/Reproducer.cpp =================================================================== --- source/Utility/Reproducer.cpp +++ source/Utility/Reproducer.cpp @@ -223,3 +223,4 @@ void ProviderBase::anchor() {} char ProviderBase::ID = 0; +char FileProvider::ID = 0;