Index: include/clang/Basic/VirtualFileSystem.h =================================================================== --- include/clang/Basic/VirtualFileSystem.h +++ include/clang/Basic/VirtualFileSystem.h @@ -310,6 +310,7 @@ IntrusiveRefCntPtr getVFSFromYAML(std::unique_ptr Buffer, llvm::SourceMgr::DiagHandlerTy DiagHandler, + StringRef YAMLFilePath, void *DiagContext = nullptr, IntrusiveRefCntPtr ExternalFS = getRealFileSystem()); @@ -323,6 +324,8 @@ class YAMLVFSWriter { std::vector Mappings; Optional IsCaseSensitive; + Optional IsOverlayRelative; + std::string OverlayDir; public: YAMLVFSWriter() {} @@ -330,6 +333,11 @@ void setCaseSensitivity(bool CaseSensitive) { IsCaseSensitive = CaseSensitive; } + void setOverlayDir(StringRef OverlayDirectory) { + IsOverlayRelative = true; + OverlayDir.assign(OverlayDirectory.str()); + } + void write(llvm::raw_ostream &OS); }; Index: lib/Basic/VirtualFileSystem.cpp =================================================================== --- lib/Basic/VirtualFileSystem.cpp +++ lib/Basic/VirtualFileSystem.cpp @@ -790,6 +790,7 @@ /// All configuration options are optional. /// 'case-sensitive': /// 'use-external-names': +/// 'overlay-relative': /// /// Virtual directories are represented as /// \verbatim @@ -832,6 +833,10 @@ std::vector> Roots; /// \brief The file system to use for external references. IntrusiveRefCntPtr ExternalFS; + /// If IsRelativeOverlay is set, this represents the directory + /// path that should be prefixed to each 'external-contents' entry + /// when reading from YAML files. + std::string ExternalContentsPrefixDir; /// @name Configuration /// @{ @@ -841,6 +846,10 @@ /// Currently, case-insensitive matching only works correctly with ASCII. bool CaseSensitive; + /// IsRelativeOverlay marks whether a IsExternalContentsPrefixDir path must + /// be prefixed in every 'external-contents' when reading from YAML files. + bool IsRelativeOverlay = false; + /// \brief Whether to use to use the value of 'external-contents' for the /// names of files. This global value is overridable on a per-file basis. bool UseExternalNames; @@ -868,8 +877,8 @@ /// returns a virtual file system representing its contents. static RedirectingFileSystem * create(std::unique_ptr Buffer, - SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext, - IntrusiveRefCntPtr ExternalFS); + SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath, + void *DiagContext, IntrusiveRefCntPtr ExternalFS); ErrorOr status(const Twine &Path) override; ErrorOr> openFileForRead(const Twine &Path) override; @@ -902,6 +911,15 @@ return directory_iterator(std::make_shared(Dir, *this, D->contents_begin(), D->contents_end(), EC)); } + + void setExternalContentsPrefixDir(StringRef PrefixDir) { + ExternalContentsPrefixDir = PrefixDir.str(); + } + + StringRef getExternalContentsPrefixDir() const { + return ExternalContentsPrefixDir; + } + }; /// \brief A helper class to hold the common YAML parsing state. @@ -981,7 +999,7 @@ return true; } - std::unique_ptr parseEntry(yaml::Node *N) { + std::unique_ptr parseEntry(yaml::Node *N, RedirectingFileSystem *FS) { yaml::MappingNode *M = dyn_cast(N); if (!M) { error(N, "expected mapping node for file or directory entry"); @@ -1057,7 +1075,7 @@ for (yaml::SequenceNode::iterator I = Contents->begin(), E = Contents->end(); I != E; ++I) { - if (std::unique_ptr E = parseEntry(&*I)) + if (std::unique_ptr E = parseEntry(&*I, FS)) EntryArrayContents.push_back(std::move(E)); else return nullptr; @@ -1071,12 +1089,22 @@ HasContents = true; if (!parseScalarString(I->getValue(), Value, Buffer)) return nullptr; - SmallString<256> Path(Value); + + SmallString<128> FullPath; + if (FS->IsRelativeOverlay) { + FullPath = FS->getExternalContentsPrefixDir(); + assert(!FullPath.empty() && + "External contents prefix directory must exist"); + llvm::sys::path::append(FullPath, Value); + } else { + FullPath = Value; + } + // Guarantee that old YAML files containing paths with ".." and "." are // properly canonicalized before read into the VFS. - Path = sys::path::remove_leading_dotslash(Path); - sys::path::remove_dots(Path, /*remove_dot_dot=*/true); - ExternalContentsPath = Path.str(); + FullPath = sys::path::remove_leading_dotslash(FullPath); + sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true); + ExternalContentsPath = FullPath.str(); } else if (Key == "use-external-name") { bool Val; if (!parseScalarBool(I->getValue(), Val)) @@ -1162,6 +1190,7 @@ KeyStatusPair("version", true), KeyStatusPair("case-sensitive", false), KeyStatusPair("use-external-names", false), + KeyStatusPair("overlay-relative", false), KeyStatusPair("roots", true), }; @@ -1187,7 +1216,7 @@ for (yaml::SequenceNode::iterator I = Roots->begin(), E = Roots->end(); I != E; ++I) { - if (std::unique_ptr E = parseEntry(&*I)) + if (std::unique_ptr E = parseEntry(&*I, FS)) FS->Roots.push_back(std::move(E)); else return false; @@ -1213,6 +1242,9 @@ } else if (Key == "case-sensitive") { if (!parseScalarBool(I->getValue(), FS->CaseSensitive)) return false; + } else if (Key == "overlay-relative") { + if (!parseScalarBool(I->getValue(), FS->IsRelativeOverlay)) + return false; } else if (Key == "use-external-names") { if (!parseScalarBool(I->getValue(), FS->UseExternalNames)) return false; @@ -1233,9 +1265,11 @@ Entry::~Entry() = default; -RedirectingFileSystem *RedirectingFileSystem::create( - std::unique_ptr Buffer, SourceMgr::DiagHandlerTy DiagHandler, - void *DiagContext, IntrusiveRefCntPtr ExternalFS) { +RedirectingFileSystem * +RedirectingFileSystem::create(std::unique_ptr Buffer, + SourceMgr::DiagHandlerTy DiagHandler, + StringRef YAMLFilePath, void *DiagContext, + IntrusiveRefCntPtr ExternalFS) { SourceMgr SM; yaml::Stream Stream(Buffer->getMemBufferRef(), SM); @@ -1252,6 +1286,23 @@ std::unique_ptr FS( new RedirectingFileSystem(ExternalFS)); + + if (!YAMLFilePath.empty()) { + // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed + // to each 'external-contents' path. + // + // Example: + // -ivfsoverlay dummy.cache/vfs/vfs.yaml + // yields: + // FS->ExternalContentsPrefixDir => //dummy.cache/vfs + // + SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath); + std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir); + assert(!EC && "Overlay dir final path must be absolute"); + (void)EC; + FS->setExternalContentsPrefixDir(OverlayAbsDir); + } + if (!P.parse(Root, FS.get())) return nullptr; @@ -1396,10 +1447,12 @@ IntrusiveRefCntPtr vfs::getVFSFromYAML(std::unique_ptr Buffer, - SourceMgr::DiagHandlerTy DiagHandler, void *DiagContext, + SourceMgr::DiagHandlerTy DiagHandler, + StringRef YAMLFilePath, + void *DiagContext, IntrusiveRefCntPtr ExternalFS) { return RedirectingFileSystem::create(std::move(Buffer), DiagHandler, - DiagContext, ExternalFS); + YAMLFilePath, DiagContext, ExternalFS); } UniqueID vfs::getNextVirtualUniqueID() { @@ -1431,7 +1484,8 @@ public: JSONWriter(llvm::raw_ostream &OS) : OS(OS) {} - void write(ArrayRef Entries, Optional IsCaseSensitive); + void write(ArrayRef Entries, Optional IsCaseSensitive, + Optional IsOverlayRelative, StringRef OverlayDir); }; } @@ -1484,7 +1538,9 @@ } void JSONWriter::write(ArrayRef Entries, - Optional IsCaseSensitive) { + Optional IsCaseSensitive, + Optional IsOverlayRelative, + StringRef OverlayDir) { using namespace llvm::sys; OS << "{\n" @@ -1492,12 +1548,27 @@ if (IsCaseSensitive.hasValue()) OS << " 'case-sensitive': '" << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n"; + bool UseOverlayRelative = false; + if (IsOverlayRelative.hasValue()) { + UseOverlayRelative = IsOverlayRelative.getValue(); + OS << " 'overlay-relative': '" + << (UseOverlayRelative ? "true" : "false") << "',\n"; + } OS << " 'roots': [\n"; if (!Entries.empty()) { const YAMLVFSEntry &Entry = Entries.front(); startDirectory(path::parent_path(Entry.VPath)); - writeEntry(path::filename(Entry.VPath), Entry.RPath); + + StringRef RPath = Entry.RPath; + if (UseOverlayRelative) { + unsigned OverlayDirLen = OverlayDir.size(); + assert(RPath.substr(0, OverlayDirLen) == OverlayDir && + "Overlay dir must be contained in RPath"); + RPath = RPath.slice(OverlayDirLen, RPath.size()); + } + + writeEntry(path::filename(Entry.VPath), RPath); for (const auto &Entry : Entries.slice(1)) { StringRef Dir = path::parent_path(Entry.VPath); @@ -1511,7 +1582,14 @@ OS << ",\n"; startDirectory(Dir); } - writeEntry(path::filename(Entry.VPath), Entry.RPath); + StringRef RPath = Entry.RPath; + if (UseOverlayRelative) { + unsigned OverlayDirLen = OverlayDir.size(); + assert(RPath.substr(0, OverlayDirLen) == OverlayDir && + "Overlay dir must be contained in RPath"); + RPath = RPath.slice(OverlayDirLen, RPath.size()); + } + writeEntry(path::filename(Entry.VPath), RPath); } while (!DirStack.empty()) { @@ -1531,7 +1609,8 @@ return LHS.VPath < RHS.VPath; }); - JSONWriter(OS).write(Mappings, IsCaseSensitive); + JSONWriter(OS).write(Mappings, IsCaseSensitive, IsOverlayRelative, + OverlayDir); } VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl( Index: lib/Frontend/CompilerInvocation.cpp =================================================================== --- lib/Frontend/CompilerInvocation.cpp +++ lib/Frontend/CompilerInvocation.cpp @@ -2314,8 +2314,8 @@ return IntrusiveRefCntPtr(); } - IntrusiveRefCntPtr FS = - vfs::getVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr); + IntrusiveRefCntPtr FS = vfs::getVFSFromYAML( + std::move(Buffer.get()), /*DiagHandler*/ nullptr, File); if (!FS.get()) { Diags.Report(diag::err_invalid_vfs_overlay) << File; return IntrusiveRefCntPtr(); Index: lib/Frontend/ModuleDependencyCollector.cpp =================================================================== --- lib/Frontend/ModuleDependencyCollector.cpp +++ lib/Frontend/ModuleDependencyCollector.cpp @@ -51,6 +51,10 @@ SmallString<256> Dest = getDest(); llvm::sys::path::append(Dest, "vfs.yaml"); + // Default to use relative overlay directories in the VFS yaml file. This + // allows crash reproducer scripts to work across machines. + VFSWriter.setOverlayDir(getDest()); + std::error_code EC; llvm::raw_fd_ostream OS(Dest, EC, llvm::sys::fs::F_Text); if (EC) { Index: test/Modules/crash-vfs-path-symlink-component.m =================================================================== --- test/Modules/crash-vfs-path-symlink-component.m +++ test/Modules/crash-vfs-path-symlink-component.m @@ -40,21 +40,21 @@ // CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml" // CHECKYAML: 'type': 'directory' -// CHECKYAML: 'name': "{{[^ ]*}}/i/usr/include", +// CHECKYAML: 'name': "/[[PATH:.*]]/i/usr/include", // CHECKYAML-NEXT: 'contents': [ // CHECKYAML-NEXT: { // CHECKYAML-NEXT: 'type': 'file', // CHECKYAML-NEXT: 'name': "module.map", -// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map" +// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/i/usr/include/module.map" // CHECKYAML-NEXT: }, // CHECKYAML: 'type': 'directory' -// CHECKYAML: 'name': "{{[^ ]*}}/i/usr", +// CHECKYAML: 'name': "/[[PATH]]/i/usr", // CHECKYAML-NEXT: 'contents': [ // CHECKYAML-NEXT: { // CHECKYAML-NEXT: 'type': 'file', // CHECKYAML-NEXT: 'name': "module.map", -// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/i/usr/include/module.map" +// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/i/usr/include/module.map" // CHECKYAML-NEXT: }, // Test that by using the previous generated YAML file clang is able to find the Index: test/Modules/crash-vfs-path-traversal.m =================================================================== --- test/Modules/crash-vfs-path-traversal.m +++ test/Modules/crash-vfs-path-traversal.m @@ -35,12 +35,12 @@ // CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml" // CHECKYAML: 'type': 'directory' -// CHECKYAML: 'name': "{{[^ ]*}}/Inputs/System/usr/include", +// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/System/usr/include", // CHECKYAML-NEXT: 'contents': [ // CHECKYAML-NEXT: { // CHECKYAML-NEXT: 'type': 'file', // CHECKYAML-NEXT: 'name': "module.map", -// CHECKYAML-NEXT: 'external-contents': "{{[^ ]*}}/Inputs/System/usr/include/module.map" +// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/Inputs/System/usr/include/module.map" // CHECKYAML-NEXT: }, // Replace the paths in the YAML files with relative ".." traversals @@ -49,9 +49,10 @@ // RUN: sed -e "s@usr/include@usr/include/../include@g" \ // RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml > %t/vfs.yaml +// RUN: cp %t/vfs.yaml %t/crash-vfs-*.cache/vfs/vfs.yaml // RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH // RUN: %clang -E %s -I %S/Inputs/System -isysroot %/t/i/ \ -// RUN: -ivfsoverlay %t/vfs.yaml -fmodules \ +// RUN: -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \ // RUN: -fmodules-cache-path=%t/m/ 2>&1 \ // RUN: | FileCheck %s --check-prefix=CHECKOVERLAY Index: test/Modules/crash-vfs-relative-overlay.m =================================================================== --- /dev/null +++ test/Modules/crash-vfs-relative-overlay.m @@ -0,0 +1,56 @@ +// REQUIRES: crash-recovery, shell + +// FIXME: This XFAIL is cargo-culted from crash-report.c. Do we need it? +// XFAIL: mingw32 + +// RUN: rm -rf %t +// RUN: mkdir -p %t/i %t/m %t + +// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \ +// RUN: %clang -fsyntax-only %s -I %S/Inputs/System -isysroot %/t/i/ \ +// RUN: -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s + +// RUN: FileCheck --check-prefix=CHECKSRC %s -input-file %t/crash-vfs-*.m +// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh +// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \ +// RUN: %t/crash-vfs-*.cache/vfs/vfs.yaml +// RUN: find %t/crash-vfs-*.cache/vfs | \ +// RUN: grep "Inputs/System/usr/include/stdio.h" | count 1 + +#include "usr/include/stdio.h" + +// CHECK: Preprocessed source(s) and associated run script(s) are located at: +// CHECK-NEXT: note: diagnostic msg: {{.*}}.m +// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache + +// CHECKSRC: @import cstd.stdio; + +// CHECKSH: # Crash reproducer +// CHECKSH-NEXT: # Driver args: "-fsyntax-only" +// CHECKSH-NEXT: # Original command: {{.*$}} +// CHECKSH-NEXT: "-cc1" +// CHECKSH: "-isysroot" "{{[^"]*}}/i/" +// CHECKSH-NOT: "-fmodules-cache-path=" +// CHECKSH: "crash-vfs-{{[^ ]*}}.m" +// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml" + +// CHECKYAML: 'type': 'directory' +// CHECKYAML: 'name': "/[[PATH:.*]]/Inputs/System/usr/include", +// CHECKYAML-NEXT: 'contents': [ +// CHECKYAML-NEXT: { +// CHECKYAML-NEXT: 'type': 'file', +// CHECKYAML-NEXT: 'name': "module.map", +// CHECKYAML-NOT: 'external-contents': "{{[^ ]*}}.cache +// CHECKYAML-NEXT: 'external-contents': "/[[PATH]]/Inputs/System/usr/include/module.map" +// CHECKYAML-NEXT: }, + +// Test that reading the YAML file will yield the correct path after +// the overlay dir is prefixed to access headers in .cache/vfs directory. + +// RUN: unset FORCE_CLANG_DIAGNOSTICS_CRASH +// RUN: %clang -E %s -I %S/Inputs/System -isysroot %/t/i/ \ +// RUN: -ivfsoverlay %t/crash-vfs-*.cache/vfs/vfs.yaml -fmodules \ +// RUN: -fmodules-cache-path=%t/m/ 2>&1 \ +// RUN: | FileCheck %s --check-prefix=CHECKOVERLAY + +// CHECKOVERLAY: @import cstd.stdio; /* clang -E: implicit import for "{{[^ ]*}}.cache/vfs/{{[^ ]*}}/usr/include/stdio.h"