Index: COFF/CMakeLists.txt =================================================================== --- COFF/CMakeLists.txt +++ COFF/CMakeLists.txt @@ -30,7 +30,9 @@ Option Support - LINK_LIBS ${PTHREAD_LIB} + LINK_LIBS + lldCore + ${PTHREAD_LIB} ) add_dependencies(lldCOFF COFFOptionsTableGen) Index: COFF/Driver.h =================================================================== --- COFF/Driver.h +++ COFF/Driver.h @@ -13,6 +13,7 @@ #include "Config.h" #include "SymbolTable.h" #include "lld/Core/LLVM.h" +#include "lld/Core/Reproduce.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/Object/COFF.h" @@ -69,6 +70,8 @@ // Used by the resolver to parse .drectve section contents. void parseDirectives(StringRef S); + std::unique_ptr Cpio; // for /reproduce + private: llvm::BumpPtrAllocator AllocAux; llvm::StringSaver Alloc; Index: COFF/Driver.cpp =================================================================== --- COFF/Driver.cpp +++ COFF/Driver.cpp @@ -69,6 +69,10 @@ } static std::unique_ptr createFile(MemoryBufferRef MB) { + if (Driver->Cpio) + Driver->Cpio->append(relativeToRoot(MB.getBufferIdentifier()), + MB.getBuffer()); + // File type is detected by contents, not by file extension. file_magic Magic = identify_magic(MB.getBuffer()); if (Magic == file_magic::archive) @@ -247,6 +251,37 @@ return Config->DLL ? 0x10000000 : 0x400000; } +static std::string createResponseFile(const llvm::opt::InputArgList &Args, + ArrayRef MBs, + ArrayRef SearchPaths) { + SmallString<0> Data; + raw_svector_ostream OS(Data); + + for (auto *Arg : Args) { + switch (Arg->getOption().getID()) { + case OPT_linkrepro: + case OPT_INPUT: + case OPT_defaultlib: + case OPT_libpath: + break; + default: + OS << stringize(Arg) << "\n"; + } + } + + for (StringRef Path : SearchPaths) { + std::string RelPath = relativeToRoot(Path); + OS << "/libpath:" << quote(RelPath) << "\n"; + } + + for (MemoryBufferRef MB : MBs) { + std::string InputPath = relativeToRoot(MB.getBufferIdentifier()); + OS << quote(InputPath) << "\n"; + } + + return Data.str(); +} + void LinkerDriver::link(llvm::ArrayRef ArgsArr) { // If the first command line argument is "/lib", link.exe acts like lib.exe. // We call our own implementation of lib.exe that understands bitcode files. @@ -273,6 +308,17 @@ return; } + if (auto *Arg = Args.getLastArg(OPT_linkrepro)) { + SmallString<64> Path = StringRef(Arg->getValue()); + llvm::sys::path::append(Path, "repro"); + ErrorOr F = CpioFile::create(Path); + if (F) + Cpio.reset(*F); + else + llvm::errs() << "/reproduce: failed to open " << Path + << ".cpio: " << F.getError().message() << '\n'; + } + if (Args.filtered_begin(OPT_INPUT) == Args.filtered_end()) fatal("no input files"); @@ -511,9 +557,16 @@ if (!Resources.empty()) { std::unique_ptr MB = convertResToCOFF(Resources); Symtab.addFile(createFile(MB->getMemBufferRef())); + + MBs.push_back(MB->getMemBufferRef()); OwningMBs.push_back(std::move(MB)); // take ownership } + if (Cpio) + Cpio->append("response.txt", + createResponseFile(Args, MBs, + ArrayRef(SearchPaths).slice(1))); + // Handle /largeaddressaware if (Config->is64() || Args.hasArg(OPT_largeaddressaware)) Config->LargeAddressAware = true; Index: COFF/Error.h =================================================================== --- COFF/Error.h +++ COFF/Error.h @@ -32,6 +32,12 @@ return std::move(*E); } +template T check(ErrorOr EO) { + if (!EO) + fatal(EO.getError().message()); + return std::move(*EO); +} + } // namespace coff } // namespace lld Index: COFF/InputFiles.cpp =================================================================== --- COFF/InputFiles.cpp +++ COFF/InputFiles.cpp @@ -9,6 +9,7 @@ #include "Chunks.h" #include "Config.h" +#include "Driver.h" #include "Error.h" #include "InputFiles.h" #include "Symbols.h" @@ -93,9 +94,16 @@ // Return an empty buffer if we have already returned the same buffer. if (Seen[C.getChildOffset()].test_and_set()) return MemoryBufferRef(); - return check(C.getMemoryBufferRef(), - "could not get the buffer for the member defining symbol " + - Sym->getName()); + + MemoryBufferRef MB = + check(C.getMemoryBufferRef(), + "could not get the buffer for the member defining symbol " + + Sym->getName()); + if (C.getParent()->isThin() && Driver->Cpio) + Driver->Cpio->append(relativeToRoot(check(C.getFullName())), + MB.getBuffer()); + + return MB; } void ObjectFile::parse() { Index: COFF/Options.td =================================================================== --- COFF/Options.td +++ COFF/Options.td @@ -27,6 +27,7 @@ def heap : P<"heap", "Size of the heap">; def implib : P<"implib", "Import library name">; def libpath : P<"libpath", "Additional library search path">; +def linkrepro : P<"linkrepro", "Dump linker invocation and input files for debugging">; def machine : P<"machine", "Specify target platform">; def merge : P<"merge", "Combine sections">; def mllvm : P<"mllvm", "Options to pass to LLVM">; Index: ELF/CMakeLists.txt =================================================================== --- ELF/CMakeLists.txt +++ ELF/CMakeLists.txt @@ -44,6 +44,7 @@ LINK_LIBS lldConfig + lldCore ${PTHREAD_LIB} ) Index: ELF/Driver.h =================================================================== --- ELF/Driver.h +++ ELF/Driver.h @@ -12,6 +12,7 @@ #include "SymbolTable.h" #include "lld/Core/LLVM.h" +#include "lld/Core/Reproduce.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" @@ -23,8 +24,6 @@ extern class LinkerDriver *Driver; -class CpioFile; - class LinkerDriver { public: void main(ArrayRef Args); @@ -69,37 +68,11 @@ #undef OPTION }; -// This is the class to create a .cpio file for --reproduce. -// -// If "--reproduce foo" is given, we create a file "foo.cpio" and -// copy all input files to the archive, along with a response file -// to re-run the same command with the same inputs. -// It is useful for reporting issues to LLD developers. -// -// Cpio as a file format is a deliberate choice. It's standardized in -// POSIX and very easy to create. cpio command is available virtually -// on all Unix systems. See -// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_07 -// for the format details. -class CpioFile { -public: - static CpioFile *create(StringRef OutputPath); - void append(StringRef Path, StringRef Data); - -private: - CpioFile(std::unique_ptr OS, StringRef Basename); - - std::unique_ptr OS; - llvm::StringSet<> Seen; - std::string Basename; -}; - void printHelp(const char *Argv0); std::string getVersionString(); std::vector parseHexstring(StringRef S); std::string createResponseFile(const llvm::opt::InputArgList &Args); -std::string relativeToRoot(StringRef Path); std::string findFromSearchPaths(StringRef Path); std::string searchLibrary(StringRef Path); Index: ELF/Driver.cpp =================================================================== --- ELF/Driver.cpp +++ ELF/Driver.cpp @@ -269,11 +269,14 @@ if (const char *Path = getReproduceOption(Args)) { // Note that --reproduce is a debug option so you can ignore it // if you are trying to understand the whole picture of the code. - Cpio.reset(CpioFile::create(Path)); - if (Cpio) { + ErrorOr F = CpioFile::create(Path); + if (F) { + Cpio.reset(*F); Cpio->append("response.txt", createResponseFile(Args)); Cpio->append("version.txt", getVersionString()); - } + } else + error(F.getError(), + Twine("--reproduce: failed to open ") + Path + ".cpio"); } readConfigs(Args); Index: ELF/DriverUtils.cpp =================================================================== --- ELF/DriverUtils.cpp +++ ELF/DriverUtils.cpp @@ -16,6 +16,7 @@ #include "Driver.h" #include "Error.h" #include "lld/Config/Version.h" +#include "lld/Core/Reproduce.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Option/Option.h" @@ -105,107 +106,6 @@ return "LLD " + Version + " " + Repo + "\n"; } -// Makes a given pathname an absolute path first, and then remove -// beginning /. For example, "../foo.o" is converted to "home/john/foo.o", -// assuming that the current directory is "/home/john/bar". -std::string elf::relativeToRoot(StringRef Path) { - SmallString<128> Abs = Path; - if (std::error_code EC = fs::make_absolute(Abs)) - fatal("make_absolute failed: " + EC.message()); - path::remove_dots(Abs, /*remove_dot_dot=*/true); - - // This is Windows specific. root_name() returns a drive letter - // (e.g. "c:") or a UNC name (//net). We want to keep it as part - // of the result. - SmallString<128> Res; - StringRef Root = path::root_name(Abs); - if (Root.endswith(":")) - Res = Root.drop_back(); - else if (Root.startswith("//")) - Res = Root.substr(2); - - path::append(Res, path::relative_path(Abs)); - return Res.str(); -} - -CpioFile::CpioFile(std::unique_ptr OS, StringRef S) - : OS(std::move(OS)), Basename(S) {} - -CpioFile *CpioFile::create(StringRef OutputPath) { - std::string Path = (OutputPath + ".cpio").str(); - std::error_code EC; - auto OS = llvm::make_unique(Path, EC, fs::F_None); - if (EC) { - error(EC, "--reproduce: failed to open " + Path); - return nullptr; - } - return new CpioFile(std::move(OS), path::filename(OutputPath)); -} - -static void writeMember(raw_fd_ostream &OS, StringRef Path, StringRef Data) { - // The c_dev/c_ino pair should be unique according to the spec, - // but no one seems to care. - OS << "070707"; // c_magic - OS << "000000"; // c_dev - OS << "000000"; // c_ino - OS << "100664"; // c_mode: C_ISREG | rw-rw-r-- - OS << "000000"; // c_uid - OS << "000000"; // c_gid - OS << "000001"; // c_nlink - OS << "000000"; // c_rdev - OS << "00000000000"; // c_mtime - OS << format("%06o", Path.size() + 1); // c_namesize - OS << format("%011o", Data.size()); // c_filesize - OS << Path << '\0'; // c_name - OS << Data; // c_filedata -} - -void CpioFile::append(StringRef Path, StringRef Data) { - if (!Seen.insert(Path).second) - return; - - // Construct an in-archive filename so that /home/foo/bar is stored - // as baz/home/foo/bar where baz is the basename of the output file. - // (i.e. in that case we are creating baz.cpio.) - SmallString<128> Fullpath; - path::append(Fullpath, Basename, Path); - - // Use unix path separators so the cpio can be extracted on both unix and - // windows. - std::replace(Fullpath.begin(), Fullpath.end(), '\\', '/'); - - writeMember(*OS, Fullpath, Data); - - // Print the trailer and seek back. - // This way we have a valid archive if we crash. - uint64_t Pos = OS->tell(); - writeMember(*OS, "TRAILER!!!", ""); - OS->seek(Pos); -} - -// Quote a given string if it contains a space character. -static std::string quote(StringRef S) { - if (S.find(' ') == StringRef::npos) - return S; - return ("\"" + S + "\"").str(); -} - -static std::string rewritePath(StringRef S) { - if (fs::exists(S)) - return relativeToRoot(S); - return S; -} - -static std::string stringize(opt::Arg *Arg) { - std::string K = Arg->getSpelling(); - if (Arg->getNumValues() == 0) - return K; - std::string V = quote(Arg->getValue()); - if (Arg->getOption().getRenderStyle() == opt::Option::RenderJoinedStyle) - return K + V; - return K + " " + V; -} - // Reconstructs command line arguments so that so that you can re-run // the same command with the same inputs. This is for --reproduce. std::string elf::createResponseFile(const opt::InputArgList &Args) { @@ -226,8 +126,8 @@ case OPT_alias_script_T: case OPT_script: case OPT_version_script: - OS << Arg->getSpelling() << " " - << quote(rewritePath(Arg->getValue())) << "\n"; + OS << Arg->getSpelling() << " " << quote(rewritePath(Arg->getValue())) + << "\n"; break; default: OS << stringize(Arg) << "\n"; Index: ELF/InputFiles.h =================================================================== --- ELF/InputFiles.h +++ ELF/InputFiles.h @@ -16,6 +16,7 @@ #include "Symbols.h" #include "lld/Core/LLVM.h" +#include "lld/Core/Reproduce.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/IR/Comdat.h" Index: include/lld/Core/Reproduce.h =================================================================== --- /dev/null +++ include/lld/Core/Reproduce.h @@ -0,0 +1,70 @@ +//===- Reproduce.h - Utilities for creating reproducers ---------*- C++ -*-===// +// +// The LLVM Linker +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLD_CORE_REPRODUCE_H +#define LLD_CORE_REPRODUCE_H + +#include "lld/Core/LLVM.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/Error.h" + +namespace llvm { + +class raw_fd_ostream; + +namespace opt { class Arg; } + +} + +namespace lld { + +// This class creates a .cpio file for --reproduce (ELF) or /reproduce (COFF). +// +// If "--reproduce foo" is given, we create a file "foo.cpio" and +// copy all input files to the archive, along with a response file +// to re-run the same command with the same inputs. +// It is useful for reporting issues to LLD developers. +// +// Cpio as a file format is a deliberate choice. It's standardized in +// POSIX and very easy to create. cpio command is available virtually +// on all Unix systems. See +// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_07 +// for the format details. +class CpioFile { +public: + static ErrorOr create(StringRef OutputPath); + void append(StringRef Path, StringRef Data); + +private: + CpioFile(std::unique_ptr OS, StringRef Basename); + + std::unique_ptr OS; + llvm::StringSet<> Seen; + std::string Basename; +}; + +// Makes a given pathname an absolute path first, and then remove +// beginning /. For example, "../foo.o" is converted to "home/john/foo.o", +// assuming that the current directory is "/home/john/bar". +std::string relativeToRoot(StringRef Path); + +// Quote a given string if it contains a space character. +std::string quote(StringRef S); + +// Rewrite the given path if a file exists with that pathname, otherwise +// returns the original path. +std::string rewritePath(StringRef S); + +// Returns the string form of the given argument. +std::string stringize(llvm::opt::Arg *Arg); + +} + +#endif Index: lib/Core/CMakeLists.txt =================================================================== --- lib/Core/CMakeLists.txt +++ lib/Core/CMakeLists.txt @@ -4,6 +4,7 @@ File.cpp LinkingContext.cpp Reader.cpp + Reproduce.cpp Resolver.cpp SymbolTable.cpp Writer.cpp Index: lib/Core/Reproduce.cpp =================================================================== --- /dev/null +++ lib/Core/Reproduce.cpp @@ -0,0 +1,121 @@ +//===- Reproduce.cpp - Utilities for creating reproducers -----------------===// +// +// The LLVM Linker +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "lld/Core/Reproduce.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Option/Arg.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/Path.h" + +using namespace lld; +using namespace llvm; +using namespace sys; + +CpioFile::CpioFile(std::unique_ptr OS, StringRef S) + : OS(std::move(OS)), Basename(S) {} + +ErrorOr CpioFile::create(StringRef OutputPath) { + std::string Path = (OutputPath + ".cpio").str(); + std::error_code EC; + auto OS = make_unique(Path, EC, sys::fs::F_None); + if (EC) + return EC; + return new CpioFile(std::move(OS), path::filename(OutputPath)); +} + +static void writeMember(raw_fd_ostream &OS, StringRef Path, StringRef Data) { + // The c_dev/c_ino pair should be unique according to the spec, + // but no one seems to care. + OS << "070707"; // c_magic + OS << "000000"; // c_dev + OS << "000000"; // c_ino + OS << "100664"; // c_mode: C_ISREG | rw-rw-r-- + OS << "000000"; // c_uid + OS << "000000"; // c_gid + OS << "000001"; // c_nlink + OS << "000000"; // c_rdev + OS << "00000000000"; // c_mtime + OS << format("%06o", Path.size() + 1); // c_namesize + OS << format("%011o", Data.size()); // c_filesize + OS << Path << '\0'; // c_name + OS << Data; // c_filedata +} + +void CpioFile::append(StringRef Path, StringRef Data) { + if (!Seen.insert(Path).second) + return; + + // Construct an in-archive filename so that /home/foo/bar is stored + // as baz/home/foo/bar where baz is the basename of the output file. + // (i.e. in that case we are creating baz.cpio.) + SmallString<128> Fullpath; + path::append(Fullpath, Basename, Path); + + // Use unix path separators so the cpio can be extracted on both unix and + // windows. + std::replace(Fullpath.begin(), Fullpath.end(), '\\', '/'); + + writeMember(*OS, Fullpath, Data); + + // Print the trailer and seek back. + // This way we have a valid archive if we crash. + uint64_t Pos = OS->tell(); + writeMember(*OS, "TRAILER!!!", ""); + OS->seek(Pos); +} + +// Makes a given pathname an absolute path first, and then remove +// beginning /. For example, "../foo.o" is converted to "home/john/foo.o", +// assuming that the current directory is "/home/john/bar". +std::string lld::relativeToRoot(StringRef Path) { + SmallString<128> Abs = Path; + if (sys::fs::make_absolute(Abs)) + return Path; + path::remove_dots(Abs, /*remove_dot_dot=*/true); + + // This is Windows specific. root_name() returns a drive letter + // (e.g. "c:") or a UNC name (//net). We want to keep it as part + // of the result. + SmallString<128> Res; + StringRef Root = path::root_name(Abs); + if (Root.endswith(":")) + Res = Root.drop_back(); + else if (Root.startswith("//")) + Res = Root.substr(2); + + path::append(Res, path::relative_path(Abs)); + + return Res.str(); +} + +// Quote a given string if it contains a space character. +std::string lld::quote(StringRef S) { + if (S.find(' ') == StringRef::npos) + return S; + return ("\"" + S + "\"").str(); +} + +std::string lld::rewritePath(StringRef S) { + if (fs::exists(S)) + return relativeToRoot(S); + return S; +} + +std::string lld::stringize(opt::Arg *Arg) { + std::string K = Arg->getSpelling(); + if (Arg->getNumValues() == 0) + return K; + std::string V = quote(Arg->getValue()); + if (Arg->getOption().getRenderStyle() == opt::Option::RenderJoinedStyle) + return K + V; + return K + " " + V; +} Index: test/COFF/linkrepro.test =================================================================== --- /dev/null +++ test/COFF/linkrepro.test @@ -0,0 +1,35 @@ +# RUN: rm -rf %t.dir +# RUN: mkdir -p %t.dir/build1 %t.dir/build2 %t.dir/build3 +# RUN: yaml2obj < %p/Inputs/hello32.yaml > %t.obj + +# RUN: cd %t.dir/build1 +# RUN: lld-link %t.obj %p/Inputs/std32.lib /subsystem:console \ +# RUN: /entry:main@0 /linkrepro:. /out:%t.exe +# RUN: cpio -id < repro.cpio +# RUN: diff %t.obj repro/%:t.obj +# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib +# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt + +# RUN: cd %t.dir/build2 +# RUN: lld-link %t.obj /libpath:%p/Inputs /defaultlib:std32 /subsystem:console \ +# RUN: /entry:main@0 /linkrepro:. /out:%t.exe +# RUN: cpio -id < repro.cpio +# RUN: diff %t.obj repro/%:t.obj +# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib +# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt + +# RUN: cd %t.dir/build3 +# RUN: env LIB=%p/Inputs lld-link %t.obj /defaultlib:std32 /subsystem:console \ +# RUN: /entry:main@0 /linkrepro:. /out:%t.exe +# RUN: cpio -id < repro.cpio +# RUN: diff %t.obj repro/%:t.obj +# RUN: diff %p/Inputs/std32.lib repro/%:p/Inputs/std32.lib +# RUN: FileCheck %s --check-prefix=RSP < repro/response.txt + +# RSP: /subsystem:console +# RSP: /entry:main@0 +# RSP-NOT: /linkrepro: +# RSP: /out: +# RSP: linkrepro.test.tmp.obj +# RSP-NOT: defaultlib +# RSP: std32.lib