Add the ability to stamp a PDB / EXE with a given GUID / Age, and use it to test LLD reproducibility

[zturner]

Summary

The idea here is that we want to test

a) That we write a valid timestamp (i.e. hash) into the PE file
and
b) That this doesn't affect determinism,

but it's difficult because the generated PDB GUID is currently not deterministic, so this non-deterministic value gets written into the PE. There's ideas to fix this, but for now we would still like to be able to test that this is the *only* source of non determinism. To deal with this, this patch introduces a subcommand to llvm-pdbutil which will stamp either an EXE or PDB with a specified GUID and age, then recompute the hash of the executable and update the various timestamp fields.

Two tests are introduced, one which verifies that two identical links without /DEBUG generate the same binary, and another which verifies that two identical links with /DEBUG generate the same binary provided they are stamped with the same GUID and Age.

[rnk]

lgtm

Sorry for the delay

llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp

1174

Needs wrapping, here and elsewhere. I'm assuming clang-format will happen at some point.

lld/test/COFF/repro-debug-clean.test

+RUN: rm -rf %t
+RUN: mkdir %t
+RUN: yaml2obj %p/Inputs/pdb1.yaml > %t/a.obj
+RUN: yaml2obj %p/Inputs/pdb2.yaml > %t/b.obj
+
+RUN: lld-link /DEBUG %t/a.obj %t/b.obj /nodefaultlib /entry:main /OUT:%t/out.exe /PDB:%t/out.pdb
+RUN: mv %t/out.exe %t/out.1.exe
+RUN: mv %t/out.pdb %t/out.1.pdb
+
+RUN: lld-link /DEBUG %t/a.obj %t/b.obj /nodefaultlib /entry:main /OUT:%t/out.exe /PDB:%t/out.pdb
+RUN: mv %t/out.exe %t/out.2.exe
+RUN: mv %t/out.pdb %t/out.2.pdb
+
+; Files should be different since the debug directories will contain different
+; GUIDs.
+RUN: not diff %t/out.1.exe %t/out.2.exe
+
+RUN: llvm-pdbutil stamp -age=12 -guid={FF141A38-63A8-4972-A23B-090EC3C16BD2} %t/out.1.exe
+RUN: llvm-pdbutil stamp -age=12 -guid={FF141A38-63A8-4972-A23B-090EC3C16BD2} %t/out.2.exe
+
+; Files should be the same now that they've been stamped with the same GUID / Age
+; e.g. the only difference was the GUID and age.
+
+RUN: diff %t/out.1.exe %t/out.2.exe
+
+RUN: llvm-readobj -coff-debug-directory -file-headers %t/out.1.exe | FileCheck %s
+
+CHECK-NOT:  TimeDateStamp: 1970-01-01 00:00:00 (0x0)
+CHECK:      PDBGUID: (FF 14 1A 38 63 A8 49 72 A2 3B 09 0E C3 C1 6B D2)
+CHECK-NEXT: PDBAge: 12

lld/test/COFF/repro-nodebug.test

+RUN: rm -rf %t
+RUN: mkdir %t
+RUN: yaml2obj %p/Inputs/pdb1.yaml > %t/a.obj
+RUN: yaml2obj %p/Inputs/pdb2.yaml > %t/b.obj
+RUN: lld-link %t/a.obj %t/b.obj /nodefaultlib /entry:main /OUT:%t/out.nodebug.1.exe
+RUN: lld-link %t/a.obj %t/b.obj /nodefaultlib /entry:main /OUT:%t/out.nodebug.2.exe
+RUN: diff %t/out.nodebug.1.exe %t/out.nodebug.2.exe

llvm/include/llvm/ADT/StringExtras.h

 inline StringRef toStringRef(bool B) { return StringRef(B ? "true" : "false"); }
 
 /// Construct a string ref from an array ref of unsigned chars.
 inline StringRef toStringRef(ArrayRef<uint8_t> Input) {
   return StringRef(reinterpret_cast<const char *>(Input.begin()), Input.size());
 }
 
 /// Construct a string ref from an array ref of unsigned chars.
+inline StringRef toStringRef(MutableArrayRef<uint8_t> Input) {
+  return StringRef(reinterpret_cast<const char *>(Input.begin()), Input.size());
+}
+
+/// Construct a string ref from an array ref of unsigned chars.
+inline StringRef toStringRef(ArrayRef<char> Input) {
+  return StringRef(Input.begin(), Input.size());
+}
+
+/// Construct a string ref from an array ref of unsigned chars.
+inline StringRef toStringRef(MutableArrayRef<char> Input) {
+  return StringRef(Input.begin(), Input.size());
+}
+
+/// Construct a string ref from an array ref of unsigned chars.
+inline StringRef toStringRef(const std::vector<char> &Input) {
+  return StringRef(Input.data(), Input.size());
+}
+
+/// Construct a string ref from an array ref of unsigned chars.
+inline StringRef toStringRef(const std::vector<uint8_t> &Input) {
+  return StringRef(reinterpret_cast<const char *>(Input.data()), Input.size());
+}
+
+/// Construct a string ref from an array ref of unsigned chars.
 inline ArrayRef<uint8_t> arrayRefFromStringRef(StringRef Input) {
   return {Input.bytes_begin(), Input.bytes_end()};
 }
 
 /// Interpret the given character \p C as a hexadecimal digit and return its
 /// value.
 ///
 /// If \p C is not a valid hex digit, -1U is returned.

llvm/include/llvm/DebugInfo/PDB/PDB.h

 //===- PDB.h - base header file for creating a PDB reader -------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_DEBUGINFO_PDB_PDB_H
 #define LLVM_DEBUGINFO_PDB_PDB_H
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/PDB/PDBTypes.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include <memory>
 
 namespace llvm {
 namespace pdb {
 
 class IPDBSession;
 
 Error loadDataForPDB(PDB_ReaderType Type, StringRef Path,

llvm/include/llvm/Object/COFF.h

iterator_range<const debug_directory *> debug_directories() const {
     return make_range(debug_directory_begin(), debug_directory_end());
   }
 
   const dos_header *getDOSHeader() const {
     if (!PE32Header && !PE32PlusHeader)
       return nullptr;
     return reinterpret_cast<const dos_header *>(base());
   }
+  std::error_code getCOFFFileHeader(const coff_file_header *&Res) const;
   std::error_code getPE32Header(const pe32_header *&Res) const;
   std::error_code getPE32PlusHeader(const pe32plus_header *&Res) const;
   std::error_code getDataDirectory(uint32_t index,
                                    const data_directory *&Res) const;
   std::error_code getSection(int32_t index, const coff_section *&Res) const;
 
   template <typename coff_symbol_type>
   std::error_code getSymbol(uint32_t Index,

llvm/lib/Object/COFFObjectFile.cpp

 COFFObjectFile::export_directories() const {
   return make_range(export_directory_begin(), export_directory_end());
 }
 
 iterator_range<base_reloc_iterator> COFFObjectFile::base_relocs() const {
   return make_range(base_reloc_begin(), base_reloc_end());
 }
 
+std::error_code COFFObjectFile::getCOFFFileHeader(const coff_file_header *&Res) const {
+  Res = COFFHeader;
+  return std::error_code();
+}
+
 std::error_code COFFObjectFile::getPE32Header(const pe32_header *&Res) const {
   Res = PE32Header;
   return std::error_code();
 }
 
 std::error_code
 COFFObjectFile::getPE32PlusHeader(const pe32plus_header *&Res) const {
   Res = PE32PlusHeader;

llvm/tools/llvm-pdbutil/llvm-pdbutil.cpp

 #include "PrettyVariableDumper.h"
 #include "YAMLOutputStyle.h"
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Config/config.h"
 #include "llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugLinesSubsection.h"
 #include "llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h"
 #include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h"
 #include "llvm/DebugInfo/CodeView/TypeStreamMerger.h"
 #include "llvm/DebugInfo/MSF/MSFBuilder.h"
 #include "llvm/DebugInfo/PDB/GenericError.h"
 #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h"
 #include "llvm/DebugInfo/PDB/IPDBRawSymbol.h"
 #include "llvm/DebugInfo/PDB/IPDBSession.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h"
+#include "llvm/DebugInfo/PDB/Native/InfoStream.h"
 #include "llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/NativeSession.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFileBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h"
 #include "llvm/DebugInfo/PDB/Native/RawConstants.h"
 #include "llvm/DebugInfo/PDB/Native/RawError.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStream.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolCompiland.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolData.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolExe.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolFunc.h"
 #include "llvm/DebugInfo/PDB/PDBSymbolThunk.h"
+#include "llvm/Object/COFF.h"
 #include "llvm/Support/BinaryByteStream.h"
 #include "llvm/Support/COM.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/LineIterator.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
-
+#include "llvm/Support/xxhash.h"
 
 using namespace llvm;
 using namespace llvm::codeview;
 using namespace llvm::msf;
 using namespace llvm::pdb;
 
 namespace opts {
 

 
 cl::SubCommand
     AnalyzeSubcommand("analyze",
                       "Analyze various aspects of a PDB's structure");
 
 cl::SubCommand MergeSubcommand("merge",
                                "Merge multiple PDBs into a single PDB");
 
+cl::SubCommand StampSubcommand("stamp", "Stamp a PDB and/or PE file with a build id");
+
 cl::OptionCategory TypeCategory("Symbol Type Options");
 cl::OptionCategory FilterCategory("Filtering and Sorting Options");
 cl::OptionCategory OtherOptions("Other Options");
 
 cl::ValuesClass ChunkValues = cl::values(
     clEnumValN(ModuleSubsection::CrossScopeExports, "cme",
                "Cross module exports (DEBUG_S_CROSSSCOPEEXPORTS subsection)"),
     clEnumValN(ModuleSubsection::CrossScopeImports, "cmi",

 namespace merge {
 cl::list<std::string> InputFilenames(cl::Positional,
                                      cl::desc("<input PDB files>"),
                                      cl::OneOrMore, cl::sub(MergeSubcommand));
 cl::opt<std::string>
     PdbOutputFile("pdb", cl::desc("the name of the PDB file to write"),
                   cl::sub(MergeSubcommand));
 }
+
+namespace stamp {
+  cl::opt<std::string> Guid("guid", cl::desc("GUID for the PDB or executable"), cl::Optional, cl::sub(StampSubcommand));
+  cl::opt<uint32_t> Age("age", cl::desc("Age for the PDB or executable"), cl::Optional, cl::sub(StampSubcommand));
+  cl::opt<std::string> InputFilename(cl::Positional, cl::Required,
+    cl::desc("<input PE or PDB file>"),
+    cl::sub(StampSubcommand));
+}
 }
 
 static ExitOnError ExitOnErr;
 
 static void yamlToPdb(StringRef Path) {
   BumpPtrAllocator Allocator;
   ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorOrBuffer =
       MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1,
static void simplifyChunkList(llvm::cl::list<opts::ModuleSubsection> &Chunks) {
   // If this list contains "All" plus some other stuff, remove the other stuff
   // and just keep "All" in the list.
   if (!llvm::is_contained(Chunks, opts::ModuleSubsection::All))
     return;
   Chunks.reset();
   Chunks.push_back(opts::ModuleSubsection::All);
 }
 
+static void findPdbGuidAndAge(PDBFile &File, Optional<uint64_t> &Guid, Optional<uint64_t> &PDBAge, Optional<uint64_t> &DBIAge) {
+  const MSFLayout &L = File.getMsfLayout();
+  if (StreamPDB >= L.StreamMap.size())
+    return;
+
+  ArrayRef<support::ulittle32_t> StreamBlocks = L.StreamMap[StreamPDB];
+  if (StreamBlocks.empty())
+    return;
+
+  uint64_t IS0 = blockToOffset(StreamBlocks[0], L.SB->BlockSize);
+  PDBAge = IS0 + offsetof(InfoStreamHeader, Age);
+  Guid = IS0 + offsetof(InfoStreamHeader, Guid);
+
+  if (StreamDBI >= L.StreamMap.size())
+    return;
+  StreamBlocks = L.StreamMap[StreamDBI];
+  if (StreamBlocks.empty())
+    return;
+
+  uint64_t DBI0 = blockToOffset(StreamBlocks[0], L.SB->BlockSize);
+  DBIAge = DBI0 + offsetof(DbiStreamHeader, Age);
+}
+
+static Expected<codeview::GUID> guidFromText(StringRef GuidStr) {
+  // We recognize the following two forms of Guid specification.
+  //   {XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
+  //    XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+  // where X is a hex digit (in upper or lowercase).
+  if (GuidStr.size() != 36 && GuidStr.size() != 38)
+    return make_error<StringError>("Invalid Guid string!",
+      inconvertibleErrorCode());
+
+  if (GuidStr.size() == 38) {
+    // Strip { and } from the beginning and end.
+    if (GuidStr.front() != '{' || GuidStr.back() != '}')
+      return make_error<StringError>("Invalid Guid string!",
+        inconvertibleErrorCode());
+    GuidStr = GuidStr.drop_front().drop_back();
+  }
+
+  assert(GuidStr.size() == 36);
+  SmallVector<StringRef, 4> Groups;
+  GuidStr.split(Groups, '-');
+  if (Groups.size() != 5 || Groups[0].size() != 8 || Groups[1].size() != 4 ||
+    Groups[2].size() != 4 || Groups[3].size() != 4 || Groups[4].size() != 12)
+    return make_error<StringError>("Invalid Guid string!",
+      inconvertibleErrorCode());
+  GUID Result;
+  uint8_t *WritePtr = Result.Guid;
+  for (StringRef G : Groups) {
+    while (!G.empty()) {
+      if (!isHexDigit(G[0]) || !isHexDigit(G[1]))
+        return make_error<StringError>("Invalid Guid string!",
+          inconvertibleErrorCode());
+
+      *WritePtr = hexFromNibbles(G[0], G[1]);
+      G = G.drop_front(2);
+      ++WritePtr;
+    }
+  }
+  return Result;
+}
+
+static void stampPdb(std::unique_ptr<WriteThroughMemoryBuffer> Buffer) {
+  bool Age = opts::stamp::Age.getNumOccurrences() > 0;
+  bool Guid = opts::stamp::Guid.getNumOccurrences() > 0;
+
+  std::unique_ptr<IPDBSession> PdbSession;
+  NativeSession *PdbNativeSession = nullptr;
+
+  WriteThroughMemoryBuffer *PdbBufferPtr = Buffer.get();
+  if (NativeSession::createFromPdb(std::move(Buffer), PdbSession)) {
+    errs() << "Unable to parse PDB file.\n";
+    exit(1);
+  }
+  PdbNativeSession = static_cast<NativeSession *>(PdbSession.get());
+  Optional<uint64_t> AOff1;
+  Optional<uint64_t> AOff2;
+  Optional<uint64_t> GOff;
+  findPdbGuidAndAge(PdbNativeSession->getPDBFile(), GOff, AOff1, AOff2);
+
+  if (Age) {
+    for (auto Off : { AOff1, AOff2 }) {
+      if (!Off)
+        continue;
+
+      support::ulittle32_t *Ptr = reinterpret_cast<support::ulittle32_t*>(PdbBufferPtr->getBufferStart() + *Off);
+      *Ptr = opts::stamp::Age;
+    }
+  }
+
+  if (Guid) {
+    codeview::GUID GuidValue = ExitOnErr(guidFromText(opts::stamp::Guid));
+
+    if (GOff.hasValue()) {
+      codeview::GUID *Ptr = reinterpret_cast<codeview::GUID*>(PdbBufferPtr->getBufferStart() + *GOff);
+      ::memcpy(Ptr, &GuidValue, sizeof(codeview::GUID));
+    } else {
+      outs() << "The PDB file does not contain a GUID.\n";
+    }
+  }
+}
+
+static void stampExecutable(std::unique_ptr<WriteThroughMemoryBuffer> Buffer) {
+  bool Age = opts::stamp::Age.getNumOccurrences() > 0;
+  bool Guid = opts::stamp::Guid.getNumOccurrences() > 0;
+
+  WriteThroughMemoryBuffer *ExeBufferPtr = Buffer.get();
+
+  std::error_code EC;
+  object::COFFObjectFile CoffBinary(*Buffer, EC);
+  if (EC) {
+    errs() << "Could not parse executable file.\n";
+    exit(1);
+  }
+
+  SmallVector<object::debug_directory *, 1> CodeViewDebugDirs;
+
+  // We need to update two places in the PE file.  Both of these places store
+  // a "time stamp" (which we treat as a hash of the executable).  This hash
+  // *includes* the GUID and age, but *does not* include the value of the hash
+  // fields themselves.  So we need to zero out the time stamp fields that we
+  // encounter, so that after re-writing the GUID and age the new hash is
+  // computed correctly.
+
+  // First we update the CodeView debug directories.
+  for (const object::debug_directory &Dir : CoffBinary.debug_directories()) {
+    // We know these were created from a read-write MemoryBuffer, so it's safe to
+    // const_cast them back to writable memory.
+    object::debug_directory *MutableDir = const_cast<object::debug_directory*>(&Dir);
+    MutableDir->TimeDateStamp = 0;
+    CodeViewDebugDirs.push_back(MutableDir);
+
+    if (Dir.Type != COFF::IMAGE_DEBUG_TYPE_CODEVIEW)
+      continue;
+    const codeview::DebugInfo *PdbInfo = nullptr;
+    StringRef PdbFileName;
+    if (CoffBinary.getDebugPDBInfo(PdbInfo, PdbFileName)) {
+      errs() << "Unable to read CV debug directory from PE file.\n";
+      exit(1);
+    }
+
+    codeview::DebugInfo *MutableBuildId = const_cast<codeview::DebugInfo*>(PdbInfo);
+    if (Age)
+      MutableBuildId->PDB70.Age = opts::stamp::Age;
+    if (Guid) {
+      codeview::GUID GuidValue = ExitOnErr(guidFromText(opts::stamp::Guid));
+      ::memcpy(&MutableBuildId->PDB70.Signature, &GuidValue, sizeof(codeview::GUID));
+    }
+  }
+
+  if (CodeViewDebugDirs.empty()) {
+    errs() << "Executable file does not contain a CV debug directory.  Cannot stamp file, re-link with /DEBUG.\n";
+    exit(1);
+  }
+
+  // Second is the COFF file header.
+  const object::coff_file_header *CFH;
+  CoffBinary.getCOFFFileHeader(CFH);
+  object::coff_file_header* MutableCFH = const_cast<object::coff_file_header*>(CFH);
+  MutableCFH->TimeDateStamp = 0;
+
+  // After all fields have been zeroed out, hash the executable and then write
+  // the hash to the appropriate places (coff file header + CV debug
+  // directories).
+  
+  StringRef ExeData = toStringRef(ExeBufferPtr->getBuffer());
+  uint32_t Hash = static_cast<uint32_t>(xxHash64(ExeData));
+  MutableCFH->TimeDateStamp = Hash;
+  for (object::debug_directory *DD : CodeViewDebugDirs)
+    DD->TimeDateStamp = Hash;
+}
+
 int main(int argc_, const char *argv_[]) {
   // Print a stack trace if we signal out.
   sys::PrintStackTraceOnErrorSignal(argv_[0]);
   PrettyStackTraceProgram X(argc_, argv_);
 
   ExitOnErr.setBanner("llvm-pdbutil: ");
 
   SmallVector<const char *, 256> argv;
if (opts::PdbToYamlSubcommand) {
 
     diff(opts::diff::Left, opts::diff::Right);
   } else if (opts::MergeSubcommand) {
     if (opts::merge::InputFilenames.size() < 2) {
       errs() << "merge subcommand requires at least 2 input files.\n";
       exit(1);
     }
     mergePdbs();
+  } else if (opts::StampSubcommand) {
+    auto BufferOrErr = WriteThroughMemoryBuffer::getFile(opts::stamp::InputFilename);
+    if (!BufferOrErr) {
+      errs() << "Unable to open input file for read-write.\n";
+      exit(1);
+    }
+
+    std::unique_ptr<WriteThroughMemoryBuffer> Buffer = std::move(*BufferOrErr);
+    file_magic Magic = identify_magic(toStringRef(Buffer->getBuffer()));
+
+    if (Magic == file_magic::pecoff_executable)
+      stampExecutable(std::move(Buffer));
+    else if (Magic == file_magic::pdb)
+      stampPdb(std::move(Buffer));
+    else {
+      errs() << "File must be a PDB file or PE/COFF executable or DLL.\n";
+      exit(1);
+    }
   }
 
   outs().flush();
   return 0;
 }