COFF: make builds more reproducible

[compnerd]

Summary

Change the way we calculate the build id to use MD5 to give reproducible build
ids. Previously we would generate random bytes for the build id GUID.

[ruiu]

COFF/Writer.cpp

84–86

Please add a comment saying that we save DI to backfill DI->PDB70.Signature in Writer::writeBuildId.

89

It's a fresh file, so as long as you don't write anything, zeros are there, so this is a nop.

98

DI = nullptr;

805

Add a comment that why we have this (to fill CVSignature) and why we do this way (to get reproducible outputs.)

[compnerd]

COFF/Writer.cpp

89

Zapped.

98

Oh, right, forgot about that. Nice! Changed to the initializer and zapped the constructor.

[compnerd]

Address comments.

[ruiu]

LGTM with a nit.

COFF/Writer.cpp

149

BuildId = nullptr, likewise.

[compnerd]

SVN r281079 with the nit addressed.

COFF/Writer.cpp

   }
 
   void writeTo(uint8_t *B) const override {
-    auto *R = reinterpret_cast<codeview::DebugInfo *>(B + OutputSectionOff);
+    // Save off the DebugInfo entry to backfill the file signature (build id)
+    // in Writer::writeBuildId
+    DI = reinterpret_cast<codeview::DebugInfo *>(B + OutputSectionOff);
 
-    R->Signature.CVSignature = OMF::Signature::PDB70;
-    // TODO(compnerd) fill in a GUID by hashing the contents of the binary to
-    // get a reproducible build
-    if (getRandomBytes(R->PDB70.Signature, sizeof(R->PDB70.Signature)))
-      fatal("entropy source failure");
-    // TODO(compnerd) track the Age
-    R->PDB70.Age = 1;
+    DI->Signature.CVSignature = OMF::Signature::PDB70;
 
     // variable sized field (PDB Path)
-    auto *P = reinterpret_cast<char *>(B + OutputSectionOff + sizeof(*R));
+    auto *P = reinterpret_cast<char *>(B + OutputSectionOff + sizeof(*DI));
     if (!Config->PDBPath.empty())
       memcpy(P, Config->PDBPath.data(), Config->PDBPath.size());
     P[Config->PDBPath.size()] = '\0';
   }
+
+public:
+  mutable codeview::DebugInfo *DI = nullptr;
 };
 
 // The writer writes a SymbolTable result to a file.
 class Writer {
 public:
-  Writer(SymbolTable *T) : Symtab(T) {}
+  Writer(SymbolTable *T) : Symtab(T), BuildId(nullptr) {}
   void run();
 
 private:

   void setSectionPermissions();
   void writeSections();
   void sortExceptionTable();
+  void writeBuildId();
   void applyRelocations();
 
   llvm::Optional<coff_symbol16> createSymbol(Defined *D);

 
   std::unique_ptr<Chunk> DebugDirectory;
   std::vector<std::unique_ptr<Chunk>> DebugRecords;
+  CVDebugRecordChunk *BuildId;
 
   uint64_t FileSize;
   uint32_t PointerToSymbolTable = 0;

   fixSafeSEHSymbols();
   writeSections();
   sortExceptionTable();
+  writeBuildId();
   if (auto EC = Buffer->commit())
     fatal(EC, "failed to write the output file");
 }

     DebugDirectory = llvm::make_unique<DebugDirectoryChunk>(DebugRecords);
 
     // TODO(compnerd) create a coffgrp entry if DebugType::CV is not enabled
-    if (Config->DebugTypes & static_cast<unsigned>(coff::DebugType::CV))
-      DebugRecords.push_back(llvm::make_unique<CVDebugRecordChunk>());
+    if (Config->DebugTypes & static_cast<unsigned>(coff::DebugType::CV)) {
+      auto Chunk = llvm::make_unique<CVDebugRecordChunk>();
+
+      BuildId = Chunk.get();
+      DebugRecords.push_back(std::move(Chunk));
+    }
 
     RData->addChunk(DebugDirectory.get());
     for (const std::unique_ptr<Chunk> &C : DebugRecords)

   errs() << "warning: don't know how to handle .pdata.\n";
 }
 
+// Backfill the CVSignature in a PDB70 Debug Record.  This backfilling allows us
+// to get reproducible builds.
+void Writer::writeBuildId() {
+  // There is nothing to backfill if BuildId was not setup.
+  if (BuildId == nullptr)
+    return;
+
+  MD5 Hash;
+  MD5::MD5Result Res;
+
+  Hash.update(ArrayRef<uint8_t>{Buffer->getBufferStart(),
+                                Buffer->getBufferEnd()});
+  Hash.final(Res);
+
+  assert(BuildId->DI->Signature.CVSignature == OMF::Signature::PDB70 &&
+         "only PDB 7.0 is supported");
+  memcpy(BuildId->DI->PDB70.Signature, Res, 16);
+  // TODO(compnerd) track the Age
+  BuildId->DI->PDB70.Age = 1;
+}
+
 OutputSection *Writer::findSection(StringRef Name) {
   for (OutputSection *Sec : OutputSections)
     if (Sec->getName() == Name)

test/COFF/rsds.test

 # CHECK:     PointerToRawData:
 # CHECK:     PDBInfo {
 # CHECK:       PDBSignature: 0x53445352
-# CHECK:       PDBGUID:
+# CHECK:       PDBGUID: (A9 01 F6 CD 45 C8 41 9F CC BC 17 55 E3 64 DF A2)
 # CHECK:       PDBAge: 1
 # CHECK:       PDBFileName: {{$}}
 # CHECK:     }

 # CHECK-PDB:     PointerToRawData:
 # CHECK-PDB:     PDBInfo {
 # CHECK-PDB:       PDBSignature: 0x53445352
-# CHECK-PDB:       PDBGUID:
+# CHECK-PDB:       PDBGUID: (9F 9F CC 10 A5 F3 90 C4 E2 48 F9 58 82 16 B3 20)
 # CHECK-PDB:       PDBAge: 1
 # CHECK-PDB:       PDBFileName: {{.*}}.pdb
 # CHECK-PDB:     }