[Support] Add a function to MD5 a file's contents.

[zturner]

Summary

In doing so, clean up the MD5 interface a little.  Most
existing users only care about the lower 8 bytes of an MD5,
but for some users that care about the upper and lower,
there wasn't a good interface.  Furthermore, consumers
of the MD5 checksum were required to handle endianness
details on their own, so it seems reasonable to abstract
this into a nicer interface that just gives you the right
value.

[zturner]

ping

[mehdi_amini]

How does it compare to the SHA1 API? Are these in sync? (If not any good reason they're not?)

[zturner]

In D31105#705792, @mehdi_amini wrote:

How does it compare to the SHA1 API? Are these in sync? (If not any good reason they're not?)

It's pretty similar. The main difference is that SHA1 only gives you a sequence of bytes, and there is no option to return the result any other way without manual effort on the part of the user. I don't know if there is any good reason for this. It seems to be fairly common practice for consumers of MD5 checksums to truncate the hash and only use the first 8 bytes or the last 8 bytes. This seems much less common with SHA1 (as in, I don't know of any clients that do this, and I don't know if this affects the security of the hash in any way either).

If we can decide on a reasonable interface that allows both to be consistent, I'm happy to try to implement it (but hopefully orthogonally to this patch), keeping in mind that I'm not a hash expert, so I would not be the person to make any functional changes, but API changes are OK.

[mehdi_amini]

LGTM. See one comment inline.

(After reading more: this is not really the MD5 interface but the "result" object only.)

llvm/unittests/Support/MD5Test.cpp

70 ↗

(On Diff #92206)

I feel we should have a test for with low() and high() here.

cfe/trunk/lib/CodeGen/CodeGenPGO.cpp

uint64_t PGOHash::finalize() {
   // Check for remaining work in Working.
   if (Working)
     MD5.update(Working);
 
   // Finalize the MD5 and return the hash.
   llvm::MD5::MD5Result Result;
   MD5.final(Result);
   using namespace llvm::support;
-  return endian::read<uint64_t, little, unaligned>(Result);
+  return Result.low();
 }
 
 void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) {
   const Decl *D = GD.getDecl();
   bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
   llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader();
   if (!InstrumentRegions && !PGOReader)
     return;

cfe/trunk/lib/Frontend/ASTUnit.cpp

return ComputedPreamble(Buffer, std::move(BufferOwner), Pre.first,
                           Pre.second);
 }
 
 ASTUnit::PreambleFileHash
 ASTUnit::PreambleFileHash::createForFile(off_t Size, time_t ModTime) {
   PreambleFileHash Result;
   Result.Size = Size;
   Result.ModTime = ModTime;
-  memset(Result.MD5, 0, sizeof(Result.MD5));
+  Result.MD5 = {};
   return Result;
 }
 
 ASTUnit::PreambleFileHash ASTUnit::PreambleFileHash::createForMemoryBuffer(
     const llvm::MemoryBuffer *Buffer) {
   PreambleFileHash Result;
   Result.Size = Buffer->getBufferSize();
   Result.ModTime = 0;
 
   llvm::MD5 MD5Ctx;
   MD5Ctx.update(Buffer->getBuffer().data());
   MD5Ctx.final(Result.MD5);
 
   return Result;
 }
 
 namespace clang {
 bool operator==(const ASTUnit::PreambleFileHash &LHS,
                 const ASTUnit::PreambleFileHash &RHS) {
   return LHS.Size == RHS.Size && LHS.ModTime == RHS.ModTime &&
-         memcmp(LHS.MD5, RHS.MD5, sizeof(LHS.MD5)) == 0;
+         LHS.MD5 == RHS.MD5;
 }
 } // namespace clang
 
 static std::pair<unsigned, unsigned>
 makeStandaloneRange(CharSourceRange Range, const SourceManager &SM,
                     const LangOptions &LangOpts) {
   CharSourceRange FileRange = Lexer::makeFileCharRange(Range, SM, LangOpts);
   unsigned Offset = SM.getFileOffset(FileRange.getBegin());

lld/trunk/COFF/Writer.cpp

void Writer::writeBuildId() {
   Hash.update(ArrayRef<uint8_t>{Buffer->getBufferStart(),
                                 Buffer->getBufferEnd()});
   Hash.final(Res);
 
   assert(BuildId->DI->Signature.CVSignature == OMF::Signature::PDB70 &&
          "only PDB 7.0 is supported");
   assert(sizeof(Res) == sizeof(BuildId->DI->PDB70.Signature) &&
          "signature size mismatch");
-  memcpy(BuildId->DI->PDB70.Signature, Res,
+  memcpy(BuildId->DI->PDB70.Signature, Res.Bytes.data(),
          sizeof(codeview::PDB70DebugInfo::Signature));
   // TODO(compnerd) track the Age
   BuildId->DI->PDB70.Age = 1;
 }
 
 OutputSection *Writer::findSection(StringRef Name) {
   for (OutputSection *Sec : OutputSections)
     if (Sec->getName() == Name)

lldb/trunk/source/Host/common/FileSystem.cpp

 }
 
 bool FileSystem::CalculateMD5(const FileSpec &file_spec, uint64_t offset,
                               uint64_t length, uint64_t &low, uint64_t &high) {
   llvm::MD5::MD5Result md5_result;
   if (!CalcMD5(file_spec, offset, length, md5_result))
     return false;
 
-  const auto uint64_res = reinterpret_cast<const uint64_t *>(md5_result);
-  high = uint64_res[0];
-  low = uint64_res[1];
+  std::tie(high, low) = md5_result.words();
 
   return true;
 }
 
 bool FileSystem::CalculateMD5AsString(const FileSpec &file_spec,
                                       std::string &digest_str) {
   return CalculateMD5AsString(file_spec, 0, 0, digest_str);
 }

lldb/trunk/source/Utility/DataExtractor.cpp

void DataExtractor::Checksum(llvm::SmallVectorImpl<uint8_t> &dest,
   llvm::MD5 md5;
 
   const llvm::ArrayRef<uint8_t> data(GetDataStart(), max_data);
   md5.update(data);
 
   llvm::MD5::MD5Result result;
   md5.final(result);
 
-  dest.resize(16);
-  std::copy(result, result + 16, dest.begin());
+  dest.clear();
+  dest.append(result.Bytes.begin(), result.Bytes.end());
 }

llvm/trunk/include/llvm/Support/FileSystem.h

 #define LLVM_SUPPORT_FILESYSTEM_H
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/MD5.h"
 #include <cassert>
 #include <cstdint>
 #include <ctime>
 #include <memory>
 #include <stack>
 #include <string>
 #include <system_error>
 #include <tuple>

 /// @brief Resize path to size. File is resized as if by POSIX truncate().
 ///
 /// @param FD Input file descriptor.
 /// @param Size Size to resize to.
 /// @returns errc::success if \a path has been resized to \a size, otherwise a
 ///          platform-specific error_code.
 std::error_code resize_file(int FD, uint64_t Size);
 
+/// @brief Compute an MD5 hash of a file's contents.
+///
+/// @param FD Input file descriptor.
+/// @returns An MD5Result with the hash computed, if successful, otherwise a
+///          std::error_code.
+ErrorOr<MD5::MD5Result> md5_contents(int FD);
+
+/// @brief Version of compute_md5 that doesn't require an open file descriptor.
+ErrorOr<MD5::MD5Result> md5_contents(const Twine &Path);
+
 /// @}
 /// @name Physical Observers
 /// @{
 
 /// @brief Does file exist?
 ///
 /// @param status A file_status previously returned from stat.
 /// @returns True if the file represented by status exists, false if it does

llvm/trunk/include/llvm/Support/MD5.h

class MD5 {
   MD5_u32plus c = 0x98badcfe;
   MD5_u32plus d = 0x10325476;
   MD5_u32plus hi = 0;
   MD5_u32plus lo = 0;
   uint8_t buffer[64];
   MD5_u32plus block[16];
 
 public:
-  typedef uint8_t MD5Result[16];
+  struct MD5Result {
+    std::array<uint8_t, 16> Bytes;
+
+    operator std::array<uint8_t, 16>() const { return Bytes; }
+
+    const uint8_t &operator[](size_t I) const { return Bytes[I]; }
+    uint8_t &operator[](size_t I) { return Bytes[I]; }
+
+    SmallString<32> digest() const;
+
+    uint64_t low() const {
+      // Our MD5 implementation returns the result in little endian, so the low
+      // word is first.
+      using namespace support;
+      return endian::read<uint64_t, little, unaligned>(Bytes.data());
+    }
+
+    uint64_t high() const {
+      using namespace support;
+      return endian::read<uint64_t, little, unaligned>(Bytes.data() + 8);
+    }
+    std::pair<uint64_t, uint64_t> words() const {
+      using namespace support;
+      return std::make_pair(high(), low());
+    }
+  };
 
   MD5();
 
   /// \brief Updates the hash for the byte stream provided.
   void update(ArrayRef<uint8_t> Data);
 
   /// \brief Updates the hash for the StringRef provided.
   void update(StringRef Str);
 
   /// \brief Finishes off the hash and puts the result in result.
   void final(MD5Result &Result);
 
   /// \brief Translates the bytes in \p Res to a hex string that is
   /// deposited into \p Str. The result will be of length 32.
   static void stringifyResult(MD5Result &Result, SmallString<32> &Str);
 
   /// \brief Computes the hash for a given bytes.
   static std::array<uint8_t, 16> hash(ArrayRef<uint8_t> Data);
 
 private:
   const uint8_t *body(ArrayRef<uint8_t> Data);
 };
 
+inline bool operator==(const MD5::MD5Result &LHS, const MD5::MD5Result &RHS) {
+  return LHS.Bytes == RHS.Bytes;
+}
+
 /// Helper to compute and return lower 64 bits of the given string's MD5 hash.
 inline uint64_t MD5Hash(StringRef Str) {
   using namespace support;
 
   MD5 Hash;
   Hash.update(Str);
   MD5::MD5Result Result;
   Hash.final(Result);
-  // Return the least significant 8 bytes. Our MD5 implementation returns the
-  // result in little endian, so we may need to swap bytes.
-  return endian::read<uint64_t, little, unaligned>(Result);
+  // Return the least significant word.
+  return Result.low();
 }
 
 } // end namespace llvm
 
 #endif // LLVM_SUPPORT_MD5_H

llvm/trunk/lib/CodeGen/AsmPrinter/DIEHash.cpp

uint64_t DIEHash::computeCUSignature(const DIE &Die) {
   // Hash the DIE.
   computeHash(Die);
 
   // Now return the result.
   MD5::MD5Result Result;
   Hash.final(Result);
 
   // ... take the least significant 8 bytes and return those. Our MD5
-  // implementation always returns its results in little endian, swap bytes
-  // appropriately.
-  return support::endian::read64le(Result + 8);
+  // implementation always returns its results in little endian, so we actually
+  // need the "high" word.
+  return Result.high();
 }
 
 /// This is based on the type signature computation given in section 7.27 of the
 /// DWARF4 standard. It is an md5 hash of the flattened description of the DIE
 /// with the inclusion of additional forms not specifically called out in the
 /// standard.
 uint64_t DIEHash::computeTypeSignature(const DIE &Die) {
   Numbering.clear();
   Numbering[&Die] = 1;
 
   if (const DIE *Parent = Die.getParent())
     addParentContext(*Parent);
 
   // Hash the DIE.
   computeHash(Die);
 
   // Now return the result.
   MD5::MD5Result Result;
   Hash.final(Result);
 
   // ... take the least significant 8 bytes and return those. Our MD5
-  // implementation always returns its results in little endian, swap bytes
-  // appropriately.
-  return support::endian::read64le(Result + 8);
+  // implementation always returns its results in little endian, so we actually
+  // need the "high" word.
+  return Result.high();
 }

llvm/trunk/lib/CodeGen/AsmPrinter/DwarfDebug.cpp

 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Dwarf.h"
-#include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetFrameLowering.h"
if (SingleCU)
     SplitTypeUnitFileTable.setCompilationDir(CU.getCUNode()->getDirectory());
   return &SplitTypeUnitFileTable;
 }
 
 uint64_t DwarfDebug::makeTypeSignature(StringRef Identifier) {
   MD5 Hash;
   Hash.update(Identifier);
   // ... take the least significant 8 bytes and return those. Our MD5
-  // implementation always returns its results in little endian, swap bytes
-  // appropriately.
+  // implementation always returns its results in little endian, so we actually
+  // need the "high" word.
   MD5::MD5Result Result;
   Hash.final(Result);
-  return support::endian::read64le(Result + 8);
+  return Result.high();
 }
 
 void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU,
                                       StringRef Identifier, DIE &RefDie,
                                       const DICompositeType *CTy) {
   // Fast path if we're building some type units and one has already used the
   // address pool we know we're going to throw away all this work anyway, so
   // don't bother building dependent types.

llvm/trunk/lib/Support/MD5.cpp

void MD5::final(MD5Result &Result) {
   body(makeArrayRef(buffer, 64));
 
   support::endian::write32le(&Result[0], a);
   support::endian::write32le(&Result[4], b);
   support::endian::write32le(&Result[8], c);
   support::endian::write32le(&Result[12], d);
 }
 
-void MD5::stringifyResult(MD5Result &Result, SmallString<32> &Str) {
+SmallString<32> MD5::MD5Result::digest() const {
+  SmallString<32> Str;
   raw_svector_ostream Res(Str);
   for (int i = 0; i < 16; ++i)
-    Res << format("%.2x", Result[i]);
+    Res << format("%.2x", Bytes[i]);
+  return Str;
+}
+
+void MD5::stringifyResult(MD5Result &Result, SmallString<32> &Str) {
+  Str = Result.digest();
 }
 
 std::array<uint8_t, 16> MD5::hash(ArrayRef<uint8_t> Data) {
   MD5 Hash;
   Hash.update(Data);
   MD5::MD5Result Res;
   Hash.final(Res);
 
-  std::array<uint8_t, 16> Arr;
-  memcpy(Arr.data(), Res, sizeof(Res));
-  return Arr;
+  return Res;
 }

llvm/trunk/lib/Support/Path.cpp

 //===-- Path.cpp - Implement OS Path Concept ------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements the operating system Path API.
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/Path.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/COFF.h"
-#include "llvm/Support/MachO.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
+#include "llvm/Support/MachO.h"
 #include "llvm/Support/Process.h"
 #include <cctype>
 #include <cstring>
 
 #if !defined(_MSC_VER) && !defined(__MINGW32__)
 #include <unistd.h>
 #else
 #include <io.h>
std::error_code copy_file(const Twine &From, const Twine &To) {
   close(WriteFD);
   delete[] Buf;
 
   if (BytesRead < 0 || BytesWritten < 0)
     return std::error_code(errno, std::generic_category());
   return std::error_code();
 }
 
+ErrorOr<MD5::MD5Result> md5_contents(int FD) {
+  MD5 Hash;
+
+  constexpr size_t BufSize = 4096;
+  std::vector<uint8_t> Buf(BufSize);
+  int BytesRead = 0;
+  for (;;) {
+    BytesRead = read(FD, Buf.data(), BufSize);
+    if (BytesRead <= 0)
+      break;
+    Hash.update(makeArrayRef(Buf.data(), BytesRead));
+  }
+
+  if (BytesRead < 0)
+    return std::error_code(errno, std::generic_category());
+  MD5::MD5Result Result;
+  Hash.final(Result);
+  return Result;
+}
+
+ErrorOr<MD5::MD5Result> md5_contents(const Twine &Path) {
+  int FD;
+  if (auto EC = openFileForRead(Path, FD))
+    return EC;
+
+  auto Result = md5_contents(FD);
+  close(FD);
+  return Result;
+}
+
 bool exists(file_status status) {
   return status_known(status) && status.type() != file_type::file_not_found;
 }
 
 bool status_known(file_status s) {
   return s.type() != file_type::status_error;
 }
 

llvm/trunk/unittests/Support/MD5Test.cpp

TEST(MD5Test, MD5) {
   TestMD5Sum("abcdefghijklmnopqrstuvwxyz", "c3fcd3d76192e4007dfb496cca67e13b");
 }
 
 TEST(MD5HashTest, MD5) {
   ArrayRef<uint8_t> Input((const uint8_t *)"abcdefghijklmnopqrstuvwxyz", 26);
   std::array<uint8_t, 16> Vec = MD5::hash(Input);
   MD5::MD5Result MD5Res;
   SmallString<32> Res;
-  memcpy(MD5Res, Vec.data(), Vec.size());
+  memcpy(MD5Res.Bytes.data(), Vec.data(), Vec.size());
   MD5::stringifyResult(MD5Res, Res);
   EXPECT_EQ(Res, "c3fcd3d76192e4007dfb496cca67e13b");
+  EXPECT_EQ(0x3be167ca6c49fb7dULL, MD5Res.high());
+  EXPECT_EQ(0x00e49261d7d3fcc3ULL, MD5Res.low());
 }
 }

llvm/trunk/unittests/Support/Path.cpp

TEST_F(FileSystemTest, Resize) {
   ASSERT_NO_ERROR(fs::resize_file(FD, 123));
   fs::file_status Status;
   ASSERT_NO_ERROR(fs::status(FD, Status));
   ASSERT_EQ(Status.getSize(), 123U);
   ::close(FD);
   ASSERT_NO_ERROR(fs::remove(TempPath));
 }
 
+TEST_F(FileSystemTest, MD5) {
+  int FD;
+  SmallString<64> TempPath;
+  ASSERT_NO_ERROR(fs::createTemporaryFile("prefix", "temp", FD, TempPath));
+  StringRef Data("abcdefghijklmnopqrstuvwxyz");
+  write(FD, Data.data(), Data.size());
+  lseek(FD, 0, SEEK_SET);
+  auto Hash = fs::md5_contents(FD);
+  ::close(FD);
+  ASSERT_NO_ERROR(Hash.getError());
+
+  EXPECT_STREQ("c3fcd3d76192e4007dfb496cca67e13b", Hash->digest().c_str());
+}
+
 TEST_F(FileSystemTest, FileMapping) {
   // Create a temp file.
   int FileDescriptor;
   SmallString<64> TempPath;
   ASSERT_NO_ERROR(
       fs::createTemporaryFile("prefix", "temp", FileDescriptor, TempPath));
   unsigned Size = 4096;
   ASSERT_NO_ERROR(fs::resize_file(FileDescriptor, Size));