Use sha256 for source file debug info checksums with MSVC compat >= 1930

[hans]

Summary

From the VS2022 release notes, it sounds like newer MSVC versions are using SHA256 for these checksums: (search for "SHA-256" in https://docs.microsoft.com/en-us/visualstudio/releases/2022/release-notes#17.0.0)

Since D75785 laid the groundwork, let's hook it up.

While here, I noticed llvm::SHA256 doesn't have a method to get the hash as a hex string, like llvm::MD5. But we can use llvm::toHex() and actually that could be made more efficient and llvm::MD5 could use that too.

[hans]

After sending this, I realized we should probably have a test to make sure it gets all the way into the .o and .pdb files correctly...

And, maybe we should hook up the /ZH flag? (https://docs.microsoft.com/en-us/cpp/build/reference/zh?view=msvc-170&preserve-view=true still says MD5 is default, but I guess that's just out of date)

[arlosi]

I submitted a change last year to hook up the /ZH option, but it was never completed. https://reviews.llvm.org/D98438

[rnk]

I think it's worth taking the time to hook up /ZH. Sorry we missed the patch last March.

[hans]

In D116835#3228296, @arlosi wrote:

I submitted a change last year to hook up the /ZH option, but it was never completed. https://reviews.llvm.org/D98438

I just put some comments on that. I'm happy to provide review if you think you have time to complete it?

(I'll break out the toHex() related changes here and try to get those landed separately.)

clang/include/clang/Basic/LangOptions.h

enum MSVCMajorVersion {
     MSVC2012 = 1700,
     MSVC2013 = 1800,
     MSVC2015 = 1900,
     MSVC2017 = 1910,
     MSVC2017_5 = 1912,
     MSVC2017_7 = 1914,
     MSVC2019 = 1920,
     MSVC2019_8 = 1928,
+    MSVC2022 = 1930,
   };
 
   enum SYCLMajorVersion {
     SYCL_None,
     SYCL_2017,
     SYCL_2020,
     // The "default" SYCL version to be used when none is specified on the
     // frontend command line.

clang/lib/CodeGen/CGDebugInfo.h

private:
   /// Return current directory name.
   StringRef getCurrentDirname();
 
   /// Create new compile unit.
   void CreateCompileUnit();
 
   /// Compute the file checksum debug info for input file ID.
   Optional<llvm::DIFile::ChecksumKind>
-  computeChecksum(FileID FID, SmallString<32> &Checksum) const;
+  computeChecksum(FileID FID, SmallString<64> &Checksum) const;
 
   /// Get the source of the given file ID.
   Optional<StringRef> getSource(const SourceManager &SM, FileID FID);
 
   /// Convenience function to get the file debug info descriptor for the input
   /// location.
   llvm::DIFile *getOrCreateFile(SourceLocation Loc);
 

clang/lib/CodeGen/CGDebugInfo.cpp

 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/SHA256.h"
 #include "llvm/Support/TimeProfiler.h"
 using namespace clang;
 using namespace clang::CodeGen;
 
 static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) {
   auto TI = Ctx.getTypeInfo(Ty);
   return TI.isAlignRequired() ? TI.Align : 0;
 }
if (CGM.getLangOpts().CPlusPlus) {
       }
     }
   }
 
   return StringRef();
 }
 
 Optional<llvm::DIFile::ChecksumKind>
-CGDebugInfo::computeChecksum(FileID FID, SmallString<32> &Checksum) const {
+CGDebugInfo::computeChecksum(FileID FID, SmallString<64> &Checksum) const {
   Checksum.clear();
 
   if (!CGM.getCodeGenOpts().EmitCodeView &&
       CGM.getCodeGenOpts().DwarfVersion < 5)
     return None;
 
   SourceManager &SM = CGM.getContext().getSourceManager();
   Optional<llvm::MemoryBufferRef> MemBuffer = SM.getBufferOrNone(FID);
   if (!MemBuffer)
     return None;
 
+  if (CGM.getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2022)) {
+    llvm::toHex(
+        llvm::SHA256::hash(llvm::arrayRefFromStringRef(MemBuffer->getBuffer())),
+        /*LowerCase*/ true, Checksum);
+    return llvm::DIFile::CSK_SHA256;
+  }
+
   llvm::MD5 Hash;
   llvm::MD5::MD5Result Result;
 
   Hash.update(MemBuffer->getBuffer());
   Hash.final(Result);
 
   Hash.stringifyResult(Result, Checksum);
   return llvm::DIFile::CSK_MD5;
llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
   // Cache the results.
   auto It = DIFileCache.find(FileName.data());
   if (It != DIFileCache.end()) {
     // Verify that the information still exists.
     if (llvm::Metadata *V = It->second)
       return cast<llvm::DIFile>(V);
   }
 
-  SmallString<32> Checksum;
+  SmallString<64> Checksum;
 
   Optional<llvm::DIFile::ChecksumKind> CSKind = computeChecksum(FID, Checksum);
   Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo;
   if (CSKind)
     CSInfo.emplace(*CSKind, Checksum);
   return createFile(FileName, CSInfo, getSource(SM, SM.getFileID(Loc)));
 }
 
StringRef CGDebugInfo::getCurrentDirname() {
   if (!CWDName.empty())
     return CWDName;
   SmallString<256> CWD;
   llvm::sys::fs::current_path(CWD);
   return CWDName = internString(CWD);
 }
 
 void CGDebugInfo::CreateCompileUnit() {
-  SmallString<32> Checksum;
+  SmallString<64> Checksum;
   Optional<llvm::DIFile::ChecksumKind> CSKind;
   Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo;
 
   // Should we be asking the SourceManager for the main file name, instead of
   // accepting it as an argument? This just causes the main file name to
   // mismatch with source locations and create extra lexical scopes or
   // mismatched debug info (a CU with a DW_AT_file of "-", because that's what
   // the driver passed, but functions/other things have DW_AT_file of "<stdin>"

clang/test/CodeGen/debug-info-file-checksum.c

 // NOCHECKSUM: !DIFile(filename: "{{.*}}code-coverage-filter1.h", directory: "{{[^"]*}}")
 // NOCHECKSUM: !DIFile(filename: "{{.*}}code-coverage-filter2.h", directory: "{{[^"]*}}")
 // NOCHECKSUM: !DIFile(filename: "{{.*}}debug-info-file-checksum.c", directory: "{{[^"]*}}")
 
 // Ensure #line directives without name do emit checksums
 // RUN: %clang -emit-llvm -S -g -gcodeview -x c %S/Inputs/debug-info-file-checksum-line.cpp -o - | FileCheck %s --check-prefix CHECKSUM
 
 // CHECKSUM: !DIFile(filename: "{{.*}}debug-info-file-checksum-line.cpp", directory:{{.*}}, checksumkind: CSK_MD5, checksum: "7b568574d0e3c56c28e5e0234d1f4a06")
+
+// Later MSVC versions use SHA256.
+// RUN: %clang -target i686-pc-windows-msvc19.30.0 -emit-llvm -S -g -gcodeview -x c %S/Inputs/debug-info-file-checksum-line.cpp -o - | FileCheck %s --check-prefix SHA256
+// SHA256: !DIFile(filename: "{{.*}}debug-info-file-checksum-line.cpp", directory:{{.*}}, checksumkind: CSK_SHA256, checksum: "e28b727db04814680c02d0d2ec72bcd77f5e54131dafc34163bd1576e9428d42")

llvm/include/llvm/ADT/StringExtras.h

while (X) {
     unsigned char Mod = static_cast<unsigned char>(X) & 15;
     *--BufPtr = hexdigit(Mod, LowerCase);
     X >>= 4;
   }
 
   return std::string(BufPtr, std::end(Buffer));
 }
 
-/// Convert buffer \p Input to its hexadecimal representation.
-/// The returned string is double the size of \p Input.
-inline std::string toHex(StringRef Input, bool LowerCase = false) {
+inline void toHex(ArrayRef<uint8_t> Input, bool LowerCase,
+                  SmallVectorImpl<char> &Output) {
   static const char *const LUT = "0123456789ABCDEF";
   const uint8_t Offset = LowerCase ? 32 : 0;
   size_t Length = Input.size();
+  Output.resize_for_overwrite(Length * 2);
 
-  std::string Output;
-  Output.reserve(2 * Length);
   for (size_t i = 0; i < Length; ++i) {
-    const unsigned char c = Input[i];
-    Output.push_back(LUT[c >> 4] | Offset);
-    Output.push_back(LUT[c & 15] | Offset);
-  }
-  return Output;
+    const uint8_t c = Input[i];
+    Output[i * 2    ] = LUT[c >> 4] | Offset;
+    Output[i * 2 + 1] = LUT[c & 15] | Offset;
+  }
+
+/// Convert buffer \p Input to its hexadecimal representation.
+/// The returned string is double the size of \p Input.
+inline std::string toHex(StringRef Input, bool LowerCase = false) {
+  SmallString<16> Output;
+  toHex(arrayRefFromStringRef(Input), LowerCase, Output);
+  return std::string(Output);
 }
 
 inline std::string toHex(ArrayRef<uint8_t> Input, bool LowerCase = false) {
   return toHex(toStringRef(Input), LowerCase);
 }
 
 /// Store the binary representation of the two provided values, \p MSB and
 /// \p LSB, that make up the nibbles of a hexadecimal digit. If \p MSB or \p LSB

llvm/include/llvm/Support/MD5.h

public:
 
   /// Finishes off the hash, and returns a reference to the 16-byte hash data.
   /// This is suitable for getting the MD5 at any time without invalidating the
   /// internal state, so that more calls can be made into `update`.
   StringRef result();
 
   /// Translates the bytes in \p Res to a hex string that is
   /// deposited into \p Str. The result will be of length 32.
-  static void stringifyResult(MD5Result &Result, SmallString<32> &Str);
+  static void stringifyResult(MD5Result &Result, SmallVectorImpl<char> &Str);
 
   /// Computes the hash for a given bytes.
   static std::array<uint8_t, 16> hash(ArrayRef<uint8_t> Data);
 
 private:
   // Any 32-bit or wider unsigned integer data type will do.
   typedef uint32_t MD5_u32plus;
 

llvm/lib/Support/MD5.cpp

  * It is meant to be fast, but not as fast as possible.  Some known
  * optimizations are not included to reduce source code size and avoid
  * compile-time configuration.
  */
 
 #include "llvm/Support/MD5.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Endian.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
 #include <array>
 #include <cstdint>
 #include <cstring>
 
 // The basic MD5 functions.
 
 // F and G are optimized compared to their RFC 1321 definitions for
 // architectures that lack an AND-NOT instruction, just like in Colin Plumb's
StringRef MD5::result() {
   // Restore the state
   InternalState = StateToRestore;
 
   return Hash;
 }
 
 SmallString<32> MD5::MD5Result::digest() const {
   SmallString<32> Str;
-  raw_svector_ostream Res(Str);
-  for (int i = 0; i < 16; ++i)
-    Res << format("%.2x", Bytes[i]);
+  toHex(Bytes, /*LowerCase*/ true, Str);
   return Str;
 }
 
-void MD5::stringifyResult(MD5Result &Result, SmallString<32> &Str) {
-  Str = Result.digest();
+void MD5::stringifyResult(MD5Result &Result, SmallVectorImpl<char> &Str) {
+  toHex(Result.Bytes, /*LowerCase*/ true, Str);
 }
 
 std::array<uint8_t, 16> MD5::hash(ArrayRef<uint8_t> Data) {
   MD5 Hash;
   Hash.update(Data);
   MD5::MD5Result Res;
   Hash.final(Res);
 
   return Res;
 }