Clang: Allow selecting the hash algorithm for file checksums in debug info.

[arlosi]

Summary

Adds clang-cl support for the /ZH: option used to select MD5, SHA1,
or SHA_256 hashing algorithms in debug info.

Previously only the MD5 algorithm was supported.

[jansvoboda11]

clang/include/clang/Driver/Options.td

2775

Can you please use a marshalling annotation here and remove the code in CompilerInvocation.cpp? https://clang.llvm.org/docs/InternalsManual.html#adding-new-command-line-option

[arlosi]

Use automatic marshaling for command line arguments.

[arlosi]

Ping

[hans]

Sorry for not seeing this sooner.

I think this looks great, just a few comments, and it also needs tests.

clang/include/clang/Basic/CodeGenOptions.def

303

Why does it need 4 bits? Wouldn't 2 be enough?

clang/lib/CodeGen/CGDebugInfo.cpp

368–369

I'm not sure changing the return type helps that much? I suppose it makes the calling code a little cleaner, but I think it makes the function itself a little more confusing.

398

toHex() from StringExtras.h is much nicer, and MD5 should be using that internally too.

[hans]

Do you think you'll have time to work on this? Otherwise I'd be happy to take it over.

[arlosi]

I wouldn't be able to look at this for a few weeks. If you have the time to
take it over, that works for me. Thanks!

[thakis]

I addressed pending comments, did some other minor tweaks, and uploaded the modified diff to D134544.

[probinson]

So this one can be marked abandoned? You might need to commandeer it first.

[arlosi]

Closing in favor of D134544

clang/include/clang/Basic/CodeGenOptions.h

public:
 
   enum EmbedBitcodeKind {
     Embed_Off,      // No embedded bitcode.
     Embed_All,      // Embed both bitcode and commandline in the output.
     Embed_Bitcode,  // Embed just the bitcode in the output.
     Embed_Marker    // Embed a marker as a placeholder for bitcode.
   };
 
+  enum SrcHashAlgorithm {
+    CSK_MD5,
+    CSK_SHA1,
+    CSK_SHA256,
+  };
+
   // This field stores one of the allowed values for the option
   // -fbasic-block-sections=.  The allowed values with this option are:
   // {"labels", "all", "list=<file>", "none"}.
   //
   // "labels":      Only generate basic block symbols (labels) for all basic
   //                blocks, do not generate unique sections for basic blocks.
   //                Use the machine basic block id in the symbol name to
   //                associate profile info from virtual address to machine

clang/include/clang/Basic/CodeGenOptions.def

 
 CODEGENOPT(DebugTypeExtRefs, 1, 0) ///< Whether or not debug info should contain
                                    ///< external references to a PCH or module.
 
 CODEGENOPT(DebugExplicitImport, 1, 0)  ///< Whether or not debug info should
                                        ///< contain explicit imports for
                                        ///< anonymous namespaces
 
+/// Set debug info source file hashing algorithm
+ENUM_CODEGENOPT(DebugSrcHashAlgorithm, SrcHashAlgorithm, 4, CSK_MD5)
+
 CODEGENOPT(SplitDwarfInlining, 1, 1) ///< Whether to include inlining info in the
                                      ///< skeleton CU to allow for symbolication
                                      ///< of inline stack frames without .dwo files.
 CODEGENOPT(DebugFwdTemplateParams, 1, 0) ///< Whether to emit complete
                                          ///< template parameter descriptions in
                                          ///< forward declarations (versus just
                                          ///< including them in the name).
 CODEGENOPT(EmitLLVMUseLists, 1, 0) ///< Control whether to serialize use-lists.

clang/include/clang/Driver/Options.td

defm column_info : BoolOption<"g", "column-info",
   CodeGenOpts<"DebugColumnInfo">, DefaultTrue,
   NegFlag<SetFalse, [CC1Option]>, PosFlag<SetTrue>, BothFlags<[CoreOption]>>,
   Group<g_flags_Group>;
 def gsplit_dwarf : Flag<["-"], "gsplit-dwarf">, Group<g_flags_Group>;
 def gsplit_dwarf_EQ : Joined<["-"], "gsplit-dwarf=">, Group<g_flags_Group>,
   HelpText<"Set DWARF fission mode to either 'split' or 'single'">,
   Values<"split,single">;
 def gno_split_dwarf : Flag<["-"], "gno-split-dwarf">, Group<g_flags_Group>;
+def gsrc_hash_algorithm_EQ : Joined<["-"], "gsrc-hash-algorithm=">, Group<g_flags_Group>, Flags<[CC1Option]>,
+  HelpText<"Hashing algorithm for file checksum in debug info 'md5', 'sha1', or 'sha256'">,
+  Values<"md5,sha1,sha256">;
 def ggnu_pubnames : Flag<["-"], "ggnu-pubnames">, Group<g_flags_Group>, Flags<[CC1Option]>;
 def gno_gnu_pubnames : Flag<["-"], "gno-gnu-pubnames">, Group<g_flags_Group>;
 def gpubnames : Flag<["-"], "gpubnames">, Group<g_flags_Group>, Flags<[CC1Option]>;
 def gno_pubnames : Flag<["-"], "gno-pubnames">, Group<g_flags_Group>;
 def gdwarf_aranges : Flag<["-"], "gdwarf-aranges">, Group<g_flags_Group>;
 def gmodules : Flag <["-"], "gmodules">, Group<gN_Group>,
   HelpText<"Generate debug info with external references to clang modules"
            " or precompiled headers">;

 def _SLASH_Zc_twoPhase : CLFlag<"Zc:twoPhase">,
   HelpText<"Enable two-phase name lookup in templates">,
   Alias<fno_delayed_template_parsing>;
 def _SLASH_Zc_twoPhase_ : CLFlag<"Zc:twoPhase-">,
   HelpText<"Disable two-phase name lookup in templates (default)">,
   Alias<fdelayed_template_parsing>;
 def _SLASH_Z7 : CLFlag<"Z7">,
   HelpText<"Enable CodeView debug information in object files">;
+def _SLASH_ZH : CLCompileJoined<"ZH:">,
+  HelpText<"Hash algorithm for file checksum in debug info (MD5, SHA1, or SHA_256)">,
+  Values<"MD5,SHA1,SHA_256">;
 def _SLASH_Zi : CLFlag<"Zi">, Alias<_SLASH_Z7>,
   HelpText<"Like /Z7">;
 def _SLASH_Zp : CLJoined<"Zp">,
   HelpText<"Set default maximum struct packing alignment">,
   Alias<fpack_struct_EQ>;
 def _SLASH_Zp_flag : CLFlag<"Zp">,
   HelpText<"Set default maximum struct packing alignment to 1">,
   Alias<fpack_struct_EQ>, AliasArgs<["1"]>;

 def _SLASH_w : CLIgnoredJoined<"w">;
 def _SLASH_Zc___cplusplus : CLIgnoredFlag<"Zc:__cplusplus">;
 def _SLASH_Zc_auto : CLIgnoredFlag<"Zc:auto">;
 def _SLASH_Zc_forScope : CLIgnoredFlag<"Zc:forScope">;
 def _SLASH_Zc_inline : CLIgnoredFlag<"Zc:inline">;
 def _SLASH_Zc_rvalueCast : CLIgnoredFlag<"Zc:rvalueCast">;
 def _SLASH_Zc_ternary : CLIgnoredFlag<"Zc:ternary">;
 def _SLASH_Zc_wchar_t : CLIgnoredFlag<"Zc:wchar_t">;
-def _SLASH_ZH_MD5 : CLIgnoredFlag<"ZH:MD5">;
-def _SLASH_ZH_SHA1 : CLIgnoredFlag<"ZH:SHA1">;
-def _SLASH_ZH_SHA_256 : CLIgnoredFlag<"ZH:SHA_256">;
 def _SLASH_Zm : CLIgnoredJoined<"Zm">;
 def _SLASH_Zo : CLIgnoredFlag<"Zo">;
 def _SLASH_Zo_ : CLIgnoredFlag<"Zo-">;
 
 
 // Unsupported:
 
 def _SLASH_await : CLFlag<"await">;

clang/lib/CodeGen/CGDebugInfo.h

private:
 
   /// Return current directory name.
   StringRef getCurrentDirname();
 
   /// Create new compile unit.
   void CreateCompileUnit();
 
   /// Compute the file checksum debug info for input file ID.
-  Optional<llvm::DIFile::ChecksumKind>
-  computeChecksum(FileID FID, SmallString<32> &Checksum) const;
+  /// Storage of the checksum string is owned by the caller.
+  Optional<llvm::DIFile::ChecksumInfo<StringRef>>
+  computeChecksum(FileID FID, SmallString<64> &Checksum) const;
 
   /// Get the source of the given file ID.
   Optional<StringRef> getSource(const SourceManager &SM, FileID FID);
 
   /// Convenience function to get the file debug info descriptor for the input
   /// location.
   llvm::DIFile *getOrCreateFile(SourceLocation Loc);
 

clang/lib/CodeGen/CGDebugInfo.cpp

 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/SHA1.h"
+#include "llvm/Support/SHA256.h"
 #include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/raw_ostream.h"
 using namespace clang;
 using namespace clang::CodeGen;
 
 static uint32_t getTypeAlignIfRequired(const Type *Ty, const ASTContext &Ctx) {
   auto TI = Ctx.getTypeInfo(Ty);
   return TI.AlignIsRequired ? TI.Align : 0;
 }
 
if (CGM.getLangOpts().CPlusPlus) {
         return internString(UnnamedType);
       }
     }
   }
 
   return StringRef();
 }
 
-Optional<llvm::DIFile::ChecksumKind>
-CGDebugInfo::computeChecksum(FileID FID, SmallString<32> &Checksum) const {
+Optional<llvm::DIFile::ChecksumInfo<StringRef>>
+CGDebugInfo::computeChecksum(FileID FID, SmallString<64> &Checksum) const {
   Checksum.clear();
 
+  llvm::DIFile::ChecksumKind CSKind;
+  Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo;
   if (!CGM.getCodeGenOpts().EmitCodeView &&
       CGM.getCodeGenOpts().DwarfVersion < 5)
     return None;
 
   SourceManager &SM = CGM.getContext().getSourceManager();
   Optional<llvm::MemoryBufferRef> MemBuffer = SM.getBufferOrNone(FID);
   if (!MemBuffer)
     return None;
 
+  switch (CGM.getCodeGenOpts().getDebugSrcHashAlgorithm()) {
+  case clang::CodeGenOptions::CSK_MD5: {
+    CSKind = llvm::DIFile::CSK_MD5;
     llvm::MD5 Hash;
     llvm::MD5::MD5Result Result;
-
     Hash.update(MemBuffer->getBuffer());
     Hash.final(Result);
+    Checksum = Result.digest();
+    break;
+  }
+  case clang::CodeGenOptions::CSK_SHA1: {
+    CSKind = llvm::DIFile::CSK_SHA1;
+    llvm::SHA1 Hash;
+    Hash.update(MemBuffer->getBuffer());
+    StringRef Result = Hash.final();
+    llvm::raw_svector_ostream Res(Checksum);
+    for (int i = 0; i < 20; ++i)
+      Res << llvm::format("%.2x", static_cast<uint8_t>(Result[i]));
+    break;
+  }
+  case clang::CodeGenOptions::CSK_SHA256: {
+    CSKind = llvm::DIFile::CSK_SHA256;
+    llvm::SHA256 Hash;
+    Hash.update(MemBuffer->getBuffer());
+    StringRef Result = Hash.final();
+    llvm::raw_svector_ostream Res(Checksum);
+    for (int i = 0; i < 32; ++i)
+      Res << llvm::format("%.2x", static_cast<uint8_t>(Result[i]));
+    break;
+  }
+  default:
+    return None;
+  }
 
-  Hash.stringifyResult(Result, Checksum);
-  return llvm::DIFile::CSK_MD5;
+  CSInfo.emplace(CSKind, Checksum);
+  return CSInfo;
 }
 
 Optional<StringRef> CGDebugInfo::getSource(const SourceManager &SM,
                                            FileID FID) {
   if (!CGM.getCodeGenOpts().EmbedSource)
     return None;
 
   bool SourceInvalid = false;
llvm::DIFile *CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
   // Cache the results.
   auto It = DIFileCache.find(FileName.data());
   if (It != DIFileCache.end()) {
     // Verify that the information still exists.
     if (llvm::Metadata *V = It->second)
       return cast<llvm::DIFile>(V);
   }
 
-  SmallString<32> Checksum;
+  SmallString<64> Checksum;
+  Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo =
+      computeChecksum(FID, Checksum);
 
-  Optional<llvm::DIFile::ChecksumKind> CSKind = computeChecksum(FID, Checksum);
-  Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo;
-  if (CSKind)
-    CSInfo.emplace(*CSKind, Checksum);
   return createFile(FileName, CSInfo, getSource(SM, SM.getFileID(Loc)));
 }
 
 llvm::DIFile *
 CGDebugInfo::createFile(StringRef FileName,
                         Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo,
                         Optional<StringRef> Source) {
   StringRef Dir;
StringRef CGDebugInfo::getCurrentDirname() {
   if (!CWDName.empty())
     return CWDName;
   SmallString<256> CWD;
   llvm::sys::fs::current_path(CWD);
   return CWDName = internString(CWD);
 }
 
 void CGDebugInfo::CreateCompileUnit() {
-  SmallString<32> Checksum;
-  Optional<llvm::DIFile::ChecksumKind> CSKind;
+  SmallString<64> Checksum;
   Optional<llvm::DIFile::ChecksumInfo<StringRef>> CSInfo;
 
   // Should we be asking the SourceManager for the main file name, instead of
   // accepting it as an argument? This just causes the main file name to
   // mismatch with source locations and create extra lexical scopes or
   // mismatched debug info (a CU with a DW_AT_file of "-", because that's what
   // the driver passed, but functions/other things have DW_AT_file of "<stdin>"
   // because that's what the SourceManager says)
if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     // if the input file is a preprocessed source, use the module name for
     // debug info. The module name comes from the name specified in the first
     // linemarker if the input is a preprocessed source.
     if (MainFile->getName() == MainFileName &&
         FrontendOptions::getInputKindForExtension(
             MainFile->getName().rsplit('.').second)
             .isPreprocessed())
       MainFileName = CGM.getModule().getName().str();
-
-    CSKind = computeChecksum(SM.getMainFileID(), Checksum);
+    CSInfo = computeChecksum(SM.getMainFileID(), Checksum);
   }
 
   llvm::dwarf::SourceLanguage LangTag;
   const LangOptions &LO = CGM.getLangOpts();
   if (LO.CPlusPlus) {
     if (LO.ObjC)
       LangTag = llvm::dwarf::DW_LANG_ObjC_plus_plus;
     else if (LO.CPlusPlus14)
void CGDebugInfo::CreateCompileUnit() {
   }
 
   uint64_t DwoId = 0;
   auto &CGOpts = CGM.getCodeGenOpts();
   // The DIFile used by the CU is distinct from the main source
   // file. Its directory part specifies what becomes the
   // DW_AT_comp_dir (the compilation directory), even if the source
   // file was specified with an absolute path.
-  if (CSKind)
-    CSInfo.emplace(*CSKind, Checksum);
   llvm::DIFile *CUFile = DBuilder.createFile(
       remapDIPath(MainFileName), remapDIPath(getCurrentDirname()), CSInfo,
       getSource(SM, SM.getMainFileID()));
 
   StringRef Sysroot, SDK;
   if (CGM.getCodeGenOpts().getDebuggerTuning() == llvm::DebuggerKind::LLDB) {
     Sysroot = CGM.getHeaderSearchOpts().Sysroot;
     auto B = llvm::sys::path::rbegin(Sysroot);

clang/lib/Driver/ToolChains/Clang.cpp

if (GuardArgs.equals_lower("cf")) {
       CmdArgs.push_back("-ehcontguard");
     } else if (GuardArgs.equals_lower("cf-") ||
                GuardArgs.equals_lower("ehcont-")) {
       // Do nothing, but we might want to emit a security warning in future.
     } else {
       D.Diag(diag::err_drv_invalid_value) << A->getSpelling() << GuardArgs;
     }
   }
+
+  if (Arg *A = Args.getLastArg(options::OPT__SLASH_ZH)) {
+    StringRef Val = A->getValue();
+    if (Val.equals("MD5")) {
+      CmdArgs.push_back("-gsrc-hash-algorithm=md5");
+    } else if (Val.equals("SHA1")) {
+      CmdArgs.push_back("-gsrc-hash-algorithm=sha1");
+    } else if (Val.equals("SHA_256")) {
+      CmdArgs.push_back("-gsrc-hash-algorithm=sha256");
+    } else {
+      D.Diag(diag::err_drv_invalid_value) << A->getAsString(Args) << Val;
+    }
+  }
 }
 
 const char *Clang::getBaseInputName(const ArgList &Args,
                                     const InputInfo &Input) {
   return Args.MakeArgString(llvm::sys::path::filename(Input.getBaseInput()));
 }
 
 const char *Clang::getBaseInputStem(const ArgList &Args,

clang/lib/Frontend/CompilerInvocation.cpp

#undef CODEGEN_OPTION_WITH_MARSHALLING
 
   Opts.EmitVersionIdentMetadata = Args.hasFlag(OPT_Qy, OPT_Qn, true);
 
   if (Args.hasArg(options::OPT_ffinite_loops))
     Opts.FiniteLoops = CodeGenOptions::FiniteLoopsKind::Always;
   else if (Args.hasArg(options::OPT_fno_finite_loops))
     Opts.FiniteLoops = CodeGenOptions::FiniteLoopsKind::Never;
 
+  if (Arg *A = Args.getLastArg(OPT_gsrc_hash_algorithm_EQ)) {
+    unsigned Val = llvm::StringSwitch<unsigned>(A->getValue())
+                       .Case("md5", CodeGenOptions::CSK_MD5)
+                       .Case("sha1", CodeGenOptions::CSK_SHA1)
+                       .Case("sha256", CodeGenOptions::CSK_SHA256)
+                       .Default(~0U);
+    if (Val == ~0U)
+      Diags.Report(diag::err_drv_invalid_value)
+          << A->getAsString(Args) << A->getValue();
+    else
+      Opts.setDebugSrcHashAlgorithm(
+          static_cast<CodeGenOptions::SrcHashAlgorithm>(Val));
+  }
+
   return Success && Diags.getNumErrors() == NumErrorsBefore;
 }
 
 static void
 GenerateDependencyOutputArgs(const DependencyOutputOptions &Opts,
                              SmallVectorImpl<const char *> &Args,
                              CompilerInvocation::StringAllocator SA) {
   const DependencyOutputOptions &DependencyOutputOpts = Opts;