Add AST_SIGNATURE record to unhashed control block of pcm files (Patch series 2/3)

[dang]

Summary

This lets users of PCM get a clean hash of the AST for uniquing based
on that. This required looking up implicit modules in the module offset map to
be looked up by name instead of by file path.

The previous patch is at D79993.
The next patch is at D80003.

[dang]

Should I add the test here or in the clang-scan-deps patch?

[Bigcheese]

In D79998#2038430, @dang wrote:

Should I add the test here or in the clang-scan-deps patch?

It's best to have a test in every non-nfc patch. You should be able to test this with llvm-bcanalyzer.

[dang]

Add a small bug fix and test case

[dang]

Uploaded the wrong diff

[dang]

This revision is superseded by D80383

clang/include/clang/Serialization/ASTBitCodes.h

enum OptionsRecordTypes {
       PREPROCESSOR_OPTIONS,
     };
 
     /// Record codes for the unhashed control block.
     enum UnhashedControlBlockRecordTypes {
       /// Record code for the signature that identifiers this AST file.
       SIGNATURE = 1,
 
+      /// Record code for the signature of the AST block.
+      AST_SIGNATURE,
+
       /// Record code for the diagnostic options table.
       DIAGNOSTIC_OPTIONS,
 
       /// Record code for \#pragma diagnostic mappings.
       DIAG_PRAGMA_MAPPINGS,
     };
 
     /// Record code for extension blocks.

clang/include/clang/Serialization/ASTWriter.h

private:
   /// Write the given subexpression to the bitstream.
   void WriteSubStmt(Stmt *S);
 
   void WriteBlockInfoBlock();
   void WriteControlBlock(Preprocessor &PP, ASTContext &Context,
                          StringRef isysroot, const std::string &OutputFile);
 
   /// Write out the signature and diagnostic options, and return the signature.
-  ASTFileSignature writeUnhashedControlBlock(Preprocessor &PP,
-                                             ASTContext &Context);
+  ASTFileSignature
+  writeUnhashedControlBlock(Preprocessor &PP, ASTContext &Context,
+                            const std::pair<uint64_t, uint64_t> ASTBlockRange);
 
   /// Calculate hash of the pcm content.
   static ASTFileSignature createSignature(StringRef Bytes);
 
   void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts,
                        bool Modules);
   void WriteSourceManagerBlock(SourceManager &SourceMgr,
                                const Preprocessor &PP);

clang/include/clang/Serialization/ModuleFile.h

public:
 
   /// The file entry for the module file.
   const FileEntry *File = nullptr;
 
   /// The signature of the module file, which may be used instead of the size
   /// and modification time to identify this particular file.
   ASTFileSignature Signature;
 
+  /// The signature of the AST block of the module file, this can be used to
+  /// unique module files based on AST contents.
+  ASTFileSignature ASTSignature;
+
   /// Whether this module has been directly imported by the
   /// user.
   bool DirectlyImported = false;
 
   /// The generation of which this module file is a part.
   unsigned Generation;
 
   /// The memory buffer that stores the data associated with

clang/lib/Serialization/ASTReader.cpp

void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
   RemapBuilder PreprocessedEntityRemap(F.PreprocessedEntityRemap);
   RemapBuilder SubmoduleRemap(F.SubmoduleRemap);
   RemapBuilder SelectorRemap(F.SelectorRemap);
   RemapBuilder DeclRemap(F.DeclRemap);
   RemapBuilder TypeRemap(F.TypeRemap);
 
   while (Data < DataEnd) {
     // FIXME: Looking up dependency modules by filename is horrible. Let's
-    // start fixing this with prebuilt and explicit modules and see how it
-    // goes...
+    // start fixing this with prebuilt, explicit and implicit modules and see
+    // how it goes...
     using namespace llvm::support;
     ModuleKind Kind = static_cast<ModuleKind>(
       endian::readNext<uint8_t, little, unaligned>(Data));
     uint16_t Len = endian::readNext<uint16_t, little, unaligned>(Data);
     StringRef Name = StringRef((const char*)Data, Len);
     Data += Len;
-    ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule
+    ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule ||
+                              Kind == MK_ImplicitModule
                           ? ModuleMgr.lookupByModuleName(Name)
                           : ModuleMgr.lookupByFileName(Name));
     if (!OM) {
       std::string Msg =
           "SourceLocation remap refers to unknown module, cannot find ";
       Msg.append(std::string(Name));
       Error(Msg);
       return;
     }
 
if (!MaybeRecordType) {
       // FIXME this drops the error.
       return Failure;
     }
     switch ((UnhashedControlBlockRecordTypes)MaybeRecordType.get()) {
     case SIGNATURE:
       if (F)
         std::copy(Record.begin(), Record.end(), F->Signature.data());
       break;
+    case AST_SIGNATURE:
+      if (F)
+        std::copy(Record.begin(), Record.end(), F->ASTSignature.data());
+      break;
     case DIAGNOSTIC_OPTIONS: {
       bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
       if (Listener && ValidateDiagnosticOptions &&
           !AllowCompatibleConfigurationMismatch &&
           ParseDiagnosticOptions(Record, Complain, *Listener))
         Result = OutOfDate; // Don't return early.  Read the signature.
       break;
     }

clang/lib/Serialization/ASTWriter.cpp

 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Lex/Token.h"
 #include "clang/Sema/IdentifierResolver.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Sema.h"
 #include "clang/Sema/Weak.h"
+#include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTRecordWriter.h"
 #include "clang/Serialization/InMemoryModuleCache.h"
 #include "clang/Serialization/ModuleFile.h"
 #include "clang/Serialization/ModuleFileExtension.h"
 #include "clang/Serialization/SerializationDiagnostic.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
#define RECORD(X) EmitRecordID(X, #X, Stream, Record)
   RECORD(PPD_INCLUSION_DIRECTIVE);
 
   // Decls and Types block.
   BLOCK(EXTENSION_BLOCK);
   RECORD(EXTENSION_METADATA);
 
   BLOCK(UNHASHED_CONTROL_BLOCK);
   RECORD(SIGNATURE);
+  RECORD(AST_SIGNATURE);
   RECORD(DIAGNOSTIC_OPTIONS);
   RECORD(DIAG_PRAGMA_MAPPINGS);
 
 #undef RECORD
 #undef BLOCK
   Stream.ExitBlock();
 }
 
ASTFileSignature ASTWriter::createSignature(StringRef Bytes) {
   };
   for (int I = 0; I != 5; ++I)
     Signature[I] = LShift(Hash[I * 4 + 0], 24) | LShift(Hash[I * 4 + 1], 16) |
                    LShift(Hash[I * 4 + 2], 8) | LShift(Hash[I * 4 + 3], 0);
 
   return Signature;
 }
 
-ASTFileSignature ASTWriter::writeUnhashedControlBlock(Preprocessor &PP,
-                                                      ASTContext &Context) {
+ASTFileSignature ASTWriter::writeUnhashedControlBlock(
+    Preprocessor &PP, ASTContext &Context,
+    const std::pair<uint64_t, uint64_t> ASTBlockRange) {
   // Flush first to prepare the PCM hash (signature).
   Stream.FlushToWord();
   auto StartOfUnhashedControl = Stream.GetCurrentBitNo() >> 3;
 
   // Enter the block and prepare to write records.
   RecordData Record;
   Stream.EnterSubblock(UNHASHED_CONTROL_BLOCK_ID, 5);
 
   // For implicit modules, write the hash of the PCM as its signature.
   ASTFileSignature Signature;
   if (WritingModule &&
       PP.getHeaderSearchInfo().getHeaderSearchOpts().ModulesHashContent) {
     Signature = createSignature(StringRef(Buffer.begin(), StartOfUnhashedControl));
     Record.append(Signature.begin(), Signature.end());
     Stream.EmitRecord(SIGNATURE, Record);
     Record.clear();
+    auto ASTSignature = createSignature(
+        StringRef(Buffer.begin() + ASTBlockRange.first, ASTBlockRange.second));
+    Record.append(ASTSignature.begin(), ASTSignature.end());
+    Stream.EmitRecord(AST_SIGNATURE, Record);
+    Record.clear();
   }
 
   // Diagnostic options.
   const auto &Diags = Context.getDiagnostics();
   const DiagnosticOptions &DiagOpts = Diags.getDiagnosticOptions();
 #define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
 #define ENUM_DIAGOPT(Name, Type, Bits, Default)                                \
   Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   } else if (!isysroot.empty()) {
     // Write out paths relative to the sysroot if possible.
     BaseDirectory = std::string(isysroot);
   }
 
   populateInputFileIDs(Context.SourceMgr);
 
   // Write the remaining AST contents.
+  Stream.FlushToWord();
+  auto StartOfASTBlock = Stream.GetCurrentBitNo() >> 3;
   Stream.EnterSubblock(AST_BLOCK_ID, 5);
 
   // This is so that older clang versions, before the introduction
   // of the control block, can read and reject the newer PCH format.
   {
     RecordData Record = {VERSION_MAJOR};
     Stream.EmitRecord(METADATA_OLD_FORMAT, Record);
   }
if (Chain) {
     //   preprocessed-entity-id:i32
     //   macro-definition-id:i32
     //   submodule-id:i32
     //   selector-id:i32
     //   declaration-id:i32
     //   c++-base-specifiers-id:i32
     //   type-id:i32)
     //
-    // module-kind is the ModuleKind enum value. If it is MK_PrebuiltModule or
-    // MK_ExplicitModule, then the module-name is the module name. Otherwise,
-    // it is the module file name.
+    // module-kind is the ModuleKind enum value. If it is MK_PrebuiltModule,
+    // MK_ExplicitModule or MK_ImplicitModule, then the module-name is the
+    // module name. Otherwise, it is the module file name.
     auto Abbrev = std::make_shared<BitCodeAbbrev>();
     Abbrev->Add(BitCodeAbbrevOp(MODULE_OFFSET_MAP));
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     unsigned ModuleOffsetMapAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
     SmallString<2048> Buffer;
     {
       llvm::raw_svector_ostream Out(Buffer);
       for (ModuleFile &M : Chain->ModuleMgr) {
         using namespace llvm::support;
 
         endian::Writer LE(Out, little);
         LE.write<uint8_t>(static_cast<uint8_t>(M.Kind));
-        StringRef Name =
-          M.Kind == MK_PrebuiltModule || M.Kind == MK_ExplicitModule
+        StringRef Name = M.Kind == MK_PrebuiltModule ||
+                                 M.Kind == MK_ExplicitModule ||
+                                 M.Kind == MK_ImplicitModule
                              ? M.ModuleName
                              : M.FileName;
         LE.write<uint16_t>(Name.size());
         Out.write(Name.data(), Name.size());
 
         // Note: if a base ID was uint max, it would not be possible to load
         // another module after it or have more than one entity inside it.
         uint32_t None = std::numeric_limits<uint32_t>::max();
 
         auto writeBaseIDOrNone = [&](uint32_t BaseID, bool ShouldWrite) {
ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   WritePackPragmaOptions(SemaRef);
   WriteFloatControlPragmaOptions(SemaRef);
 
   // Some simple statistics
   RecordData::value_type Record[] = {
       NumStatements, NumMacros, NumLexicalDeclContexts, NumVisibleDeclContexts};
   Stream.EmitRecord(STATISTICS, Record);
   Stream.ExitBlock();
+  auto EndOfASTBlock = Stream.GetCurrentBitNo() >> 3;
 
   // Write the control block
   WriteControlBlock(PP, Context, isysroot, OutputFile);
 
   // Write the module file extension blocks.
   for (const auto &ExtWriter : ModuleFileExtensionWriters)
     WriteModuleFileExtension(SemaRef, *ExtWriter);
 
-  return writeUnhashedControlBlock(PP, Context);
+  return writeUnhashedControlBlock(
+      PP, Context,
+      std::make_pair(StartOfASTBlock, EndOfASTBlock - StartOfASTBlock));
 }
 
 void ASTWriter::WriteDeclUpdatesBlocks(RecordDataImpl &OffsetsRecord) {
   if (DeclUpdates.empty())
     return;
 
   DeclUpdateMap LocalUpdates;
   LocalUpdates.swap(DeclUpdates);

clang/test/Modules/ASTSignature.c

+// RUN: rm -rf %t
+// RUN: %clang_cc1 -iquote %S/Inputs/ASTHash/ -fsyntax-only -fmodules \
+// RUN:   -fimplicit-module-maps -fmodules-strict-context-hash \
+// RUN:   -fmodules-cache-path=%t %s -Rmodule-build 2> %t1
+// RUN: %clang_cc1 -iquote "/dev/null" -iquote %S/Inputs/ASTHash/ -fsyntax-only \
+// RUN:   -fmodules -fimplicit-module-maps -fmodules-strict-context-hash \
+// RUN:   -fmodules-cache-path=%t %s -Rmodule-build 2> %t2
+// RUN: cat %t1 | grep "remark: building module 'MyHeader2'" | cut -d "'" -f 4 \
+// RUN:  | while read -r MODULE; do llvm-bcanalyzer --dump --disable-histogram \
+// RUN:  $MODULE | grep "AST_SIGNATURE" > %t1.sig; done
+// RUN: cat %t2 | grep "remark: building module 'MyHeader2'" | cut -d "'" -f 4 \
+// RUN:  | while read -r MODULE; do llvm-bcanalyzer --dump --disable-histogram \
+// RUN:  $MODULE | grep "AST_SIGNATURE" > %t1.sig; done
+// RUN: diff %t1.sig %t2.sig
+
+#include "my_header_2.h"
+
+my_int var = 42;

clang/test/Modules/Inputs/ASTHash/module.modulemap

+module MyHeader1 {
+  header "my_header_1.h"
+}
+
+module MyHeader2 {
+  header "my_header_2.h"
+  export *
+}

clang/test/Modules/Inputs/ASTHash/my_header_1.h

+typedef int my_int;

clang/test/Modules/Inputs/ASTHash/my_header_2.h

+#include "my_header_1.h"
+
+extern my_int var;