Index: include/clang/AST/ExternalASTSource.h =================================================================== --- include/clang/AST/ExternalASTSource.h +++ include/clang/AST/ExternalASTSource.h @@ -16,6 +16,7 @@ #include "clang/AST/CharUnits.h" #include "clang/AST/DeclBase.h" +#include "clang/Basic/Module.h" #include "llvm/ADT/DenseMap.h" namespace clang { @@ -149,20 +150,20 @@ StringRef PCHModuleName; StringRef Path; StringRef ASTFile; - uint64_t Signature = 0; + ASTFileSignature Signature = {{0}}; const Module *ClangModule = nullptr; public: ASTSourceDescriptor(){}; ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile, - uint64_t Signature) + ASTFileSignature Signature) : PCHModuleName(std::move(Name)), Path(std::move(Path)), ASTFile(std::move(ASTFile)), Signature(Signature){}; ASTSourceDescriptor(const Module &M); std::string getModuleName() const; StringRef getPath() const { return Path; } StringRef getASTFile() const { return ASTFile; } - uint64_t getSignature() const { return Signature; } + ASTFileSignature getSignature() const { return Signature; } const Module *getModuleOrNull() const { return ClangModule; } }; Index: include/clang/Basic/Module.h =================================================================== --- include/clang/Basic/Module.h +++ include/clang/Basic/Module.h @@ -43,6 +43,7 @@ /// \brief Describes the name of a module. typedef SmallVector, 2> ModuleId; +typedef std::array ASTFileSignature; /// \brief Describes a module or submodule. class Module { public: @@ -65,7 +66,7 @@ llvm::PointerUnion Umbrella; /// \brief The module signature. - uint64_t Signature; + ASTFileSignature Signature; /// \brief The name of the umbrella entry, as written in the module map. std::string UmbrellaAsWritten; Index: include/clang/Driver/CC1Options.td =================================================================== --- include/clang/Driver/CC1Options.td +++ include/clang/Driver/CC1Options.td @@ -658,6 +658,8 @@ HelpText<"Disable standard system #include directories">; def fdisable_module_hash : Flag<["-"], "fdisable-module-hash">, HelpText<"Disable the module hash">; +def fmodules_hash_content : Flag<["-"], "fmodules-hash-content">, + HelpText<"Enable hashing the content of a module file">; def c_isystem : JoinedOrSeparate<["-"], "c-isystem">, MetaVarName<"">, HelpText<"Add directory to the C SYSTEM include search path">; def objc_isystem : JoinedOrSeparate<["-"], "objc-isystem">, Index: include/clang/Frontend/PCHContainerOperations.h =================================================================== --- include/clang/Frontend/PCHContainerOperations.h +++ include/clang/Frontend/PCHContainerOperations.h @@ -10,6 +10,7 @@ #ifndef LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H #define LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H +#include "clang/Basic/Module.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/MemoryBuffer.h" @@ -29,7 +30,7 @@ class CompilerInstance; struct PCHBuffer { - uint64_t Signature; + ASTFileSignature Signature; llvm::SmallVector Data; bool IsComplete; }; Index: include/clang/Lex/HeaderSearchOptions.h =================================================================== --- include/clang/Lex/HeaderSearchOptions.h +++ include/clang/Lex/HeaderSearchOptions.h @@ -178,6 +178,8 @@ unsigned ModulesValidateDiagnosticOptions : 1; + unsigned ModulesHashContent : 1; + HeaderSearchOptions(StringRef _Sysroot = "/") : Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(0), ImplicitModuleMaps(0), ModuleMapFileHomeIsCwd(0), @@ -187,7 +189,8 @@ UseStandardCXXIncludes(true), UseLibcxx(false), Verbose(false), ModulesValidateOncePerBuildSession(false), ModulesValidateSystemHeaders(false), - UseDebugInfo(false), ModulesValidateDiagnosticOptions(true) {} + UseDebugInfo(false), ModulesValidateDiagnosticOptions(true), + ModulesHashContent(false) {} /// AddPath - Add the \p Path path to the specified \p Group list. void AddPath(StringRef Path, frontend::IncludeDirGroup Group, Index: include/clang/Serialization/ASTBitCodes.h =================================================================== --- include/clang/Serialization/ASTBitCodes.h +++ include/clang/Serialization/ASTBitCodes.h @@ -253,6 +253,10 @@ /// \brief A block containing a module file extension. EXTENSION_BLOCK_ID, + + /// \brief A block containing unhashed contents. It currently holds + /// Diagnostic Options and Signature. + UNHASHED_CONTROL_BLOCK_ID, }; /// \brief Record types that occur within the control block. @@ -288,9 +292,6 @@ /// AST file. MODULE_MAP_FILE, - /// \brief Record code for the signature that identifiers this AST file. - SIGNATURE, - /// \brief Record code for the module build directory. MODULE_DIRECTORY, }; @@ -309,9 +310,6 @@ /// \brief Record code for the target options table. TARGET_OPTIONS, - /// \brief Record code for the diagnostic options table. - DIAGNOSTIC_OPTIONS, - /// \brief Record code for the filesystem options table. FILE_SYSTEM_OPTIONS, @@ -322,7 +320,16 @@ PREPROCESSOR_OPTIONS, }; - /// \brief Record code for extension blocks. + /// \brief Record code for the unhashed control block. + enum UnhashedControlBlockRecordTypes { + /// \brief Record code for the diagnostic options table. + DIAGNOSTIC_OPTIONS = 1, + + /// \brief Record code for the signature that identifiers this AST file. + SIGNATURE, + }; + + /// \brief Record code for extension blocks. enum ExtensionBlockRecordTypes { /// Metadata describing this particular extension. EXTENSION_METADATA = 1, Index: include/clang/Serialization/ASTReader.h =================================================================== --- include/clang/Serialization/ASTReader.h +++ include/clang/Serialization/ASTReader.h @@ -1174,7 +1174,7 @@ SourceLocation ImportLoc, ModuleFile *ImportedBy, SmallVectorImpl &Loaded, off_t ExpectedSize, time_t ExpectedModTime, - serialization::ASTFileSignature ExpectedSignature, + ASTFileSignature ExpectedSignature, unsigned ClientLoadCapabilities); ASTReadResult ReadControlBlock(ModuleFile &F, SmallVectorImpl &Loaded, @@ -1183,7 +1183,11 @@ static ASTReadResult ReadOptionsBlock( llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities, bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, - std::string &SuggestedPredefines, bool ValidateDiagnosticOptions); + std::string &SuggestedPredefines); + static ASTReadResult ReadDiagnosticOptionsBlock(ModuleFile *F, + llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities, + bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, + bool ValidateDiagnosticOptions); ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities); ASTReadResult ReadExtensionBlock(ModuleFile &F); bool ParseLineTable(ModuleFile &F, const RecordData &Record); Index: include/clang/Serialization/ASTWriter.h =================================================================== --- include/clang/Serialization/ASTWriter.h +++ include/clang/Serialization/ASTWriter.h @@ -106,6 +106,9 @@ /// \brief The bitstream writer used to emit this precompiled header. llvm::BitstreamWriter &Stream; + /// \brief The buffer associated with the bitstream. + const SmallVectorImpl &Buffer; + /// \brief The ASTContext we're writing. ASTContext *Context = nullptr; @@ -417,6 +420,9 @@ std::vector> ModuleFileExtensionWriters; + /// \brief Bit position for start of UNHASHED_CONTROL_BLOCK. + size_t StartOfUnhashedControl; + /// \brief Retrieve or create a submodule ID for this module. unsigned getSubmoduleID(Module *Mod); @@ -424,8 +430,12 @@ void WriteSubStmt(Stmt *S); void WriteBlockInfoBlock(); - uint64_t WriteControlBlock(Preprocessor &PP, ASTContext &Context, - StringRef isysroot, const std::string &OutputFile); + void WriteControlBlock(Preprocessor &PP, ASTContext &Context, + StringRef isysroot, const std::string &OutputFile); + ASTFileSignature WriteUnhashedControlBlock(Preprocessor &PP, + ASTContext &Context); + /// \brief Calculate hash of the pcm content and write it to SIGNATURE record. + ASTFileSignature WriteHash(size_t BlockStartPos); void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts, bool Modules); void WriteSourceManagerBlock(SourceManager &SourceMgr, @@ -492,14 +502,14 @@ void WriteDeclAbbrevs(); void WriteDecl(ASTContext &Context, Decl *D); - uint64_t WriteASTCore(Sema &SemaRef, - StringRef isysroot, const std::string &OutputFile, - Module *WritingModule); + ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot, + const std::string &OutputFile, + Module *WritingModule); public: /// \brief Create a new precompiled header writer that outputs to /// the given bitstream. - ASTWriter(llvm::BitstreamWriter &Stream, + ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, ArrayRef> Extensions, bool IncludeTimestamps = true); ~ASTWriter() override; @@ -525,9 +535,9 @@ /// /// \return the module signature, which eventually will be a hash of /// the module but currently is merely a random 32-bit number. - uint64_t WriteAST(Sema &SemaRef, const std::string &OutputFile, - Module *WritingModule, StringRef isysroot, - bool hasErrors = false); + ASTFileSignature WriteAST(Sema &SemaRef, const std::string &OutputFile, + Module *WritingModule, StringRef isysroot, + bool hasErrors = false); /// \brief Emit a token. void AddToken(const Token &Tok, RecordDataImpl &Record); Index: include/clang/Serialization/Module.h =================================================================== --- include/clang/Serialization/Module.h +++ include/clang/Serialization/Module.h @@ -16,6 +16,7 @@ #define LLVM_CLANG_SERIALIZATION_MODULE_H #include "clang/Basic/FileManager.h" +#include "clang/Basic/Module.h" #include "clang/Basic/SourceLocation.h" #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" @@ -89,8 +90,6 @@ bool isNotFound() const { return Val.getInt() == NotFound; } }; -typedef unsigned ASTFileSignature; - /// \brief Information about a module that has been loaded by the ASTReader. /// /// Each instance of the Module class corresponds to a single AST file, which Index: lib/Basic/Module.cpp =================================================================== --- lib/Basic/Module.cpp +++ lib/Basic/Module.cpp @@ -27,7 +27,7 @@ Module::Module(StringRef Name, SourceLocation DefinitionLoc, Module *Parent, bool IsFramework, bool IsExplicit, unsigned VisibilityID) : Name(Name), DefinitionLoc(DefinitionLoc), Parent(Parent), Directory(), - Umbrella(), Signature(0), ASTFile(nullptr), VisibilityID(VisibilityID), + Umbrella(), Signature({{0}}), ASTFile(nullptr), VisibilityID(VisibilityID), IsMissingRequirement(false), HasIncompatibleModuleFile(false), IsAvailable(true), IsFromModuleFile(false), IsFramework(IsFramework), IsExplicit(IsExplicit), IsSystem(false), IsExternC(false), Index: lib/CodeGen/CGDebugInfo.cpp =================================================================== --- lib/CodeGen/CGDebugInfo.cpp +++ lib/CodeGen/CGDebugInfo.cpp @@ -2009,7 +2009,9 @@ if (CreateSkeletonCU && IsRootModule) { // PCH files don't have a signature field in the control block, // but LLVM detects skeleton CUs by looking for a non-zero DWO id. - uint64_t Signature = Mod.getSignature() ? Mod.getSignature() : ~1ULL; + // We use the lower 64 bits for debug info. + uint64_t Signature = Mod.getSignature() != ASTFileSignature({{0}}) ? + (uint64_t)Mod.getSignature()[1] << 32 | Mod.getSignature()[0] : ~1ULL; llvm::DIBuilder DIB(CGM.getModule()); DIB.createCompileUnit(TheCU->getSourceLanguage(), DIB.createFile(Mod.getModuleName(), Mod.getPath()), Index: lib/CodeGen/ObjectFilePCHContainerOperations.cpp =================================================================== --- lib/CodeGen/ObjectFilePCHContainerOperations.cpp +++ lib/CodeGen/ObjectFilePCHContainerOperations.cpp @@ -171,7 +171,8 @@ // Prepare CGDebugInfo to emit debug info for a clang module. auto *DI = Builder->getModuleDebugInfo(); StringRef ModuleName = llvm::sys::path::filename(MainFileName); - DI->setPCHDescriptor({ModuleName, "", OutputFileName, ~1ULL}); + DI->setPCHDescriptor({ModuleName, "", OutputFileName, + ASTFileSignature{{~0U, ~0U, ~0U, ~0U, ~1U}} }); DI->setModuleMap(MMap); } @@ -241,7 +242,9 @@ // PCH files don't have a signature field in the control block, // but LLVM detects DWO CUs by looking for a non-zero DWO id. - uint64_t Signature = Buffer->Signature ? Buffer->Signature : ~1ULL; + // We use the lower 64 bits for debug info. + uint64_t Signature = Buffer->Signature != ASTFileSignature({{0}}) ? + (uint64_t)Buffer->Signature[1] << 32 | Buffer->Signature[0] : ~1ULL; Builder->getModuleDebugInfo()->setDwoId(Signature); // Finalize the Builder. Index: lib/Frontend/ASTUnit.cpp =================================================================== --- lib/Frontend/ASTUnit.cpp +++ lib/Frontend/ASTUnit.cpp @@ -185,7 +185,7 @@ llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } + ASTWriterData() : Stream(Buffer), Writer(Stream, Buffer, { }) { } }; void ASTUnit::clearFileLevelDecls() { @@ -2516,7 +2516,7 @@ SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream, { }); + ASTWriter Writer(Stream, Buffer, { }); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } Index: lib/Frontend/CompilerInstance.cpp =================================================================== --- lib/Frontend/CompilerInstance.cpp +++ lib/Frontend/CompilerInstance.cpp @@ -1032,7 +1032,7 @@ // Remove any macro definitions that are explicitly ignored by the module. // They aren't supposed to affect how the module is built anyway. - const HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts(); + HeaderSearchOptions &HSOpts = Invocation->getHeaderSearchOpts(); PPOpts.Macros.erase( std::remove_if(PPOpts.Macros.begin(), PPOpts.Macros.end(), [&HSOpts](const std::pair &def) { @@ -1063,6 +1063,8 @@ FrontendOpts.DisableFree = false; FrontendOpts.GenerateGlobalModuleIndex = false; FrontendOpts.BuildingImplicitModule = true; + // Force implicitly-built modules to hash the content of the module file. + HSOpts.ModulesHashContent = true; FrontendOpts.Inputs.clear(); InputKind IK = getSourceInputKindFromOptions(*Invocation->getLangOpts()); Index: lib/Frontend/CompilerInvocation.cpp =================================================================== --- lib/Frontend/CompilerInvocation.cpp +++ lib/Frontend/CompilerInvocation.cpp @@ -1422,6 +1422,7 @@ for (const Arg *A : Args.filtered(OPT_fprebuilt_module_path)) Opts.AddPrebuiltModulePath(A->getValue()); Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash); + Opts.ModulesHashContent = Args.hasArg(OPT_fmodules_hash_content); Opts.ModulesValidateDiagnosticOptions = !Args.hasArg(OPT_fmodules_disable_diagnostic_validation); Opts.ImplicitModuleMaps = Args.hasArg(OPT_fimplicit_module_maps); Index: lib/Serialization/ASTReader.cpp =================================================================== --- lib/Serialization/ASTReader.cpp +++ lib/Serialization/ASTReader.cpp @@ -2146,10 +2146,59 @@ llvm_unreachable("unknown ASTReadResult"); } +ASTReader::ASTReadResult ASTReader::ReadDiagnosticOptionsBlock( + ModuleFile *F, BitstreamCursor &Stream, unsigned ClientLoadCapabilities, + bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, + bool ValidateDiagnosticOptions) { + // Read all of the records in the options block. + RecordData Record; + ASTReadResult Result = Success; + while (1) { + llvm::BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + case llvm::BitstreamEntry::SubBlock: + return Failure; + + case llvm::BitstreamEntry::EndBlock: + return Result; + + case llvm::BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read and process a record. + Record.clear(); + switch ((UnhashedControlBlockRecordTypes)Stream.readRecord(Entry.ID, Record)) { + case DIAGNOSTIC_OPTIONS: { + bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0; + if (ValidateDiagnosticOptions && + !AllowCompatibleConfigurationMismatch && + ParseDiagnosticOptions(Record, Complain, Listener)) + return OutOfDate; + break; + } + case SIGNATURE: { + if (!F) + break; + int Pos = 0; + F->Signature[Pos++] = Record[0]; + F->Signature[Pos++] = Record[1]; + F->Signature[Pos++] = Record[2]; + F->Signature[Pos++] = Record[3]; + F->Signature[Pos++] = Record[4]; + break; + } + } + } +} + ASTReader::ASTReadResult ASTReader::ReadOptionsBlock( BitstreamCursor &Stream, unsigned ClientLoadCapabilities, bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, - std::string &SuggestedPredefines, bool ValidateDiagnosticOptions) { + std::string &SuggestedPredefines) { if (Stream.EnterSubBlock(OPTIONS_BLOCK_ID)) return Failure; @@ -2191,15 +2240,6 @@ break; } - case DIAGNOSTIC_OPTIONS: { - bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0; - if (ValidateDiagnosticOptions && - !AllowCompatibleConfigurationMismatch && - ParseDiagnosticOptions(Record, Complain, Listener)) - return OutOfDate; - break; - } - case FILE_SYSTEM_OPTIONS: { bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0; if (!AllowCompatibleConfigurationMismatch && @@ -2323,13 +2363,10 @@ // FIXME: Allow this for files explicitly specified with -include-pch. bool AllowCompatibleConfigurationMismatch = F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule; - const HeaderSearchOptions &HSOpts = - PP.getHeaderSearchInfo().getHeaderSearchOpts(); Result = ReadOptionsBlock(Stream, ClientLoadCapabilities, AllowCompatibleConfigurationMismatch, - *Listener, SuggestedPredefines, - HSOpts.ModulesValidateDiagnosticOptions); + *Listener, SuggestedPredefines); if (Result == Failure) { Error("malformed block record in AST file"); return Result; @@ -2403,11 +2440,6 @@ break; } - case SIGNATURE: - assert((!F.Signature || F.Signature == Record[0]) && "signature changed"); - F.Signature = Record[0]; - break; - case IMPORTS: { // Load each of the imported PCH files. unsigned Idx = 0, N = Record.size(); @@ -2421,7 +2453,10 @@ ReadUntranslatedSourceLocation(Record[Idx++]); off_t StoredSize = (off_t)Record[Idx++]; time_t StoredModTime = (time_t)Record[Idx++]; - ASTFileSignature StoredSignature = Record[Idx++]; + ASTFileSignature StoredSignature = + {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++]}}; auto ImportedFile = ReadPath(F, Record, Idx); // If our client can't cope with us being out of date, we can't cope with @@ -3623,7 +3658,7 @@ SmallVector Loaded; switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type, ImportLoc, /*ImportedBy=*/nullptr, Loaded, - 0, 0, 0, + 0, 0, {{0}}, ClientLoadCapabilities)) { case Failure: case Missing: @@ -3932,6 +3967,7 @@ // This is used for compatibility with older PCH formats. bool HaveReadControlBlock = false; + BitstreamCursor ASTCursor; while (true) { llvm::BitstreamEntry Entry = Stream.advance(); @@ -3982,10 +4018,48 @@ return VersionMismatch; } - // Record that we've loaded this module. - Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc)); - return Success; + ASTCursor = Stream; + if (Stream.SkipBlock()) { + Error("malformed block record in AST file"); + return Failure; + } + break; + case UNHASHED_CONTROL_BLOCK_ID: { + if (Stream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID)) { + Error("malformed block record in AST file"); + return Failure; + } + + ASTReadResult Result = Success; + if (Listener && !ImportedBy) { + const HeaderSearchOptions &HSOpts = + PP.getHeaderSearchInfo().getHeaderSearchOpts(); + bool AllowCompatibleConfigurationMismatch = + F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule; + Result = ReadDiagnosticOptionsBlock(&F, Stream, ClientLoadCapabilities, + AllowCompatibleConfigurationMismatch, + *Listener, + HSOpts.ModulesValidateDiagnosticOptions); + + if (Result == Failure) { + Error("malformed block record in AST file"); + return Failure; + } + + if (DisableValidation || + (AllowConfigurationMismatch && Result == ConfigurationMismatch)) + Result = Success; + } + + if (Result == Success) + // Record that we've loaded this module. + Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc)); + + // Move Stream backward to point to ASTCursor. + Stream = ASTCursor; + return Result; + } default: if (Stream.SkipBlock()) { Error("malformed block record in AST file"); @@ -4214,23 +4288,24 @@ static ASTFileSignature readASTFileSignature(StringRef PCH) { BitstreamCursor Stream(PCH); if (!startsWithASTFileMagic(Stream)) - return 0; + return {{0}}; - // Scan for the CONTROL_BLOCK_ID block. - if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID)) - return 0; + // Scan for the UNHASHED_CONTROL_BLOCK_ID block. + if (SkipCursorToBlock(Stream, UNHASHED_CONTROL_BLOCK_ID)) + return {{0}}; - // Scan for SIGNATURE inside the control block. + // Scan for SIGNATURE inside the diagnostic options block. ASTReader::RecordData Record; while (true) { llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); if (Entry.Kind != llvm::BitstreamEntry::Record) - return 0; + return {{0}}; Record.clear(); StringRef Blob; if (SIGNATURE == Stream.readRecord(Entry.ID, Record, &Blob)) - return Record[0]; + return {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2], + (uint32_t)Record[3], (uint32_t)Record[4]}}; } } @@ -4377,8 +4452,7 @@ std::string IgnoredSuggestedPredefines; if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate, /*AllowCompatibleConfigurationMismatch*/ false, - Listener, IgnoredSuggestedPredefines, - ValidateDiagnosticOptions) != Success) + Listener, IgnoredSuggestedPredefines) != Success) return true; break; } @@ -4499,6 +4573,7 @@ // Look for module file extension blocks, if requested. if (FindModuleFileExtensions) { + BitstreamCursor SavedStream = Stream; while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) { bool DoneWithExtensionBlock = false; while (!DoneWithExtensionBlock) { @@ -4537,8 +4612,18 @@ } } } + Stream = SavedStream; } + // Scan for the UNHASHED_CONTROL_BLOCK_ID block. + if (SkipCursorToBlock(Stream, UNHASHED_CONTROL_BLOCK_ID)) + return true; + if (ReadDiagnosticOptionsBlock(nullptr, Stream, + ARR_ConfigurationMismatch | ARR_OutOfDate, + /*AllowCompatibleConfigurationMismatch*/ false, + Listener, ValidateDiagnosticOptions) != Success) + return true; + return false; } Index: lib/Serialization/ASTWriter.cpp =================================================================== --- lib/Serialization/ASTWriter.cpp +++ lib/Serialization/ASTWriter.cpp @@ -79,6 +79,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/SHA1.h" #include #include #include @@ -1000,7 +1001,6 @@ // Control Block. BLOCK(CONTROL_BLOCK); RECORD(METADATA); - RECORD(SIGNATURE); RECORD(MODULE_NAME); RECORD(MODULE_DIRECTORY); RECORD(MODULE_MAP_FILE); @@ -1013,7 +1013,6 @@ BLOCK(OPTIONS_BLOCK); RECORD(LANGUAGE_OPTIONS); RECORD(TARGET_OPTIONS); - RECORD(DIAGNOSTIC_OPTIONS); RECORD(FILE_SYSTEM_OPTIONS); RECORD(HEADER_SEARCH_OPTIONS); RECORD(PREPROCESSOR_OPTIONS); @@ -1241,6 +1240,10 @@ BLOCK(EXTENSION_BLOCK); RECORD(EXTENSION_METADATA); + BLOCK(UNHASHED_CONTROL_BLOCK); + RECORD(DIAGNOSTIC_OPTIONS); + RECORD(SIGNATURE); + #undef RECORD #undef BLOCK Stream.ExitBlock(); @@ -1303,21 +1306,71 @@ return Filename + Pos; } -static ASTFileSignature getSignature() { - while (true) { - if (ASTFileSignature S = llvm::sys::Process::GetRandomNumber()) - return S; - // Rely on GetRandomNumber to eventually return non-zero... +ASTFileSignature ASTWriter::WriteHash(size_t BlockStartPos) { + // Calculate the hash till start of UNHASHED_CONTROL_BLOCK. + llvm::SHA1 Hasher; + Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], + (StartOfUnhashedControl>>3) - BlockStartPos)); + auto Hash = Hasher.result(); + + SmallVector Vals; + auto LShift = [&](unsigned char Val, unsigned Amount) + -> uint64_t { return ((uint64_t)Val) << Amount; }; + for (int Pos = 0; Pos < 20; Pos += 4) { + uint32_t SubHash = LShift(Hash[Pos + 0], 24); + SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | + (unsigned)(unsigned char)Hash[Pos + 3]; + Vals.push_back(SubHash); } + + // Emit the finished record. SIGNATURE: [5*i32] + Stream.EmitRecord(SIGNATURE, Vals); + return {{(uint32_t)Vals[0], (uint32_t)Vals[1], (uint32_t)Vals[2], + (uint32_t)Vals[3], (uint32_t)Vals[4]}}; } -/// \brief Write the control block. -uint64_t ASTWriter::WriteControlBlock(Preprocessor &PP, - ASTContext &Context, - StringRef isysroot, - const std::string &OutputFile) { - ASTFileSignature Signature = 0; +ASTFileSignature ASTWriter::WriteUnhashedControlBlock(Preprocessor &PP, + ASTContext &Context) { + StartOfUnhashedControl = Stream.GetCurrentBitNo(); + // Write the unhashed control block. + Stream.EnterSubblock(UNHASHED_CONTROL_BLOCK_ID, 5); + + // Diagnostic options. + RecordData Record; + const DiagnosticOptions &DiagOpts + = Context.getDiagnostics().getDiagnosticOptions(); +#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name); +#define ENUM_DIAGOPT(Name, Type, Bits, Default) \ + Record.push_back(static_cast(DiagOpts.get##Name())); +#include "clang/Basic/DiagnosticOptions.def" + Record.push_back(DiagOpts.Warnings.size()); + for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I) + AddString(DiagOpts.Warnings[I], Record); + Record.push_back(DiagOpts.Remarks.size()); + for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I) + AddString(DiagOpts.Remarks[I], Record); + // Note: we don't serialize the log or serialization file names, because they + // are generally transient files and will almost always be overridden. + Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record); + + // For implicit modules we output a signature that is the hash of the pcm + // content. + ASTFileSignature RetCode{{0}}; + const HeaderSearchOptions &HSOpts + = PP.getHeaderSearchInfo().getHeaderSearchOpts(); + if (WritingModule && HSOpts.ModulesHashContent) + RetCode = WriteHash(0); + // Leave the options block. + Stream.ExitBlock(); + return RetCode; +} + +/// \brief Write the control block. +void ASTWriter::WriteControlBlock(Preprocessor &PP, + ASTContext &Context, + StringRef isysroot, + const std::string &OutputFile) { using namespace llvm; Stream.EnterSubblock(CONTROL_BLOCK_ID, 5); RecordData Record; @@ -1345,15 +1398,6 @@ getClangFullRepositoryVersion()); } if (WritingModule) { - // For implicit modules we output a signature that we can use to ensure - // duplicate module builds don't collide in the cache as their output order - // is non-deterministic. - // FIXME: Remove this when output is deterministic. - if (Context.getLangOpts().ImplicitModules) { - Signature = getSignature(); - RecordData::value_type Record[] = {Signature}; - Stream.EmitRecord(SIGNATURE, Record); - } // Module name auto Abbrev = std::make_shared(); @@ -1426,9 +1470,18 @@ Record.push_back((unsigned)M->Kind); // FIXME: Stable encoding AddSourceLocation(M->ImportLoc, Record); - Record.push_back(M->File->getSize()); - Record.push_back(getTimestampForOutput(M->File)); - Record.push_back(M->Signature); + + // If we have calculated signature, there is no need to store + // the size or timestamp. + bool HasSignature = M->Signature != ASTFileSignature({{0}}); + Record.push_back(HasSignature ? 0 : M->File->getSize()); + Record.push_back(HasSignature ? 0 : getTimestampForOutput(M->File)); + + Record.push_back(M->Signature[0]); + Record.push_back(M->Signature[1]); + Record.push_back(M->Signature[2]); + Record.push_back(M->Signature[3]); + Record.push_back(M->Signature[4]); AddPath(M->FileName, Record); } Stream.EmitRecord(IMPORTS, Record); @@ -1491,24 +1544,6 @@ } Stream.EmitRecord(TARGET_OPTIONS, Record); - // Diagnostic options. - Record.clear(); - const DiagnosticOptions &DiagOpts - = Context.getDiagnostics().getDiagnosticOptions(); -#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name); -#define ENUM_DIAGOPT(Name, Type, Bits, Default) \ - Record.push_back(static_cast(DiagOpts.get##Name())); -#include "clang/Basic/DiagnosticOptions.def" - Record.push_back(DiagOpts.Warnings.size()); - for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I) - AddString(DiagOpts.Warnings[I], Record); - Record.push_back(DiagOpts.Remarks.size()); - for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I) - AddString(DiagOpts.Remarks[I], Record); - // Note: we don't serialize the log or serialization file names, because they - // are generally transient files and will almost always be overridden. - Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record); - // File system options. Record.clear(); const FileSystemOptions &FSOpts = @@ -1622,7 +1657,6 @@ PP.getHeaderSearchInfo().getHeaderSearchOpts(), PP.getLangOpts().Modules); Stream.ExitBlock(); - return Signature; } namespace { @@ -4221,10 +4255,10 @@ SelectorOffsets[ID - FirstSelectorID] = Offset; } -ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, +ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, ArrayRef> Extensions, bool IncludeTimestamps) - : Stream(Stream), IncludeTimestamps(IncludeTimestamps) { + : Stream(Stream), Buffer(Buffer), IncludeTimestamps(IncludeTimestamps) { for (const auto &Ext : Extensions) { if (auto Writer = Ext->createExtensionWriter(*this)) ModuleFileExtensionWriters.push_back(std::move(Writer)); @@ -4244,7 +4278,7 @@ return IncludeTimestamps ? E->getModificationTime() : 0; } -uint64_t ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile, +ASTFileSignature ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile, Module *WritingModule, StringRef isysroot, bool hasErrors) { WritingAST = true; @@ -4282,7 +4316,7 @@ } } -uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, +ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, const std::string &OutputFile, Module *WritingModule) { using namespace llvm; @@ -4432,7 +4466,7 @@ } // Write the control block - uint64_t Signature = WriteControlBlock(PP, Context, isysroot, OutputFile); + WriteControlBlock(PP, Context, isysroot, OutputFile); // Write the remaining AST contents. Stream.EnterSubblock(AST_BLOCK_ID, 5); @@ -4775,6 +4809,8 @@ for (const auto &ExtWriter : ModuleFileExtensionWriters) WriteModuleFileExtension(SemaRef, *ExtWriter); + ASTFileSignature Signature = WriteUnhashedControlBlock(PP, Context); + return Signature; } Index: lib/Serialization/GeneratePCH.cpp =================================================================== --- lib/Serialization/GeneratePCH.cpp +++ lib/Serialization/GeneratePCH.cpp @@ -28,7 +28,7 @@ bool AllowASTWithErrors, bool IncludeTimestamps) : PP(PP), OutputFile(OutputFile), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data), - Writer(Stream, Extensions, IncludeTimestamps), + Writer(Stream, Buffer->Data, Extensions, IncludeTimestamps), AllowASTWithErrors(AllowASTWithErrors) { Buffer->IsComplete = false; } Index: lib/Serialization/GlobalModuleIndex.cpp =================================================================== --- lib/Serialization/GlobalModuleIndex.cpp +++ lib/Serialization/GlobalModuleIndex.cpp @@ -376,6 +376,16 @@ /// \brief The set of modules on which this module depends. Each entry is /// a module ID. SmallVector Dependencies; + ASTFileSignature Signature; + }; + + struct ImportedModuleFileInfo { + off_t StoredSize; + time_t StoredModTime; + ASTFileSignature StoredSignature; + ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig) : + StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) { + } }; /// \brief Builder that generates the global module index file. @@ -389,6 +399,14 @@ /// \brief Information about each of the known module files. ModuleFilesMap ModuleFiles; + /// \brief Mapping from the imported module file to the imported + /// information. + typedef std::multimap + ImportedModuleFilesMap; + + /// \brief Information about each importing of a module file. + ImportedModuleFilesMap ImportedModuleFiles; + /// \brief Mapping from identifiers to the list of module file IDs that /// consider this identifier to be interesting. typedef llvm::StringMap > InterestingIdentifierMap; @@ -410,6 +428,7 @@ unsigned NewID = ModuleFiles.size(); ModuleFileInfo &Info = ModuleFiles[File]; Info.ID = NewID; + Info.Signature = {{0}}; return Info; } @@ -424,7 +443,8 @@ bool loadModuleFile(const FileEntry *File); /// \brief Write the index to the given bitstream. - void writeIndex(llvm::BitstreamWriter &Stream); + /// \returns true if an error occurred, false otherwise. + bool writeIndex(llvm::BitstreamWriter &Stream); }; } @@ -515,7 +535,7 @@ unsigned ID = getModuleFileInfo(File).ID; // Search for the blocks and records we care about. - enum { Other, ControlBlock, ASTBlock } State = Other; + enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other; bool Done = false; while (!Done) { llvm::BitstreamEntry Entry = InStream.advance(); @@ -553,6 +573,15 @@ continue; } + if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) { + if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID)) + return true; + + // Found the Diagnostic Options block. + State = DiagnosticOptionsBlock; + continue; + } + if (InStream.SkipBlock()) return true; @@ -587,7 +616,10 @@ // Skip the stored signature. // FIXME: we could read the signature out of the import and validate it. - Idx++; + ASTFileSignature StoredSignature = + {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++]}}; // Retrieve the imported file name. unsigned Length = Record[Idx++]; @@ -599,11 +631,15 @@ const FileEntry *DependsOnFile = FileMgr.getFile(ImportedFile, /*openFile=*/false, /*cacheFailure=*/false); - if (!DependsOnFile || - (StoredSize != DependsOnFile->getSize()) || - (StoredModTime != DependsOnFile->getModificationTime())) + + if (!DependsOnFile) return true; + // Save the information in ImportedModuleFileInfo so we can verify after + // loading all pcms. + ImportedModuleFiles.insert(std::make_pair(DependsOnFile, + ImportedModuleFileInfo(StoredSize, StoredModTime, StoredSignature))); + // Record the dependency. unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID; getModuleFileInfo(File).Dependencies.push_back(DependsOnID); @@ -632,6 +668,12 @@ } } + // Get Signature. + if (State == DiagnosticOptionsBlock && Code == SIGNATURE) + getModuleFileInfo(File).Signature = {{(uint32_t)Record[0], + (uint32_t)Record[1], (uint32_t)Record[2], + (uint32_t)Record[3], (uint32_t)Record[4]}}; + // We don't care about this record. } @@ -680,7 +722,21 @@ } -void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { +bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { + for (auto MapEntry : ImportedModuleFiles) { + auto *File = MapEntry.first; + ImportedModuleFileInfo &Info = MapEntry.second; + if (getModuleFileInfo(File).Signature != ASTFileSignature({{0}})) { + if (getModuleFileInfo(File).Signature != Info.StoredSignature) + // Verify Signature. + return true; + } + else if (Info.StoredSize != File->getSize() || + Info.StoredModTime != File->getModificationTime()) + // Verify Size and ModTime. + return true; + } + using namespace llvm; // Emit the file header. @@ -756,6 +812,7 @@ } Stream.ExitBlock(); + return false; } GlobalModuleIndex::ErrorCode @@ -816,7 +873,8 @@ SmallVector OutputBuffer; { llvm::BitstreamWriter OutputStream(OutputBuffer); - Builder.writeIndex(OutputStream); + if (Builder.writeIndex(OutputStream)) + return EC_IOError; } // Write the global index file to a temporary file. Index: lib/Serialization/Module.cpp =================================================================== --- lib/Serialization/Module.cpp +++ lib/Serialization/Module.cpp @@ -20,7 +20,7 @@ using namespace reader; ModuleFile::ModuleFile(ModuleKind Kind, unsigned Generation) - : Kind(Kind), File(nullptr), Signature(0), DirectlyImported(false), + : Kind(Kind), File(nullptr), Signature({{0}}), DirectlyImported(false), Generation(Generation), SizeInBits(0), LocalNumSLocEntries(0), SLocEntryBaseID(0), SLocEntryBaseOffset(0), SLocEntryOffsets(nullptr), Index: lib/Serialization/ModuleManager.cpp =================================================================== --- lib/Serialization/ModuleManager.cpp +++ lib/Serialization/ModuleManager.cpp @@ -138,15 +138,15 @@ ModuleEntry->Data = PCHContainerRdr.ExtractPCH(*ModuleEntry->Buffer); } - if (ExpectedSignature) { + if (ExpectedSignature != ASTFileSignature({{0}})) { // If we've not read the control block yet, read the signature eagerly now // so that we can check it. - if (!ModuleEntry->Signature) + if (ModuleEntry->Signature == ASTFileSignature({{0}})) ModuleEntry->Signature = ReadSignature(ModuleEntry->Data); if (ModuleEntry->Signature != ExpectedSignature) { - ErrorStr = ModuleEntry->Signature ? "signature mismatch" - : "could not read module signature"; + ErrorStr = ModuleEntry->Signature != ASTFileSignature({{0}}) ? + "signature mismatch" : "could not read module signature"; if (NewModule) delete ModuleEntry; Index: test/Modules/diagnostic-options-out-of-date.m =================================================================== --- test/Modules/diagnostic-options-out-of-date.m +++ test/Modules/diagnostic-options-out-of-date.m @@ -7,6 +7,16 @@ // RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s -fmodules-disable-diagnostic-validation // Make sure we don't error out when using the pch // RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify -fmodules-disable-diagnostic-validation + +// Build A.pcm +// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s +// Build pch that imports A.pcm +// RUN: %clang_cc1 -Werror -Wno-conversion -emit-pch -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -o %t.pch -I %S/Inputs -x objective-c-header %S/Inputs/pch-import-module-out-of-date.pch +// We will rebuild A.pcm and overwrite the original A.pcm that the pch imports, but the two versions have the same hash. +// RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s +// Make sure we don't error out when using the pch +// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify + // expected-no-diagnostics @import DiagOutOfDate; Index: test/Modules/module_file_info.m =================================================================== --- test/Modules/module_file_info.m +++ test/Modules/module_file_info.m @@ -29,11 +29,6 @@ // CHECK: CPU: // CHECK: ABI: -// CHECK: Diagnostic options: -// CHECK: IgnoreWarnings: Yes -// CHECK: Diagnostic flags: -// CHECK: -Wunused - // CHECK: Header search options: // CHECK: System root [-isysroot=]: '/' // CHECK: Use builtin include directories [-nobuiltininc]: Yes @@ -47,3 +42,8 @@ // CHECK: Predefined macros: // CHECK: -DBLARG // CHECK: -DWIBBLE=WOBBLE + +// CHECK: Diagnostic options: +// CHECK: IgnoreWarnings: Yes +// CHECK: Diagnostic flags: +// CHECK: -Wunused Index: test/Modules/rebuild.m =================================================================== --- test/Modules/rebuild.m +++ test/Modules/rebuild.m @@ -19,11 +19,10 @@ // RUN: diff %t/Module.size %t/Module.size.saved // RUN: cp %t/Module.pcm %t/Module.pcm.saved.2 -// But the signature at least is expected to change, so we rebuild DependsOnModule. -// NOTE: if we change how the signature is created, this test may need updating. +// The signature is the hash of the PCM content, we will not rebuild rebuild DependsOnModule. // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s // RUN: diff %t/Module.pcm %t/Module.pcm.saved.2 -// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved +// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved // Rebuild Module, reset its timestamp, and verify its size hasn't changed // RUN: rm %t/Module.pcm @@ -34,10 +33,9 @@ // RUN: cp %t/Module.pcm %t/Module.pcm.saved.2 // Verify again with Module pre-imported. -// NOTE: if we change how the signature is created, this test may need updating. // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s // RUN: diff %t/Module.pcm %t/Module.pcm.saved.2 -// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved +// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved #ifdef PREIMPORT @import Module;