Index: include/clang/AST/ExternalASTSource.h =================================================================== --- include/clang/AST/ExternalASTSource.h +++ include/clang/AST/ExternalASTSource.h @@ -16,6 +16,7 @@ #include "clang/AST/CharUnits.h" #include "clang/AST/DeclBase.h" +#include "clang/Basic/Module.h" #include "llvm/ADT/DenseMap.h" namespace clang { @@ -149,20 +150,20 @@ StringRef PCHModuleName; StringRef Path; StringRef ASTFile; - uint64_t Signature = 0; + ASTFileSignature Signature = {{0}}; const Module *ClangModule = nullptr; public: ASTSourceDescriptor(){}; ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile, - uint64_t Signature) + ASTFileSignature Signature) : PCHModuleName(std::move(Name)), Path(std::move(Path)), ASTFile(std::move(ASTFile)), Signature(Signature){}; ASTSourceDescriptor(const Module &M); std::string getModuleName() const; StringRef getPath() const { return Path; } StringRef getASTFile() const { return ASTFile; } - uint64_t getSignature() const { return Signature; } + ASTFileSignature getSignature() const { return Signature; } const Module *getModuleOrNull() const { return ClangModule; } }; Index: include/clang/Basic/Module.h =================================================================== --- include/clang/Basic/Module.h +++ include/clang/Basic/Module.h @@ -43,6 +43,7 @@ /// \brief Describes the name of a module. typedef SmallVector, 2> ModuleId; +typedef std::array ASTFileSignature; /// \brief Describes a module or submodule. class Module { public: @@ -65,7 +66,7 @@ llvm::PointerUnion Umbrella; /// \brief The module signature. - uint64_t Signature; + ASTFileSignature Signature; /// \brief The name of the umbrella entry, as written in the module map. std::string UmbrellaAsWritten; Index: include/clang/Frontend/PCHContainerOperations.h =================================================================== --- include/clang/Frontend/PCHContainerOperations.h +++ include/clang/Frontend/PCHContainerOperations.h @@ -10,6 +10,7 @@ #ifndef LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H #define LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H +#include "clang/Basic/Module.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/Support/MemoryBuffer.h" @@ -29,7 +30,7 @@ class CompilerInstance; struct PCHBuffer { - uint64_t Signature; + ASTFileSignature Signature; llvm::SmallVector Data; bool IsComplete; }; Index: include/clang/Serialization/ASTBitCodes.h =================================================================== --- include/clang/Serialization/ASTBitCodes.h +++ include/clang/Serialization/ASTBitCodes.h @@ -253,6 +253,7 @@ /// \brief A block containing a module file extension. EXTENSION_BLOCK_ID, + DIAGNOSTIC_OPTIONS_BLOCK_ID }; /// \brief Record types that occur within the control block. @@ -288,9 +289,6 @@ /// AST file. MODULE_MAP_FILE, - /// \brief Record code for the signature that identifiers this AST file. - SIGNATURE, - /// \brief Record code for the module build directory. MODULE_DIRECTORY, }; @@ -320,6 +318,9 @@ /// \brief Record code for the preprocessor options table. PREPROCESSOR_OPTIONS, + + /// \brief Record code for the signature that identifiers this AST file. + SIGNATURE, }; /// \brief Record code for extension blocks. Index: include/clang/Serialization/ASTReader.h =================================================================== --- include/clang/Serialization/ASTReader.h +++ include/clang/Serialization/ASTReader.h @@ -1163,7 +1163,7 @@ SourceLocation ImportLoc, ModuleFile *ImportedBy, SmallVectorImpl &Loaded, off_t ExpectedSize, time_t ExpectedModTime, - serialization::ASTFileSignature ExpectedSignature, + ASTFileSignature ExpectedSignature, unsigned ClientLoadCapabilities); ASTReadResult ReadControlBlock(ModuleFile &F, SmallVectorImpl &Loaded, @@ -1172,7 +1172,11 @@ static ASTReadResult ReadOptionsBlock( llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities, bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, - std::string &SuggestedPredefines, bool ValidateDiagnosticOptions); + std::string &SuggestedPredefines); + static ASTReadResult ReadDiagnosticOptionsBlock(ModuleFile *F, + llvm::BitstreamCursor &Stream, unsigned ClientLoadCapabilities, + bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, + bool ValidateDiagnosticOptions); ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities); ASTReadResult ReadExtensionBlock(ModuleFile &F); bool ParseLineTable(ModuleFile &F, const RecordData &Record); Index: include/clang/Serialization/ASTWriter.h =================================================================== --- include/clang/Serialization/ASTWriter.h +++ include/clang/Serialization/ASTWriter.h @@ -106,6 +106,9 @@ /// \brief The bitstream writer used to emit this precompiled header. llvm::BitstreamWriter &Stream; + /// \brief The buffer associated with the bitstream. + const SmallVectorImpl &Buffer; + /// \brief The ASTContext we're writing. ASTContext *Context; @@ -415,6 +418,9 @@ std::vector> ModuleFileExtensionWriters; + /// \brief Bit position for start of DIAGNOSTIC_OPTIONS_BLOCK. + size_t StartOfDiagOptions; + /// \brief Retrieve or create a submodule ID for this module. unsigned getSubmoduleID(Module *Mod); @@ -422,8 +428,11 @@ void WriteSubStmt(Stmt *S); void WriteBlockInfoBlock(); - uint64_t WriteControlBlock(Preprocessor &PP, ASTContext &Context, + void WriteControlBlock(Preprocessor &PP, ASTContext &Context, StringRef isysroot, const std::string &OutputFile); + ASTFileSignature WriteDiagnosticOptionBlock(ASTContext &Context); + /// \brief Calculate hash of the pcm content and write it to SIGNATURE record. + ASTFileSignature WriteHash(size_t); void WriteInputFiles(SourceManager &SourceMgr, HeaderSearchOptions &HSOpts, bool Modules); void WriteSourceManagerBlock(SourceManager &SourceMgr, @@ -488,14 +497,14 @@ void WriteDeclAbbrevs(); void WriteDecl(ASTContext &Context, Decl *D); - uint64_t WriteASTCore(Sema &SemaRef, + ASTFileSignature WriteASTCore(Sema &SemaRef, StringRef isysroot, const std::string &OutputFile, Module *WritingModule); public: /// \brief Create a new precompiled header writer that outputs to /// the given bitstream. - ASTWriter(llvm::BitstreamWriter &Stream, + ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, ArrayRef> Extensions, bool IncludeTimestamps = true); ~ASTWriter() override; @@ -521,7 +530,7 @@ /// /// \return the module signature, which eventually will be a hash of /// the module but currently is merely a random 32-bit number. - uint64_t WriteAST(Sema &SemaRef, const std::string &OutputFile, + ASTFileSignature WriteAST(Sema &SemaRef, const std::string &OutputFile, Module *WritingModule, StringRef isysroot, bool hasErrors = false); Index: include/clang/Serialization/Module.h =================================================================== --- include/clang/Serialization/Module.h +++ include/clang/Serialization/Module.h @@ -16,6 +16,7 @@ #define LLVM_CLANG_SERIALIZATION_MODULE_H #include "clang/Basic/FileManager.h" +#include "clang/Basic/Module.h" #include "clang/Basic/SourceLocation.h" #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" @@ -89,8 +90,6 @@ bool isNotFound() const { return Val.getInt() == NotFound; } }; -typedef unsigned ASTFileSignature; - /// \brief Information about a module that has been loaded by the ASTReader. /// /// Each instance of the Module class corresponds to a single AST file, which Index: lib/Basic/Module.cpp =================================================================== --- lib/Basic/Module.cpp +++ lib/Basic/Module.cpp @@ -27,7 +27,7 @@ Module::Module(StringRef Name, SourceLocation DefinitionLoc, Module *Parent, bool IsFramework, bool IsExplicit, unsigned VisibilityID) : Name(Name), DefinitionLoc(DefinitionLoc), Parent(Parent), Directory(), - Umbrella(), Signature(0), ASTFile(nullptr), VisibilityID(VisibilityID), + Umbrella(), Signature({{0}}), ASTFile(nullptr), VisibilityID(VisibilityID), IsMissingRequirement(false), HasIncompatibleModuleFile(false), IsAvailable(true), IsFromModuleFile(false), IsFramework(IsFramework), IsExplicit(IsExplicit), IsSystem(false), IsExternC(false), Index: lib/CodeGen/CGDebugInfo.cpp =================================================================== --- lib/CodeGen/CGDebugInfo.cpp +++ lib/CodeGen/CGDebugInfo.cpp @@ -1975,7 +1975,9 @@ if (CreateSkeletonCU && IsRootModule) { // PCH files don't have a signature field in the control block, // but LLVM detects skeleton CUs by looking for a non-zero DWO id. - uint64_t Signature = Mod.getSignature() ? Mod.getSignature() : ~1ULL; + // We use the lower 64 bits for debug info. + uint64_t Signature = Mod.getSignature() != ASTFileSignature({{0}}) ? + (uint64_t)Mod.getSignature()[1] << 32 | Mod.getSignature()[0] : ~1ULL; llvm::DIBuilder DIB(CGM.getModule()); DIB.createCompileUnit(TheCU->getSourceLanguage(), Mod.getModuleName(), Mod.getPath(), TheCU->getProducer(), true, Index: lib/CodeGen/ObjectFilePCHContainerOperations.cpp =================================================================== --- lib/CodeGen/ObjectFilePCHContainerOperations.cpp +++ lib/CodeGen/ObjectFilePCHContainerOperations.cpp @@ -171,7 +171,8 @@ // Prepare CGDebugInfo to emit debug info for a clang module. auto *DI = Builder->getModuleDebugInfo(); StringRef ModuleName = llvm::sys::path::filename(MainFileName); - DI->setPCHDescriptor({ModuleName, "", OutputFileName, ~1ULL}); + DI->setPCHDescriptor({ModuleName, "", OutputFileName, + ASTFileSignature{{~0U, ~0U, ~0U, ~0U, ~1U}} }); DI->setModuleMap(MMap); } @@ -241,7 +242,9 @@ // PCH files don't have a signature field in the control block, // but LLVM detects DWO CUs by looking for a non-zero DWO id. - uint64_t Signature = Buffer->Signature ? Buffer->Signature : ~1ULL; + // We use the lower 64 bits for debug info. + uint64_t Signature = Buffer->Signature != ASTFileSignature({{0}}) ? + (uint64_t)Buffer->Signature[1] << 32 | Buffer->Signature[0] : ~1ULL; Builder->getModuleDebugInfo()->setDwoId(Signature); // Finalize the Builder. Index: lib/Frontend/ASTUnit.cpp =================================================================== --- lib/Frontend/ASTUnit.cpp +++ lib/Frontend/ASTUnit.cpp @@ -185,7 +185,7 @@ llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } + ASTWriterData() : Stream(Buffer), Writer(Stream, Buffer, { }) { } }; void ASTUnit::clearFileLevelDecls() { @@ -2519,7 +2519,7 @@ SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream, { }); + ASTWriter Writer(Stream, Buffer, { }); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } Index: lib/Serialization/ASTReader.cpp =================================================================== --- lib/Serialization/ASTReader.cpp +++ lib/Serialization/ASTReader.cpp @@ -2146,10 +2146,61 @@ llvm_unreachable("unknown ASTReadResult"); } +ASTReader::ASTReadResult ASTReader::ReadDiagnosticOptionsBlock( + ModuleFile *F, BitstreamCursor &Stream, unsigned ClientLoadCapabilities, + bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, + bool ValidateDiagnosticOptions) { + // Read all of the records in the options block. + RecordData Record; + ASTReadResult Result = Success; + while (1) { + llvm::BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + case llvm::BitstreamEntry::SubBlock: + return Failure; + + case llvm::BitstreamEntry::EndBlock: + return Result; + + case llvm::BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read and process a record. + Record.clear(); + switch ((OptionsRecordTypes)Stream.readRecord(Entry.ID, Record)) { + case DIAGNOSTIC_OPTIONS: { + bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0; + if (ValidateDiagnosticOptions && + !AllowCompatibleConfigurationMismatch && + ParseDiagnosticOptions(Record, Complain, Listener)) + return OutOfDate; + break; + } + case SIGNATURE: { + if (!F) + break; + int Pos = 0; + F->Signature[Pos++] = Record[0]; + F->Signature[Pos++] = Record[1]; + F->Signature[Pos++] = Record[2]; + F->Signature[Pos++] = Record[3]; + F->Signature[Pos++] = Record[4]; + break; + } + default: + llvm_unreachable("unknown record in DIAGNOSTIC_OPTIONS_BLOCK"); + } + } +} + ASTReader::ASTReadResult ASTReader::ReadOptionsBlock( BitstreamCursor &Stream, unsigned ClientLoadCapabilities, bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, - std::string &SuggestedPredefines, bool ValidateDiagnosticOptions) { + std::string &SuggestedPredefines) { if (Stream.EnterSubBlock(OPTIONS_BLOCK_ID)) return Failure; @@ -2191,13 +2242,9 @@ break; } - case DIAGNOSTIC_OPTIONS: { - bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0; - if (ValidateDiagnosticOptions && - !AllowCompatibleConfigurationMismatch && - ParseDiagnosticOptions(Record, Complain, Listener)) - return OutOfDate; - break; + case DIAGNOSTIC_OPTIONS: + case SIGNATURE: { + llvm_unreachable("unknown record in OPTIONS_BLOCK"); } case FILE_SYSTEM_OPTIONS: { @@ -2323,13 +2370,10 @@ // FIXME: Allow this for files explicitly specified with -include-pch. bool AllowCompatibleConfigurationMismatch = F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule; - const HeaderSearchOptions &HSOpts = - PP.getHeaderSearchInfo().getHeaderSearchOpts(); Result = ReadOptionsBlock(Stream, ClientLoadCapabilities, AllowCompatibleConfigurationMismatch, - *Listener, SuggestedPredefines, - HSOpts.ModulesValidateDiagnosticOptions); + *Listener, SuggestedPredefines); if (Result == Failure) { Error("malformed block record in AST file"); return Result; @@ -2403,11 +2447,6 @@ break; } - case SIGNATURE: - assert((!F.Signature || F.Signature == Record[0]) && "signature changed"); - F.Signature = Record[0]; - break; - case IMPORTS: { // Load each of the imported PCH files. unsigned Idx = 0, N = Record.size(); @@ -2421,7 +2460,10 @@ ReadUntranslatedSourceLocation(Record[Idx++]); off_t StoredSize = (off_t)Record[Idx++]; time_t StoredModTime = (time_t)Record[Idx++]; - ASTFileSignature StoredSignature = Record[Idx++]; + ASTFileSignature StoredSignature = + {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++]}}; auto ImportedFile = ReadPath(F, Record, Idx); // If our client can't cope with us being out of date, we can't cope with @@ -3592,7 +3634,7 @@ SmallVector Loaded; switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type, ImportLoc, /*ImportedBy=*/nullptr, Loaded, - 0, 0, 0, + 0, 0, {{0}}, ClientLoadCapabilities)) { case Failure: case Missing: @@ -3897,6 +3939,7 @@ // This is used for compatibility with older PCH formats. bool HaveReadControlBlock = false; + BitstreamCursor ASTCursor; while (true) { llvm::BitstreamEntry Entry = Stream.advance(); @@ -3947,10 +3990,48 @@ return VersionMismatch; } - // Record that we've loaded this module. - Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc)); - return Success; + ASTCursor = Stream; + if (Stream.SkipBlock()) { + Error("malformed block record in AST file"); + return Failure; + } + break; + case DIAGNOSTIC_OPTIONS_BLOCK_ID: { + if (Stream.EnterSubBlock(DIAGNOSTIC_OPTIONS_BLOCK_ID)) { + Error("malformed block record in AST file"); + return Failure; + } + + ASTReadResult Result = Success; + if (Listener && !ImportedBy) { + const HeaderSearchOptions &HSOpts = + PP.getHeaderSearchInfo().getHeaderSearchOpts(); + bool AllowCompatibleConfigurationMismatch = + F.Kind == MK_ExplicitModule || F.Kind == MK_PrebuiltModule; + Result = ReadDiagnosticOptionsBlock(&F, Stream, ClientLoadCapabilities, + AllowCompatibleConfigurationMismatch, + *Listener, + HSOpts.ModulesValidateDiagnosticOptions); + + if (Result == Failure) { + Error("malformed block record in AST file"); + return Failure; + } + + if (DisableValidation || + (AllowConfigurationMismatch && Result == ConfigurationMismatch)) + Result = Success; + } + + if (Result == Success) + // Record that we've loaded this module. + Loaded.push_back(ImportedModule(M, ImportedBy, ImportLoc)); + + // Move Stream backward to point to ASTCursor. + Stream = ASTCursor; + return Result; + } default: if (Stream.SkipBlock()) { Error("malformed block record in AST file"); @@ -4179,23 +4260,24 @@ static ASTFileSignature readASTFileSignature(StringRef PCH) { BitstreamCursor Stream(PCH); if (!startsWithASTFileMagic(Stream)) - return 0; + return {{0}}; - // Scan for the CONTROL_BLOCK_ID block. - if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID)) - return 0; + // Scan for the DIAGNOSTIC_OPTIONS_BLOCK_ID block. + if (SkipCursorToBlock(Stream, DIAGNOSTIC_OPTIONS_BLOCK_ID)) + return {{0}}; - // Scan for SIGNATURE inside the control block. + // Scan for SIGNATURE inside the diagnostic options block. ASTReader::RecordData Record; while (true) { llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); if (Entry.Kind != llvm::BitstreamEntry::Record) - return 0; + return {{0}}; Record.clear(); StringRef Blob; if (SIGNATURE == Stream.readRecord(Entry.ID, Record, &Blob)) - return Record[0]; + return {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2], + (uint32_t)Record[3], (uint32_t)Record[4]}}; } } @@ -4342,8 +4424,7 @@ std::string IgnoredSuggestedPredefines; if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate, /*AllowCompatibleConfigurationMismatch*/ false, - Listener, IgnoredSuggestedPredefines, - ValidateDiagnosticOptions) != Success) + Listener, IgnoredSuggestedPredefines) != Success) return true; break; } @@ -4464,6 +4545,7 @@ // Look for module file extension blocks, if requested. if (FindModuleFileExtensions) { + BitstreamCursor SavedStream = Stream; while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) { bool DoneWithExtensionBlock = false; while (!DoneWithExtensionBlock) { @@ -4502,8 +4584,18 @@ } } } + Stream = SavedStream; } + // Scan for the DIAGNOSTIC_OPTIONS_BLOCK_ID block. + if (SkipCursorToBlock(Stream, DIAGNOSTIC_OPTIONS_BLOCK_ID)) + return true; + if (ReadDiagnosticOptionsBlock(nullptr, Stream, + ARR_ConfigurationMismatch | ARR_OutOfDate, + /*AllowCompatibleConfigurationMismatch*/ false, + Listener, ValidateDiagnosticOptions) != Success) + return true; + return false; } Index: lib/Serialization/ASTWriter.cpp =================================================================== --- lib/Serialization/ASTWriter.cpp +++ lib/Serialization/ASTWriter.cpp @@ -79,6 +79,7 @@ #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/SHA1.h" #include #include #include @@ -1000,7 +1001,6 @@ // Control Block. BLOCK(CONTROL_BLOCK); RECORD(METADATA); - RECORD(SIGNATURE); RECORD(MODULE_NAME); RECORD(MODULE_DIRECTORY); RECORD(MODULE_MAP_FILE); @@ -1013,7 +1013,6 @@ BLOCK(OPTIONS_BLOCK); RECORD(LANGUAGE_OPTIONS); RECORD(TARGET_OPTIONS); - RECORD(DIAGNOSTIC_OPTIONS); RECORD(FILE_SYSTEM_OPTIONS); RECORD(HEADER_SEARCH_OPTIONS); RECORD(PREPROCESSOR_OPTIONS); @@ -1239,6 +1238,10 @@ BLOCK(EXTENSION_BLOCK); RECORD(EXTENSION_METADATA); + BLOCK(DIAGNOSTIC_OPTIONS_BLOCK); + RECORD(DIAGNOSTIC_OPTIONS); + RECORD(SIGNATURE); + #undef RECORD #undef BLOCK Stream.ExitBlock(); @@ -1301,21 +1304,68 @@ return Filename + Pos; } -static ASTFileSignature getSignature() { - while (true) { - if (ASTFileSignature S = llvm::sys::Process::GetRandomNumber()) - return S; - // Rely on GetRandomNumber to eventually return non-zero... - } +ASTFileSignature ASTWriter::WriteHash(size_t BlockStartPos) { + // Calculate the hash till start of DIAGNOSTIC_OPTIONS_BLOCK. + llvm::SHA1 Hasher; + Hasher.update(ArrayRef((const uint8_t *)&(Buffer)[BlockStartPos], + (StartOfDiagOptions>>3) - BlockStartPos)); + auto Hash = Hasher.result(); + + SmallVector Vals; + auto LShift = [&](unsigned char Val, unsigned Amount) + -> uint64_t { return ((uint64_t)Val) << Amount; }; + for (int Pos = 0; Pos < 20; Pos += 4) { + uint32_t SubHash = LShift(Hash[Pos + 0], 24); + SubHash |= LShift(Hash[Pos + 1], 16) | LShift(Hash[Pos + 2], 8) | + (unsigned)(unsigned char)Hash[Pos + 3]; + Vals.push_back(SubHash); + } + + // Emit the finished record. SIGNATURE: [5*i32] + Stream.EmitRecord(SIGNATURE, Vals); + return {{(uint32_t)Vals[0], (uint32_t)Vals[1], (uint32_t)Vals[2], + (uint32_t)Vals[3], (uint32_t)Vals[4]}}; +} + +ASTFileSignature ASTWriter::WriteDiagnosticOptionBlock(ASTContext &Context) { + StartOfDiagOptions = Stream.GetCurrentBitNo(); + // Write the diagnostic options block. + Stream.EnterSubblock(DIAGNOSTIC_OPTIONS_BLOCK_ID, 5); + + // Diagnostic options. + RecordData Record; + const DiagnosticOptions &DiagOpts + = Context.getDiagnostics().getDiagnosticOptions(); +#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name); +#define ENUM_DIAGOPT(Name, Type, Bits, Default) \ + Record.push_back(static_cast(DiagOpts.get##Name())); +#include "clang/Basic/DiagnosticOptions.def" + Record.push_back(DiagOpts.Warnings.size()); + for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I) + AddString(DiagOpts.Warnings[I], Record); + Record.push_back(DiagOpts.Remarks.size()); + for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I) + AddString(DiagOpts.Remarks[I], Record); + // Note: we don't serialize the log or serialization file names, because they + // are generally transient files and will almost always be overridden. + Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record); + + // For implicit modules we output a signature that is the hash of the pcm + // content. + ASTFileSignature RetCode{{0}}; + if (WritingModule && Context.getLangOpts().ImplicitModules) + RetCode = WriteHash(0); + + // Leave the options block. + Stream.ExitBlock(); + return RetCode; } /// \brief Write the control block. -uint64_t ASTWriter::WriteControlBlock(Preprocessor &PP, - ASTContext &Context, - StringRef isysroot, - const std::string &OutputFile) { - ASTFileSignature Signature = 0; - +void ASTWriter::WriteControlBlock(Preprocessor &PP, + ASTContext &Context, + StringRef isysroot, + const std::string &OutputFile) { using namespace llvm; Stream.EnterSubblock(CONTROL_BLOCK_ID, 5); RecordData Record; @@ -1343,15 +1393,6 @@ getClangFullRepositoryVersion()); } if (WritingModule) { - // For implicit modules we output a signature that we can use to ensure - // duplicate module builds don't collide in the cache as their output order - // is non-deterministic. - // FIXME: Remove this when output is deterministic. - if (Context.getLangOpts().ImplicitModules) { - Signature = getSignature(); - RecordData::value_type Record[] = {Signature}; - Stream.EmitRecord(SIGNATURE, Record); - } // Module name auto *Abbrev = new BitCodeAbbrev(); @@ -1424,9 +1465,18 @@ Record.push_back((unsigned)M->Kind); // FIXME: Stable encoding AddSourceLocation(M->ImportLoc, Record); - Record.push_back(M->File->getSize()); - Record.push_back(getTimestampForOutput(M->File)); - Record.push_back(M->Signature); + + // If we have calculated signature, there is no need to store + // the size or timestamp. + bool HasSignature = M->Signature != ASTFileSignature({{0}}); + Record.push_back(HasSignature ? 0 : M->File->getSize()); + Record.push_back(HasSignature ? 0 : getTimestampForOutput(M->File)); + + Record.push_back(M->Signature[0]); + Record.push_back(M->Signature[1]); + Record.push_back(M->Signature[2]); + Record.push_back(M->Signature[3]); + Record.push_back(M->Signature[4]); AddPath(M->FileName, Record); } Stream.EmitRecord(IMPORTS, Record); @@ -1489,24 +1539,6 @@ } Stream.EmitRecord(TARGET_OPTIONS, Record); - // Diagnostic options. - Record.clear(); - const DiagnosticOptions &DiagOpts - = Context.getDiagnostics().getDiagnosticOptions(); -#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name); -#define ENUM_DIAGOPT(Name, Type, Bits, Default) \ - Record.push_back(static_cast(DiagOpts.get##Name())); -#include "clang/Basic/DiagnosticOptions.def" - Record.push_back(DiagOpts.Warnings.size()); - for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I) - AddString(DiagOpts.Warnings[I], Record); - Record.push_back(DiagOpts.Remarks.size()); - for (unsigned I = 0, N = DiagOpts.Remarks.size(); I != N; ++I) - AddString(DiagOpts.Remarks[I], Record); - // Note: we don't serialize the log or serialization file names, because they - // are generally transient files and will almost always be overridden. - Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record); - // File system options. Record.clear(); const FileSystemOptions &FSOpts = @@ -1620,7 +1652,6 @@ PP.getHeaderSearchInfo().getHeaderSearchOpts(), PP.getLangOpts().Modules); Stream.ExitBlock(); - return Signature; } namespace { @@ -4185,10 +4216,10 @@ } ASTWriter::ASTWriter( - llvm::BitstreamWriter &Stream, + llvm::BitstreamWriter &Stream, SmallVectorImpl &Buffer, ArrayRef> Extensions, bool IncludeTimestamps) - : Stream(Stream), Context(nullptr), PP(nullptr), Chain(nullptr), + : Stream(Stream), Buffer(Buffer), Context(nullptr), PP(nullptr), Chain(nullptr), WritingModule(nullptr), IncludeTimestamps(IncludeTimestamps), WritingAST(false), DoneWritingDeclsAndTypes(false), ASTHasCompilerErrors(false), FirstDeclID(NUM_PREDEF_DECL_IDS), @@ -4226,7 +4257,7 @@ return IncludeTimestamps ? E->getModificationTime() : 0; } -uint64_t ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile, +ASTFileSignature ASTWriter::WriteAST(Sema &SemaRef, const std::string &OutputFile, Module *WritingModule, StringRef isysroot, bool hasErrors) { WritingAST = true; @@ -4264,7 +4295,7 @@ } } -uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, +ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, const std::string &OutputFile, Module *WritingModule) { using namespace llvm; @@ -4414,7 +4445,7 @@ } // Write the control block - uint64_t Signature = WriteControlBlock(PP, Context, isysroot, OutputFile); + WriteControlBlock(PP, Context, isysroot, OutputFile); // Write the remaining AST contents. Stream.EnterSubblock(AST_BLOCK_ID, 5); @@ -4766,6 +4797,8 @@ for (const auto &ExtWriter : ModuleFileExtensionWriters) WriteModuleFileExtension(SemaRef, *ExtWriter); + ASTFileSignature Signature = WriteDiagnosticOptionBlock(Context); + return Signature; } Index: lib/Serialization/GeneratePCH.cpp =================================================================== --- lib/Serialization/GeneratePCH.cpp +++ lib/Serialization/GeneratePCH.cpp @@ -28,7 +28,7 @@ bool AllowASTWithErrors, bool IncludeTimestamps) : PP(PP), OutputFile(OutputFile), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data), - Writer(Stream, Extensions, IncludeTimestamps), + Writer(Stream, Buffer->Data, Extensions, IncludeTimestamps), AllowASTWithErrors(AllowASTWithErrors) { Buffer->IsComplete = false; } Index: lib/Serialization/GlobalModuleIndex.cpp =================================================================== --- lib/Serialization/GlobalModuleIndex.cpp +++ lib/Serialization/GlobalModuleIndex.cpp @@ -376,6 +376,16 @@ /// \brief The set of modules on which this module depends. Each entry is /// a module ID. SmallVector Dependencies; + ASTFileSignature Signature; + }; + + struct ImportedModuleFileInfo { + off_t StoredSize; + time_t StoredModTime; + ASTFileSignature StoredSignature; + ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig) : + StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) { + } }; /// \brief Builder that generates the global module index file. @@ -389,6 +399,14 @@ /// \brief Information about each of the known module files. ModuleFilesMap ModuleFiles; + /// \brief Mapping from the imported module file to the imported + /// information. + typedef std::multimap + ImportedModuleFilesMap; + + /// \brief Information about each importing of a module file. + ImportedModuleFilesMap ImportedModuleFiles; + /// \brief Mapping from identifiers to the list of module file IDs that /// consider this identifier to be interesting. typedef llvm::StringMap > InterestingIdentifierMap; @@ -410,6 +428,7 @@ unsigned NewID = ModuleFiles.size(); ModuleFileInfo &Info = ModuleFiles[File]; Info.ID = NewID; + Info.Signature = {{0}}; return Info; } @@ -424,7 +443,8 @@ bool loadModuleFile(const FileEntry *File); /// \brief Write the index to the given bitstream. - void writeIndex(llvm::BitstreamWriter &Stream); + /// \returns true if an error occurred, false otherwise. + bool writeIndex(llvm::BitstreamWriter &Stream); }; } @@ -515,7 +535,7 @@ unsigned ID = getModuleFileInfo(File).ID; // Search for the blocks and records we care about. - enum { Other, ControlBlock, ASTBlock } State = Other; + enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other; bool Done = false; while (!Done) { llvm::BitstreamEntry Entry = InStream.advance(); @@ -553,6 +573,15 @@ continue; } + if (Entry.ID == DIAGNOSTIC_OPTIONS_BLOCK_ID) { + if (InStream.EnterSubBlock(DIAGNOSTIC_OPTIONS_BLOCK_ID)) + return true; + + // Found the Diagnostic Options block. + State = DiagnosticOptionsBlock; + continue; + } + if (InStream.SkipBlock()) return true; @@ -587,7 +616,10 @@ // Skip the stored signature. // FIXME: we could read the signature out of the import and validate it. - Idx++; + ASTFileSignature StoredSignature = + {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++], (uint32_t)Record[Idx++], + (uint32_t)Record[Idx++]}}; // Retrieve the imported file name. unsigned Length = Record[Idx++]; @@ -599,11 +631,15 @@ const FileEntry *DependsOnFile = FileMgr.getFile(ImportedFile, /*openFile=*/false, /*cacheFailure=*/false); - if (!DependsOnFile || - (StoredSize != DependsOnFile->getSize()) || - (StoredModTime != DependsOnFile->getModificationTime())) + + if (!DependsOnFile) return true; + // Save the information in ImportedModuleFileInfo so we can verify after + // loading all pcms. + ImportedModuleFiles.insert(std::make_pair(DependsOnFile, + ImportedModuleFileInfo(StoredSize, StoredModTime, StoredSignature))); + // Record the dependency. unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID; getModuleFileInfo(File).Dependencies.push_back(DependsOnID); @@ -632,6 +668,12 @@ } } + // Get Signature. + if (State == DiagnosticOptionsBlock && Code == SIGNATURE) + getModuleFileInfo(File).Signature = {{(uint32_t)Record[0], + (uint32_t)Record[1], (uint32_t)Record[2], + (uint32_t)Record[3], (uint32_t)Record[4]}}; + // We don't care about this record. } @@ -680,7 +722,21 @@ } -void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { +bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { + for (auto MapEntry : ImportedModuleFiles) { + auto *File = MapEntry.first; + ImportedModuleFileInfo &Info = MapEntry.second; + if (getModuleFileInfo(File).Signature != ASTFileSignature({{0}})) { + if (getModuleFileInfo(File).Signature != Info.StoredSignature) + // Verify Signature. + return true; + } + else if (Info.StoredSize != File->getSize() || + Info.StoredModTime != File->getModificationTime()) + // Verify Size and ModTime. + return true; + } + using namespace llvm; // Emit the file header. @@ -756,6 +812,7 @@ } Stream.ExitBlock(); + return false; } GlobalModuleIndex::ErrorCode @@ -816,7 +873,8 @@ SmallVector OutputBuffer; { llvm::BitstreamWriter OutputStream(OutputBuffer); - Builder.writeIndex(OutputStream); + if (Builder.writeIndex(OutputStream)) + return EC_IOError; } // Write the global index file to a temporary file. Index: lib/Serialization/Module.cpp =================================================================== --- lib/Serialization/Module.cpp +++ lib/Serialization/Module.cpp @@ -20,7 +20,7 @@ using namespace reader; ModuleFile::ModuleFile(ModuleKind Kind, unsigned Generation) - : Kind(Kind), File(nullptr), Signature(0), DirectlyImported(false), + : Kind(Kind), File(nullptr), Signature({{0}}), DirectlyImported(false), Generation(Generation), SizeInBits(0), LocalNumSLocEntries(0), SLocEntryBaseID(0), SLocEntryBaseOffset(0), SLocEntryOffsets(nullptr), Index: lib/Serialization/ModuleManager.cpp =================================================================== --- lib/Serialization/ModuleManager.cpp +++ lib/Serialization/ModuleManager.cpp @@ -138,15 +138,15 @@ ModuleEntry->Data = PCHContainerRdr.ExtractPCH(*ModuleEntry->Buffer); } - if (ExpectedSignature) { + if (ExpectedSignature != ASTFileSignature({{0}})) { // If we've not read the control block yet, read the signature eagerly now // so that we can check it. - if (!ModuleEntry->Signature) + if (ModuleEntry->Signature == ASTFileSignature({{0}})) ModuleEntry->Signature = ReadSignature(ModuleEntry->Data); if (ModuleEntry->Signature != ExpectedSignature) { - ErrorStr = ModuleEntry->Signature ? "signature mismatch" - : "could not read module signature"; + ErrorStr = ModuleEntry->Signature != ASTFileSignature({{0}}) ? + "signature mismatch" : "could not read module signature"; if (NewModule) delete ModuleEntry; Index: test/Modules/diagnostic-options-out-of-date.m =================================================================== --- test/Modules/diagnostic-options-out-of-date.m +++ test/Modules/diagnostic-options-out-of-date.m @@ -7,6 +7,16 @@ // RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s -fmodules-disable-diagnostic-validation // Make sure we don't error out when using the pch // RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify -fmodules-disable-diagnostic-validation + +// Build A.pcm +// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s +// Build pch that imports A.pcm +// RUN: %clang_cc1 -Werror -Wno-conversion -emit-pch -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -o %t.pch -I %S/Inputs -x objective-c-header %S/Inputs/pch-import-module-out-of-date.pch +// We will rebuild A.pcm and overwrite the original A.pcm that the pch imports, but the two versions have the same hash. +// RUN: %clang_cc1 -Werror -Wconversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs %s +// Make sure we don't error out when using the pch +// RUN: %clang_cc1 -Werror -Wno-conversion -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -fsyntax-only -I %S/Inputs -include-pch %t.pch %s -verify + // expected-no-diagnostics @import DiagOutOfDate; Index: test/Modules/explicit-build.cpp =================================================================== --- test/Modules/explicit-build.cpp +++ test/Modules/explicit-build.cpp @@ -199,6 +199,6 @@ // RUN: -fmodule-file=%t/c.pcm \ // RUN: %s -DHAVE_A -DHAVE_B -DHAVE_C 2>&1 | FileCheck --check-prefix=CHECK-MISMATCHED-B %s // -// CHECK-MISMATCHED-B: fatal error: module file '{{.*}}b.pcm' is out of date and needs to be rebuilt: module file out of date +// CHECK-MISMATCHED-B: fatal error: module file '{{.*}}b.pcm' is out of date and needs to be rebuilt: signature mismatch // CHECK-MISMATCHED-B-NEXT: note: imported by module 'c' // CHECK-MISMATCHED-B-NOT: note: Index: test/Modules/module_file_info.m =================================================================== --- test/Modules/module_file_info.m +++ test/Modules/module_file_info.m @@ -29,11 +29,6 @@ // CHECK: CPU: // CHECK: ABI: -// CHECK: Diagnostic options: -// CHECK: IgnoreWarnings: Yes -// CHECK: Diagnostic flags: -// CHECK: -Wunused - // CHECK: Header search options: // CHECK: System root [-isysroot=]: '/' // CHECK: Use builtin include directories [-nobuiltininc]: Yes @@ -47,3 +42,8 @@ // CHECK: Predefined macros: // CHECK: -DBLARG // CHECK: -DWIBBLE=WOBBLE + +// CHECK: Diagnostic options: +// CHECK: IgnoreWarnings: Yes +// CHECK: Diagnostic flags: +// CHECK: -Wunused Index: test/Modules/rebuild.m =================================================================== --- test/Modules/rebuild.m +++ test/Modules/rebuild.m @@ -19,11 +19,10 @@ // RUN: diff %t/Module.size %t/Module.size.saved // RUN: cp %t/Module.pcm %t/Module.pcm.saved.2 -// But the signature at least is expected to change, so we rebuild DependsOnModule. -// NOTE: if we change how the signature is created, this test may need updating. +// The signature is the hash of the PCM content, we will not rebuild rebuild DependsOnModule. // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s // RUN: diff %t/Module.pcm %t/Module.pcm.saved.2 -// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved +// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved // Rebuild Module, reset its timestamp, and verify its size hasn't changed // RUN: rm %t/Module.pcm @@ -34,10 +33,9 @@ // RUN: cp %t/Module.pcm %t/Module.pcm.saved.2 // Verify again with Module pre-imported. -// NOTE: if we change how the signature is created, this test may need updating. // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -fsyntax-only -F %S/Inputs %s // RUN: diff %t/Module.pcm %t/Module.pcm.saved.2 -// RUN: not diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved +// RUN: diff %t/DependsOnModule.pcm %t/DependsOnModule.pcm.saved #ifdef PREIMPORT @import Module;