Index: cfe/trunk/docs/Modules.rst =================================================================== --- cfe/trunk/docs/Modules.rst +++ cfe/trunk/docs/Modules.rst @@ -469,9 +469,16 @@ .. parsed-literal:: *header-declaration*: - ``private``:sub:`opt` ``textual``:sub:`opt` ``header`` *string-literal* - ``umbrella`` ``header`` *string-literal* - ``exclude`` ``header`` *string-literal* + ``private``:sub:`opt` ``textual``:sub:`opt` ``header`` *string-literal* *header-attrs*:sub:`opt` + ``umbrella`` ``header`` *string-literal* *header-attrs*:sub:`opt` + ``exclude`` ``header`` *string-literal* *header-attrs*:sub:`opt` + + *header-attrs*: + '{' *header-attr** '}' + + *header-attr*: + ``size`` *integer-literal* + ``mtime`` *integer-literal* A header declaration that does not contain ``exclude`` nor ``textual`` specifies a header that contributes to the enclosing module. Specifically, when the module is built, the named header will be parsed and its declarations will be (logically) placed into the enclosing submodule. @@ -504,6 +511,18 @@ A given header shall not be referenced by more than one *header-declaration*. +Two *header-declaration*\s, or a *header-declaration* and a ``#include``, are +considered to refer to the same file if the paths resolve to the same file +and the specified *header-attr*\s (if any) match the attributes of that file, +even if the file is named differently (for instance, by a relative path or +via symlinks). + +.. note:: + The use of *header-attr*\s avoids the need for Clang to speculatively + ``stat`` every header referenced by a module map. It is recommended that + *header-attr*\s only be used in machine-generated module maps, to avoid + mismatches between attribute values and the corresponding files. + Umbrella directory declaration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ An umbrella directory declaration specifies that all of the headers in the specified directory should be included within the module. Index: cfe/trunk/include/clang/Basic/DiagnosticLexKinds.td =================================================================== --- cfe/trunk/include/clang/Basic/DiagnosticLexKinds.td +++ cfe/trunk/include/clang/Basic/DiagnosticLexKinds.td @@ -664,6 +664,12 @@ InGroup; def note_mmap_rename_top_level_private_as_submodule : Note< "make '%0' a submodule of '%1' to ensure it can be found by name">; +def err_mmap_duplicate_header_attribute : Error< + "header attribute '%0' specified multiple times">; +def err_mmap_invalid_header_attribute_value : Error< + "expected integer literal as value for header attribute '%0'">; +def err_mmap_expected_header_attribute : Error< + "expected a header attribute name ('size' or 'mtime')">; def warn_auto_module_import : Warning< "treating #%select{include|import|include_next|__include_macros}0 as an " Index: cfe/trunk/include/clang/Basic/DiagnosticSerializationKinds.td =================================================================== --- cfe/trunk/include/clang/Basic/DiagnosticSerializationKinds.td +++ cfe/trunk/include/clang/Basic/DiagnosticSerializationKinds.td @@ -174,10 +174,6 @@ "method %2 with %ordinal3 parameter of type %4%select{| decayed from %6}5|" "method %2 with %ordinal3 parameter named %4}1">; -def warn_module_uses_date_time : Warning< - "%select{precompiled header|module}0 uses __DATE__ or __TIME__">, - InGroup>; - def warn_duplicate_module_file_extension : Warning< "duplicate module file extension block name '%0'">, InGroup; @@ -186,7 +182,15 @@ "module file '%0' was validated as a system module and is now being imported " "as a non-system module; any difference in diagnostic options will be ignored">, InGroup; +} // let CategoryName +let CategoryName = "AST Serialization Issue" in { +def warn_module_uses_date_time : Warning< + "%select{precompiled header|module}0 uses __DATE__ or __TIME__">, + InGroup>; +def err_module_no_size_mtime_for_header : Error< + "cannot emit module %0: %select{size|mtime}1 must be explicitly specified " + "for missing header file \"%2\"">; } // let CategoryName } // let Component Index: cfe/trunk/include/clang/Basic/Module.h =================================================================== --- cfe/trunk/include/clang/Basic/Module.h +++ cfe/trunk/include/clang/Basic/Module.h @@ -154,11 +154,19 @@ /// \brief Stored information about a header directive that was found in the /// module map file but has not been resolved to a file. struct UnresolvedHeaderDirective { + HeaderKind Kind = HK_Normal; SourceLocation FileNameLoc; std::string FileName; - bool IsUmbrella; + bool IsUmbrella = false; + bool HasBuiltinHeader = false; + Optional Size; + Optional ModTime; }; + /// Headers that are mentioned in the module map file but that we have not + /// yet attempted to resolve to a file on the file system. + SmallVector UnresolvedHeaders; + /// \brief Headers that are mentioned in the module map file but could not be /// found on the file system. SmallVector MissingHeaders; Index: cfe/trunk/include/clang/Lex/ModuleMap.h =================================================================== --- cfe/trunk/include/clang/Lex/ModuleMap.h +++ cfe/trunk/include/clang/Lex/ModuleMap.h @@ -26,6 +26,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/TinyPtrVector.h" #include "llvm/ADT/Twine.h" #include #include @@ -116,6 +117,11 @@ // Adjust ModuleMap::addHeader. }; + /// Convert a header kind to a role. Requires Kind to not be HK_Excluded. + static ModuleHeaderRole headerKindToRole(Module::HeaderKind Kind); + /// Convert a header role to a kind. + static Module::HeaderKind headerRoleToKind(ModuleHeaderRole Role); + /// \brief A header that is known to reside within a given module, /// whether it was included or excluded. class KnownHeader { @@ -165,7 +171,13 @@ /// \brief Mapping from each header to the module that owns the contents of /// that header. HeadersMap Headers; - + + /// Map from file sizes to modules with lazy header directives of that size. + mutable llvm::DenseMap> LazyHeadersBySize; + /// Map from mtimes to modules with lazy header directives with those mtimes. + mutable llvm::DenseMap> + LazyHeadersByModTime; + /// \brief Mapping from directories with umbrella headers to the module /// that is generated from the umbrella header. /// @@ -257,22 +269,30 @@ /// resolved. Module *resolveModuleId(const ModuleId &Id, Module *Mod, bool Complain) const; - /// Resolve the given header directive to an actual header file. + /// Add an unresolved header to a module. + void addUnresolvedHeader(Module *Mod, + Module::UnresolvedHeaderDirective Header); + + /// Look up the given header directive to find an actual header file. /// /// \param M The module in which we're resolving the header directive. /// \param Header The header directive to resolve. /// \param RelativePathName Filled in with the relative path name from the /// module to the resolved header. /// \return The resolved file, if any. - const FileEntry *resolveHeader(Module *M, - Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &RelativePathName); + const FileEntry *findHeader(Module *M, + const Module::UnresolvedHeaderDirective &Header, + SmallVectorImpl &RelativePathName); + + /// Resolve the given header directive. + void resolveHeader(Module *M, + const Module::UnresolvedHeaderDirective &Header); /// Attempt to resolve the specified header directive as naming a builtin /// header. - const FileEntry * - resolveAsBuiltinHeader(Module *M, Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &BuiltinPathName); + /// \return \c true if a corresponding builtin header was found. + bool resolveAsBuiltinHeader(Module *M, + const Module::UnresolvedHeaderDirective &Header); /// \brief Looks up the modules that \p File corresponds to. /// @@ -368,6 +388,15 @@ /// the preferred module for the header. ArrayRef findAllModulesForHeader(const FileEntry *File) const; + /// Resolve all lazy header directives for the specified file. + /// + /// This ensures that the HeaderFileInfo on HeaderSearch is up to date. This + /// is effectively internal, but is exposed so HeaderSearch can call it. + void resolveHeaderDirectives(const FileEntry *File) const; + + /// Resolve all lazy header directives for the specified module. + void resolveHeaderDirectives(Module *Mod) const; + /// \brief Reports errors if a module must not include a specific file. /// /// \param RequestingModule The module including a file. Index: cfe/trunk/lib/Basic/Module.cpp =================================================================== --- cfe/trunk/lib/Basic/Module.cpp +++ cfe/trunk/lib/Basic/Module.cpp @@ -394,11 +394,30 @@ {"exclude ", HK_Excluded}}; for (auto &K : Kinds) { + assert(&K == &Kinds[K.Kind] && "kinds in wrong order"); for (auto &H : Headers[K.Kind]) { OS.indent(Indent + 2); OS << K.Prefix << "header \""; OS.write_escaped(H.NameAsWritten); - OS << "\"\n"; + OS << "\" { size " << H.Entry->getSize() + << " mtime " << H.Entry->getModificationTime() << " }\n"; + } + } + for (auto *Unresolved : {&UnresolvedHeaders, &MissingHeaders}) { + for (auto &U : *Unresolved) { + OS.indent(Indent + 2); + OS << Kinds[U.Kind].Prefix << "header \""; + OS.write_escaped(U.FileName); + OS << "\""; + if (U.Size || U.ModTime) { + OS << " {"; + if (U.Size) + OS << " size " << *U.Size; + if (U.ModTime) + OS << " mtime " << *U.ModTime; + OS << " }"; + } + OS << "\n"; } } Index: cfe/trunk/lib/Frontend/FrontendAction.cpp =================================================================== --- cfe/trunk/lib/Frontend/FrontendAction.cpp +++ cfe/trunk/lib/Frontend/FrontendAction.cpp @@ -289,14 +289,28 @@ /// /// \param Includes Will be augmented with the set of \#includes or \#imports /// needed to load all of the named headers. -static std::error_code -collectModuleHeaderIncludes(const LangOptions &LangOpts, FileManager &FileMgr, - ModuleMap &ModMap, clang::Module *Module, - SmallVectorImpl &Includes) { +static std::error_code collectModuleHeaderIncludes( + const LangOptions &LangOpts, FileManager &FileMgr, DiagnosticsEngine &Diag, + ModuleMap &ModMap, clang::Module *Module, SmallVectorImpl &Includes) { // Don't collect any headers for unavailable modules. if (!Module->isAvailable()) return std::error_code(); + // Resolve all lazy header directives to header files. + ModMap.resolveHeaderDirectives(Module); + + // If any headers are missing, we can't build this module. In most cases, + // diagnostics for this should have already been produced; we only get here + // if explicit stat information was provided. + // FIXME: If the name resolves to a file with different stat information, + // produce a better diagnostic. + if (!Module->MissingHeaders.empty()) { + auto &MissingHeader = Module->MissingHeaders.front(); + Diag.Report(MissingHeader.FileNameLoc, diag::err_module_header_missing) + << MissingHeader.IsUmbrella << MissingHeader.FileName; + return std::error_code(); + } + // Add includes for each of these headers. for (auto HK : {Module::HK_Normal, Module::HK_Private}) { for (Module::Header &H : Module->Headers[HK]) { @@ -367,7 +381,7 @@ SubEnd = Module->submodule_end(); Sub != SubEnd; ++Sub) if (std::error_code Err = collectModuleHeaderIncludes( - LangOpts, FileMgr, ModMap, *Sub, Includes)) + LangOpts, FileMgr, Diag, ModMap, *Sub, Includes)) return Err; return std::error_code(); @@ -494,7 +508,7 @@ addHeaderInclude(UmbrellaHeader.NameAsWritten, HeaderContents, CI.getLangOpts(), M->IsExternC); Err = collectModuleHeaderIncludes( - CI.getLangOpts(), FileMgr, + CI.getLangOpts(), FileMgr, CI.getDiagnostics(), CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(), M, HeaderContents); Index: cfe/trunk/lib/Lex/HeaderSearch.cpp =================================================================== --- cfe/trunk/lib/Lex/HeaderSearch.cpp +++ cfe/trunk/lib/Lex/HeaderSearch.cpp @@ -1114,6 +1114,8 @@ auto TryEnterImported = [&](void) -> bool { if (!ModulesEnabled) return false; + // Ensure FileInfo bits are up to date. + ModMap.resolveHeaderDirectives(File); // Modules with builtins are special; multiple modules use builtins as // modular headers, example: // Index: cfe/trunk/lib/Lex/ModuleMap.cpp =================================================================== --- cfe/trunk/lib/Lex/ModuleMap.cpp +++ cfe/trunk/lib/Lex/ModuleMap.cpp @@ -36,6 +36,37 @@ #endif using namespace clang; +Module::HeaderKind ModuleMap::headerRoleToKind(ModuleHeaderRole Role) { + switch ((int)Role) { + default: llvm_unreachable("unknown header role"); + case NormalHeader: + return Module::HK_Normal; + case PrivateHeader: + return Module::HK_Private; + case TextualHeader: + return Module::HK_Textual; + case PrivateHeader | TextualHeader: + return Module::HK_PrivateTextual; + } +} + +ModuleMap::ModuleHeaderRole +ModuleMap::headerKindToRole(Module::HeaderKind Kind) { + switch ((int)Kind) { + case Module::HK_Normal: + return NormalHeader; + case Module::HK_Private: + return PrivateHeader; + case Module::HK_Textual: + return TextualHeader; + case Module::HK_PrivateTextual: + return ModuleHeaderRole(PrivateHeader | TextualHeader); + case Module::HK_Excluded: + llvm_unreachable("unexpected header kind"); + } + llvm_unreachable("unknown header kind"); +} + Module::ExportDecl ModuleMap::resolveExport(Module *Mod, const Module::UnresolvedExportDecl &Unresolved, @@ -104,12 +135,22 @@ } const FileEntry * -ModuleMap::resolveHeader(Module *M, Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &RelativePathName) { +ModuleMap::findHeader(Module *M, + const Module::UnresolvedHeaderDirective &Header, + SmallVectorImpl &RelativePathName) { + auto GetFile = [&](StringRef Filename) -> const FileEntry * { + auto *File = SourceMgr.getFileManager().getFile(Filename); + if (!File || + (Header.Size && File->getSize() != *Header.Size) || + (Header.ModTime && File->getModificationTime() != *Header.ModTime)) + return nullptr; + return File; + }; + if (llvm::sys::path::is_absolute(Header.FileName)) { RelativePathName.clear(); RelativePathName.append(Header.FileName.begin(), Header.FileName.end()); - return SourceMgr.getFileManager().getFile(Header.FileName); + return GetFile(Header.FileName); } // Search for the header file within the module's home directory. @@ -124,7 +165,7 @@ // Check whether this file is in the public headers. llvm::sys::path::append(RelativePathName, "Headers", Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - if (auto *File = SourceMgr.getFileManager().getFile(FullPathName)) + if (auto *File = GetFile(FullPathName)) return File; // Check whether this file is in the private headers. @@ -141,31 +182,74 @@ llvm::sys::path::append(RelativePathName, "PrivateHeaders", Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - return SourceMgr.getFileManager().getFile(FullPathName); + return GetFile(FullPathName); } // Lookup for normal headers. llvm::sys::path::append(RelativePathName, Header.FileName); llvm::sys::path::append(FullPathName, RelativePathName); - return SourceMgr.getFileManager().getFile(FullPathName); + return GetFile(FullPathName); } -const FileEntry * -ModuleMap::resolveAsBuiltinHeader(Module *M, - Module::UnresolvedHeaderDirective Header, - SmallVectorImpl &BuiltinPathName) { - if (llvm::sys::path::is_absolute(Header.FileName) || M->isPartOfFramework() || - !M->IsSystem || Header.IsUmbrella || !BuiltinIncludeDir || - BuiltinIncludeDir == M->Directory || !isBuiltinHeader(Header.FileName)) - return nullptr; +void ModuleMap::resolveHeader(Module *Mod, + const Module::UnresolvedHeaderDirective &Header) { + SmallString<128> RelativePathName; + if (const FileEntry *File = findHeader(Mod, Header, RelativePathName)) { + if (Header.IsUmbrella) { + const DirectoryEntry *UmbrellaDir = File->getDir(); + if (Module *UmbrellaMod = UmbrellaDirs[UmbrellaDir]) + Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash) + << UmbrellaMod->getFullModuleName(); + else + // Record this umbrella header. + setUmbrellaHeader(Mod, File, RelativePathName.str()); + } else { + Module::Header H = {RelativePathName.str(), File}; + if (Header.Kind == Module::HK_Excluded) + excludeHeader(Mod, H); + else + addHeader(Mod, H, headerKindToRole(Header.Kind)); + } + } else if (Header.HasBuiltinHeader && !Header.Size && !Header.ModTime) { + // There's a builtin header but no corresponding on-disk header. Assume + // this was supposed to modularize the builtin header alone. + } else if (Header.Kind == Module::HK_Excluded) { + // Ignore missing excluded header files. They're optional anyway. + } else { + // If we find a module that has a missing header, we mark this module as + // unavailable and store the header directive for displaying diagnostics. + Mod->MissingHeaders.push_back(Header); + // A missing header with stat information doesn't make the module + // unavailable; this keeps our behavior consistent as headers are lazily + // resolved. (Such a module still can't be built though, except from + // preprocessed source.) + if (!Header.Size && !Header.ModTime) + Mod->markUnavailable(); + } +} + +bool ModuleMap::resolveAsBuiltinHeader( + Module *Mod, const Module::UnresolvedHeaderDirective &Header) { + if (Header.Kind == Module::HK_Excluded || + llvm::sys::path::is_absolute(Header.FileName) || + Mod->isPartOfFramework() || !Mod->IsSystem || Header.IsUmbrella || + !BuiltinIncludeDir || BuiltinIncludeDir == Mod->Directory || + !isBuiltinHeader(Header.FileName)) + return false; // This is a system module with a top-level header. This header // may have a counterpart (or replacement) in the set of headers // supplied by Clang. Find that builtin header. - llvm::sys::path::append(BuiltinPathName, BuiltinIncludeDir->getName(), - Header.FileName); - return SourceMgr.getFileManager().getFile( - StringRef(BuiltinPathName.data(), BuiltinPathName.size())); + SmallString<128> Path; + llvm::sys::path::append(Path, BuiltinIncludeDir->getName(), Header.FileName); + auto *File = SourceMgr.getFileManager().getFile(Path); + if (!File) + return false; + + auto Role = headerKindToRole(Header.Kind); + Module::Header H = {Path.str(), File}; + addHeader(Mod, H, Role); + return true; } ModuleMap::ModuleMap(SourceManager &SourceMgr, DiagnosticsEngine &Diags, @@ -246,6 +330,7 @@ ModuleMap::HeadersMap::iterator ModuleMap::findKnownHeader(const FileEntry *File) { + resolveHeaderDirectives(File); HeadersMap::iterator Known = Headers.find(File); if (HeaderInfo.getHeaderSearchOpts().ImplicitModuleMaps && Known == Headers.end() && File->getDir() == BuiltinIncludeDir && @@ -328,8 +413,10 @@ if (getTopLevelOrNull(RequestingModule) != getTopLevelOrNull(SourceModule)) return; - if (RequestingModule) + if (RequestingModule) { resolveUses(RequestingModule, /*Complain=*/false); + resolveHeaderDirectives(RequestingModule); + } bool Excluded = false; Module *Private = nullptr; @@ -511,6 +598,7 @@ ArrayRef ModuleMap::findAllModulesForHeader(const FileEntry *File) const { + resolveHeaderDirectives(File); auto It = Headers.find(File); if (It == Headers.end()) return None; @@ -524,6 +612,7 @@ bool ModuleMap::isHeaderUnavailableInModule(const FileEntry *Header, const Module *RequestingModule) const { + resolveHeaderDirectives(Header); HeadersMap::const_iterator Known = Headers.find(Header); if (Known != Headers.end()) { for (SmallVectorImpl::const_iterator @@ -896,20 +985,65 @@ UmbrellaDirs[UmbrellaDir] = Mod; } -static Module::HeaderKind headerRoleToKind(ModuleMap::ModuleHeaderRole Role) { - switch ((int)Role) { - default: llvm_unreachable("unknown header role"); - case ModuleMap::NormalHeader: - return Module::HK_Normal; - case ModuleMap::PrivateHeader: - return Module::HK_Private; - case ModuleMap::TextualHeader: - return Module::HK_Textual; - case ModuleMap::PrivateHeader | ModuleMap::TextualHeader: - return Module::HK_PrivateTextual; +void ModuleMap::addUnresolvedHeader(Module *Mod, + Module::UnresolvedHeaderDirective Header) { + // If there is a builtin counterpart to this file, add it now so it can + // wrap the system header. + if (resolveAsBuiltinHeader(Mod, Header)) { + // If we have both a builtin and system version of the file, the + // builtin version may want to inject macros into the system header, so + // force the system header to be treated as a textual header in this + // case. + Header.Kind = headerRoleToKind(ModuleMap::ModuleHeaderRole( + headerKindToRole(Header.Kind) | ModuleMap::TextualHeader)); + Header.HasBuiltinHeader = true; + } + + // If possible, don't stat the header until we need to. This requires the + // user to have provided us with some stat information about the file. + // FIXME: Add support for lazily stat'ing umbrella headers and excluded + // headers. + if ((Header.Size || Header.ModTime) && !Header.IsUmbrella && + Header.Kind != Module::HK_Excluded) { + // We expect more variation in mtime than size, so if we're given both, + // use the mtime as the key. + if (Header.ModTime) + LazyHeadersByModTime[*Header.ModTime].push_back(Mod); + else + LazyHeadersBySize[*Header.Size].push_back(Mod); + Mod->UnresolvedHeaders.push_back(Header); + return; + } + + // We don't have stat information or can't defer looking this file up. + // Perform the lookup now. + resolveHeader(Mod, Header); +} + +void ModuleMap::resolveHeaderDirectives(const FileEntry *File) const { + auto BySize = LazyHeadersBySize.find(File->getSize()); + if (BySize != LazyHeadersBySize.end()) { + for (auto *M : BySize->second) + resolveHeaderDirectives(M); + LazyHeadersBySize.erase(BySize); + } + + auto ByModTime = LazyHeadersByModTime.find(File->getModificationTime()); + if (ByModTime != LazyHeadersByModTime.end()) { + for (auto *M : ByModTime->second) + resolveHeaderDirectives(M); + LazyHeadersByModTime.erase(ByModTime); } } +void ModuleMap::resolveHeaderDirectives(Module *Mod) const { + for (auto &Header : Mod->UnresolvedHeaders) + // This operation is logically const; we're just changing how we represent + // the header information for this file. + const_cast(this)->resolveHeader(Mod, Header); + Mod->UnresolvedHeaders.clear(); +} + void ModuleMap::addHeader(Module *Mod, Module::Header Header, ModuleHeaderRole Role, bool Imported) { KnownHeader KH(Mod, Role); @@ -1063,6 +1197,7 @@ RequiresKeyword, Star, StringLiteral, + IntegerLiteral, TextualKeyword, LBrace, RBrace, @@ -1072,7 +1207,12 @@ unsigned Location; unsigned StringLength; - const char *StringData; + union { + // If Kind != IntegerLiteral. + const char *StringData; + // If Kind == IntegerLiteral. + uint64_t IntegerValue; + }; void clear() { Kind = EndOfFile; @@ -1086,9 +1226,14 @@ SourceLocation getLocation() const { return SourceLocation::getFromRawEncoding(Location); } + + uint64_t getInteger() const { + return Kind == IntegerLiteral ? IntegerValue : 0; + } StringRef getString() const { - return StringRef(StringData, StringLength); + return Kind == IntegerLiteral ? StringRef() + : StringRef(StringData, StringLength); } }; @@ -1278,6 +1423,25 @@ Tok.StringLength = Length; break; } + + case tok::numeric_constant: { + // We don't support any suffixes or other complications. + SmallString<32> SpellingBuffer; + SpellingBuffer.resize(LToken.getLength() + 1); + const char *Start = SpellingBuffer.data(); + unsigned Length = + Lexer::getSpelling(LToken, Start, SourceMgr, L.getLangOpts()); + uint64_t Value; + if (StringRef(Start, Length).getAsInteger(0, Value)) { + Diags.Report(Tok.getLocation(), diag::err_mmap_unknown_token); + HadError = true; + goto retry; + } + + Tok.Kind = MMToken::IntegerLiteral; + Tok.IntegerValue = Value; + break; + } case tok::comment: goto retry; @@ -1904,6 +2068,9 @@ Header.FileName = Tok.getString(); Header.FileNameLoc = consumeToken(); Header.IsUmbrella = LeadingToken == MMToken::UmbrellaKeyword; + Header.Kind = + (LeadingToken == MMToken::ExcludeKeyword ? Module::HK_Excluded + : Map.headerRoleToKind(Role)); // Check whether we already have an umbrella. if (Header.IsUmbrella && ActiveModule->Umbrella) { @@ -1913,64 +2080,62 @@ return; } - // Look for this file by name if we don't have any stat information. - SmallString<128> RelativePathName, BuiltinPathName; - const FileEntry *File = - Map.resolveHeader(ActiveModule, Header, RelativePathName); - const FileEntry *BuiltinFile = - Map.resolveAsBuiltinHeader(ActiveModule, Header, BuiltinPathName); - - // If Clang supplies this header but the underlying system does not, - // just silently swap in our builtin version. Otherwise, we'll end - // up adding both (later). - if (BuiltinFile && !File) { - RelativePathName = BuiltinPathName; - File = BuiltinFile; - BuiltinFile = nullptr; - } - - // FIXME: We shouldn't be eagerly stat'ing every file named in a module map. - // Come up with a lazy way to do this. - if (File) { - if (Header.IsUmbrella) { - const DirectoryEntry *UmbrellaDir = File->getDir(); - if (Module *UmbrellaModule = Map.UmbrellaDirs[UmbrellaDir]) { - Diags.Report(LeadingLoc, diag::err_mmap_umbrella_clash) - << UmbrellaModule->getFullModuleName(); - HadError = true; - } else { - // Record this umbrella header. - Map.setUmbrellaHeader(ActiveModule, File, RelativePathName.str()); - } - } else if (LeadingToken == MMToken::ExcludeKeyword) { - Module::Header H = {RelativePathName.str(), File}; - Map.excludeHeader(ActiveModule, H); - } else { - // If there is a builtin counterpart to this file, add it now so it can - // wrap the system header. - if (BuiltinFile) { - Module::Header H = { BuiltinPathName.str(), BuiltinFile }; - Map.addHeader(ActiveModule, H, Role); - - // If we have both a builtin and system version of the file, the - // builtin version may want to inject macros into the system header, so - // force the system header to be treated as a textual header in this - // case. - Role = ModuleMap::ModuleHeaderRole(Role | ModuleMap::TextualHeader); - } + // If we were given stat information, parse it so we can skip looking for + // the file. + if (Tok.is(MMToken::LBrace)) { + SourceLocation LBraceLoc = consumeToken(); + + while (!Tok.is(MMToken::RBrace) && !Tok.is(MMToken::EndOfFile)) { + enum Attribute { Size, ModTime, Unknown }; + StringRef Str = Tok.getString(); + SourceLocation Loc = consumeToken(); + switch (llvm::StringSwitch(Str) + .Case("size", Size) + .Case("mtime", ModTime) + .Default(Unknown)) { + case Size: + if (Header.Size) + Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str; + if (!Tok.is(MMToken::IntegerLiteral)) { + Diags.Report(Tok.getLocation(), + diag::err_mmap_invalid_header_attribute_value) << Str; + skipUntil(MMToken::RBrace); + break; + } + Header.Size = Tok.getInteger(); + consumeToken(); + break; - // Record this header. - Module::Header H = { RelativePathName.str(), File }; - Map.addHeader(ActiveModule, H, Role); + case ModTime: + if (Header.ModTime) + Diags.Report(Loc, diag::err_mmap_duplicate_header_attribute) << Str; + if (!Tok.is(MMToken::IntegerLiteral)) { + Diags.Report(Tok.getLocation(), + diag::err_mmap_invalid_header_attribute_value) << Str; + skipUntil(MMToken::RBrace); + break; + } + Header.ModTime = Tok.getInteger(); + consumeToken(); + break; + + case Unknown: + Diags.Report(Loc, diag::err_mmap_expected_header_attribute); + skipUntil(MMToken::RBrace); + break; + } } - } else if (LeadingToken != MMToken::ExcludeKeyword) { - // Ignore excluded header files. They're optional anyway. - // If we find a module that has a missing header, we mark this module as - // unavailable and store the header directive for displaying diagnostics. - ActiveModule->markUnavailable(); - ActiveModule->MissingHeaders.push_back(Header); + if (Tok.is(MMToken::RBrace)) + consumeToken(); + else { + Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace); + Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match); + HadError = true; + } } + + Map.addUnresolvedHeader(ActiveModule, std::move(Header)); } static int compareModuleHeaders(const Module::Header *A, @@ -2521,6 +2686,7 @@ case MMToken::RequiresKeyword: case MMToken::Star: case MMToken::StringLiteral: + case MMToken::IntegerLiteral: case MMToken::TextualKeyword: case MMToken::UmbrellaKeyword: case MMToken::UseKeyword: Index: cfe/trunk/lib/Lex/PPDirectives.cpp =================================================================== --- cfe/trunk/lib/Lex/PPDirectives.cpp +++ cfe/trunk/lib/Lex/PPDirectives.cpp @@ -689,6 +689,8 @@ while (!Loc.isInvalid() && !SM.isInMainFile(Loc)) { auto ID = SM.getFileID(SM.getExpansionLoc(Loc)); auto *FE = SM.getFileEntryForID(ID); + if (!FE) + break; bool InTextualHeader = false; for (auto Header : HeaderInfo.getModuleMap().findAllModulesForHeader(FE)) { Index: cfe/trunk/lib/Serialization/ASTWriter.cpp =================================================================== --- cfe/trunk/lib/Serialization/ASTWriter.cpp +++ cfe/trunk/lib/Serialization/ASTWriter.cpp @@ -1856,24 +1856,31 @@ // Trait used for the on-disk hash table of header search information. class HeaderFileInfoTrait { ASTWriter &Writer; - const HeaderSearch &HS; // Keep track of the framework names we've used during serialization. SmallVector FrameworkStringData; llvm::StringMap FrameworkNameOffset; public: - HeaderFileInfoTrait(ASTWriter &Writer, const HeaderSearch &HS) - : Writer(Writer), HS(HS) { } - + HeaderFileInfoTrait(ASTWriter &Writer) : Writer(Writer) {} + struct key_type { - const FileEntry *FE; StringRef Filename; + off_t Size; + time_t ModTime; }; typedef const key_type &key_type_ref; + + using UnresolvedModule = + llvm::PointerIntPair; - typedef HeaderFileInfo data_type; + struct data_type { + const HeaderFileInfo &HFI; + ArrayRef KnownHeaders; + UnresolvedModule Unresolved; + }; typedef const data_type &data_type_ref; + typedef unsigned hash_value_type; typedef unsigned offset_type; @@ -1881,8 +1888,7 @@ // The hash is based only on size/time of the file, so that the reader can // match even when symlinking or excess path elements ("foo/../", "../") // change the form of the name. However, complete path is still the key. - return llvm::hash_combine(key.FE->getSize(), - Writer.getTimestampForOutput(key.FE)); + return llvm::hash_combine(key.Size, key.ModTime); } std::pair @@ -1892,68 +1898,74 @@ unsigned KeyLen = key.Filename.size() + 1 + 8 + 8; LE.write(KeyLen); unsigned DataLen = 1 + 2 + 4 + 4; - for (auto ModInfo : HS.getModuleMap().findAllModulesForHeader(key.FE)) + for (auto ModInfo : Data.KnownHeaders) if (Writer.getLocalOrImportedSubmoduleID(ModInfo.getModule())) DataLen += 4; + if (Data.Unresolved.getPointer()) + DataLen += 4; LE.write(DataLen); return std::make_pair(KeyLen, DataLen); } - + void EmitKey(raw_ostream& Out, key_type_ref key, unsigned KeyLen) { using namespace llvm::support; endian::Writer LE(Out); - LE.write(key.FE->getSize()); + LE.write(key.Size); KeyLen -= 8; - LE.write(Writer.getTimestampForOutput(key.FE)); + LE.write(key.ModTime); KeyLen -= 8; Out.write(key.Filename.data(), KeyLen); } - + void EmitData(raw_ostream &Out, key_type_ref key, data_type_ref Data, unsigned DataLen) { using namespace llvm::support; endian::Writer LE(Out); uint64_t Start = Out.tell(); (void)Start; - unsigned char Flags = (Data.isImport << 4) - | (Data.isPragmaOnce << 3) - | (Data.DirInfo << 1) - | Data.IndexHeaderMapHeader; + unsigned char Flags = (Data.HFI.isImport << 4) + | (Data.HFI.isPragmaOnce << 3) + | (Data.HFI.DirInfo << 1) + | Data.HFI.IndexHeaderMapHeader; LE.write(Flags); - LE.write(Data.NumIncludes); + LE.write(Data.HFI.NumIncludes); - if (!Data.ControllingMacro) - LE.write(Data.ControllingMacroID); + if (!Data.HFI.ControllingMacro) + LE.write(Data.HFI.ControllingMacroID); else - LE.write(Writer.getIdentifierRef(Data.ControllingMacro)); - + LE.write(Writer.getIdentifierRef(Data.HFI.ControllingMacro)); + unsigned Offset = 0; - if (!Data.Framework.empty()) { + if (!Data.HFI.Framework.empty()) { // If this header refers into a framework, save the framework name. llvm::StringMap::iterator Pos - = FrameworkNameOffset.find(Data.Framework); + = FrameworkNameOffset.find(Data.HFI.Framework); if (Pos == FrameworkNameOffset.end()) { Offset = FrameworkStringData.size() + 1; - FrameworkStringData.append(Data.Framework.begin(), - Data.Framework.end()); + FrameworkStringData.append(Data.HFI.Framework.begin(), + Data.HFI.Framework.end()); FrameworkStringData.push_back(0); - FrameworkNameOffset[Data.Framework] = Offset; + FrameworkNameOffset[Data.HFI.Framework] = Offset; } else Offset = Pos->second; } LE.write(Offset); - // FIXME: If the header is excluded, we should write out some - // record of that fact. - for (auto ModInfo : HS.getModuleMap().findAllModulesForHeader(key.FE)) { - if (uint32_t ModID = - Writer.getLocalOrImportedSubmoduleID(ModInfo.getModule())) { - uint32_t Value = (ModID << 2) | (unsigned)ModInfo.getRole(); + auto EmitModule = [&](Module *M, ModuleMap::ModuleHeaderRole Role) { + if (uint32_t ModID = Writer.getLocalOrImportedSubmoduleID(M)) { + uint32_t Value = (ModID << 2) | (unsigned)Role; assert((Value >> 2) == ModID && "overflow in header module info"); LE.write(Value); } - } + }; + + // FIXME: If the header is excluded, we should write out some + // record of that fact. + for (auto ModInfo : Data.KnownHeaders) + EmitModule(ModInfo.getModule(), ModInfo.getRole()); + if (Data.Unresolved.getPointer()) + EmitModule(Data.Unresolved.getPointer(), Data.Unresolved.getInt()); assert(Out.tell() - Start == DataLen && "Wrong data length"); } @@ -1968,16 +1980,71 @@ /// /// \param HS The header search structure to save. void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) { + HeaderFileInfoTrait GeneratorTrait(*this); + llvm::OnDiskChainedHashTableGenerator Generator; + SmallVector SavedStrings; + unsigned NumHeaderSearchEntries = 0; + + // Find all unresolved headers for the current module. We generally will + // have resolved them before we get here, but not necessarily: we might be + // compiling a preprocessed module, where there is no requirement for the + // original files to exist any more. + if (WritingModule) { + llvm::SmallVector Worklist(1, WritingModule); + while (!Worklist.empty()) { + Module *M = Worklist.pop_back_val(); + if (!M->isAvailable()) + continue; + + // Map to disk files where possible, to pick up any missing stat + // information. This also means we don't need to check the unresolved + // headers list when emitting resolved headers in the first loop below. + // FIXME: It'd be preferable to avoid doing this if we were given + // sufficient stat information in the module map. + HS.getModuleMap().resolveHeaderDirectives(M); + + // If the file didn't exist, we can still create a module if we were given + // enough information in the module map. + for (auto U : M->MissingHeaders) { + // Check that we were given enough information to build a module + // without this file existing on disk. + if (!U.Size || (!U.ModTime && IncludeTimestamps)) { + PP->Diag(U.FileNameLoc, diag::err_module_no_size_mtime_for_header) + << WritingModule->getFullModuleName() << U.Size.hasValue() + << U.FileName; + continue; + } + + // Form the effective relative pathname for the file. + SmallString<128> Filename(M->Directory->getName()); + llvm::sys::path::append(Filename, U.FileName); + PreparePathForOutput(Filename); + + StringRef FilenameDup = strdup(Filename.c_str()); + SavedStrings.push_back(FilenameDup.data()); + + HeaderFileInfoTrait::key_type Key = { + FilenameDup, *U.Size, IncludeTimestamps ? *U.ModTime : 0 + }; + HeaderFileInfoTrait::data_type Data = { + {}, {}, {M, ModuleMap::headerKindToRole(U.Kind)} + }; + // FIXME: Deal with cases where there are multiple unresolved header + // directives in different submodules for the same header. + Generator.insert(Key, Data, GeneratorTrait); + ++NumHeaderSearchEntries; + } + + Worklist.append(M->submodule_begin(), M->submodule_end()); + } + } + SmallVector FilesByUID; HS.getFileMgr().GetUniqueIDMapping(FilesByUID); if (FilesByUID.size() > HS.header_file_size()) FilesByUID.resize(HS.header_file_size()); - - HeaderFileInfoTrait GeneratorTrait(*this, HS); - llvm::OnDiskChainedHashTableGenerator Generator; - SmallVector SavedStrings; - unsigned NumHeaderSearchEntries = 0; + for (unsigned UID = 0, LastUID = FilesByUID.size(); UID != LastUID; ++UID) { const FileEntry *File = FilesByUID[UID]; if (!File) @@ -2004,11 +2071,16 @@ SavedStrings.push_back(Filename.data()); } - HeaderFileInfoTrait::key_type key = { File, Filename }; - Generator.insert(key, *HFI, GeneratorTrait); + HeaderFileInfoTrait::key_type Key = { + Filename, File->getSize(), getTimestampForOutput(File) + }; + HeaderFileInfoTrait::data_type Data = { + *HFI, HS.getModuleMap().findAllModulesForHeader(File), {} + }; + Generator.insert(Key, Data, GeneratorTrait); ++NumHeaderSearchEntries; } - + // Create the on-disk hash table in a buffer. SmallString<4096> TableData; uint32_t BucketOffset; Index: cfe/trunk/test/Modules/Inputs/header-attribs/bar.h =================================================================== --- cfe/trunk/test/Modules/Inputs/header-attribs/bar.h +++ cfe/trunk/test/Modules/Inputs/header-attribs/bar.h @@ -0,0 +1 @@ +extern int b; Index: cfe/trunk/test/Modules/Inputs/header-attribs/baz.h =================================================================== --- cfe/trunk/test/Modules/Inputs/header-attribs/baz.h +++ cfe/trunk/test/Modules/Inputs/header-attribs/baz.h @@ -0,0 +1 @@ +extern int c; Index: cfe/trunk/test/Modules/Inputs/header-attribs/foo.h =================================================================== --- cfe/trunk/test/Modules/Inputs/header-attribs/foo.h +++ cfe/trunk/test/Modules/Inputs/header-attribs/foo.h @@ -0,0 +1 @@ +extern int a; \ No newline at end of file Index: cfe/trunk/test/Modules/Inputs/header-attribs/modular.modulemap =================================================================== --- cfe/trunk/test/Modules/Inputs/header-attribs/modular.modulemap +++ cfe/trunk/test/Modules/Inputs/header-attribs/modular.modulemap @@ -0,0 +1,5 @@ +module A { + header "foo.h" { size 13 } + header "bar.h" { size 1000 } + header "baz.h" { mtime 1 } +} Index: cfe/trunk/test/Modules/Inputs/header-attribs/textual.modulemap =================================================================== --- cfe/trunk/test/Modules/Inputs/header-attribs/textual.modulemap +++ cfe/trunk/test/Modules/Inputs/header-attribs/textual.modulemap @@ -0,0 +1,5 @@ +module A { + textual header "foo.h" { size 13 } + textual header "bar.h" { size 1000 } + textual header "baz.h" { mtime 1 } +} Index: cfe/trunk/test/Modules/diagnostics.modulemap =================================================================== --- cfe/trunk/test/Modules/diagnostics.modulemap +++ cfe/trunk/test/Modules/diagnostics.modulemap @@ -1,4 +1,4 @@ -// RUN: not %clang_cc1 -fmodules -fmodules-cache-path=%t -fmodule-map-file=%S/Inputs/diagnostics-aux.modulemap -fmodule-map-file=%s -fsyntax-only -x c++ /dev/null 2>&1 | FileCheck %s +// RUN: not %clang_cc1 -fmodules -fmodules-cache-path=%t -fmodule-map-file=%S/Inputs/diagnostics-aux.modulemap -fmodule-map-file=%s -fsyntax-only -x c++ /dev/null 2>&1 | FileCheck %s --implicit-check-not error: // CHECK: In file included from {{.*}}diagnostics-aux.modulemap:3: // CHECK: diagnostics-aux-2.modulemap:2:3: error: expected @@ -15,3 +15,15 @@ // CHECK: diagnostics.modulemap:[[@LINE+1]]:22: error: use declarations are only allowed in top-level modules module submodule { use foo } } + +module header_attr { + // CHECK: diagnostics.modulemap:[[@LINE+1]]:20: error: expected a header attribute name + header "foo.h" { x } + // CHECK: diagnostics.modulemap:[[@LINE+1]]:27: error: header attribute 'size' specified multiple times + header "bar.h" { size 1 size 2 } + // CHECK: diagnostics.modulemap:[[@LINE+1]]:25: error: expected integer literal as value for header attribute 'size' + header "baz.h" { size "30 kilobytes" } + + header "quux.h" { size 1 mtime 2 } + header "no_attrs.h" {} +} Index: cfe/trunk/test/Modules/header-attribs.cpp =================================================================== --- cfe/trunk/test/Modules/header-attribs.cpp +++ cfe/trunk/test/Modules/header-attribs.cpp @@ -0,0 +1,10 @@ +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -I%S/Inputs/header-attribs -fmodule-map-file=%S/Inputs/header-attribs/textual.modulemap -fmodules-cache-path=%t -verify %s -fmodule-name=A -fmodules-strict-decluse +// RUN: not %clang_cc1 -fmodules -I%S/Inputs/header-attribs -emit-module -x c++-module-map %S/Inputs/header-attribs/modular.modulemap -fmodules-cache-path=%t -fmodule-name=A 2>&1 | FileCheck %s --check-prefix BUILD-MODULAR + +#include "foo.h" // ok, stats match +#include "bar.h" // expected-error {{does not depend on a module exporting 'bar.h'}} +#include "baz.h" // expected-error {{does not depend on a module exporting 'baz.h'}} + +// FIXME: Explain why the 'bar.h' found on disk doesn't match the module map. +// BUILD-MODULAR: error: header 'bar.h' not found Index: cfe/trunk/test/Modules/preprocess-missing.modulemap =================================================================== --- cfe/trunk/test/Modules/preprocess-missing.modulemap +++ cfe/trunk/test/Modules/preprocess-missing.modulemap @@ -0,0 +1,7 @@ +// RUN: %clang_cc1 -fmodules -fmodule-name=A -x c++-module-map %s -emit-module -o /dev/null -verify +module A { + header "does not exist" { size 12345 } // ok, do not need mtime for explicit module build + header "also does not exist" { mtime 12345 } +} +#pragma clang module contents +// expected-error@4 {{cannot emit module A: size must be explicitly specified for missing header file "also does not exist"}} Index: cfe/trunk/test/Modules/preprocess-module.cpp =================================================================== --- cfe/trunk/test/Modules/preprocess-module.cpp +++ cfe/trunk/test/Modules/preprocess-module.cpp @@ -28,12 +28,21 @@ // RUN: %clang_cc1 -fmodules -fmodule-file=%t/no-rewrite.pcm %s -I%t -verify -fno-modules-error-recovery -DINCLUDE -I%S/Inputs/preprocess // RUN: %clang_cc1 -fmodules -fmodule-file=%t/rewrite.pcm %s -I%t -verify -fno-modules-error-recovery -DREWRITE -DINCLUDE -I%S/Inputs/preprocess +// Now try building the module when the header files are missing. +// RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%t -x c++-module-map %t/module.modulemap -E -frewrite-includes -o %t/copy.ii +// RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -x c++-module-map-cpp-output %t/copy.ii -emit-module -o %t/copy.pcm + +// Finally, check that our module contains correct mapping information for the headers. +// RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t +// RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify -fno-modules-error-recovery -DCOPY -DINCLUDE // == module map // CHECK: # 1 "{{.*}}module.modulemap" // CHECK: module file { -// CHECK: header "file.h" -// CHECK: header "file2.h" +// CHECK: header "file.h" { size +// CHECK: header "file2.h" { size // CHECK: } // == file.h @@ -98,6 +107,8 @@ __FILE *a; // expected-error {{declaration of '__FILE' must be imported}} #ifdef REWRITE // expected-note@rewrite.ii:1 {{here}} +#elif COPY +// expected-note@copy.ii:1 {{here}} #else // expected-note@no-rewrite.ii:1 {{here}} #endif