Index: clang/include/clang/AST/ASTContext.h =================================================================== --- clang/include/clang/AST/ASTContext.h +++ clang/include/clang/AST/ASTContext.h @@ -794,6 +794,28 @@ /// redeclaration. mutable llvm::DenseMap ParsedComments; + /// Attaches \p DocComment to \p OriginalDeclForComment. + /// If \p DocComment == nullptr and we don't have Doc comment data for + /// \pOriginalDeclForComment yet, we cache the fact. + void cacheDocCommentDataForDecl(const RawComment *DocComment, + const Decl *OriginalDeclForComment) const; + + /// Attaches \p DocComment to \p D and its redeclarations. + /// If \p DocComment == nullptr we cache the lack of doc comments for any + /// redeclaration of \p D that doesn't have any doc comment data yet. + void cacheDocCommentDataForRedecls(const Decl *D, + const RawComment *DocComment, + const Decl *OriginalDeclForComment) const; + + /// \returns searches \p CommentsInFile for doc comment for \p D. + /// + /// \p RepresentativeLocForDecl is used as a location for searching doc + /// comments. \p CommentsInFile is a mapping offset -> comment of files in the + /// same file where \p RepresentativeLocForDecl is. + RawComment *getRawCommentForDeclNoCacheImpl( + const Decl *D, const SourceLocation RepresentativeLocForDecl, + const std::map &CommentsInFile) const; + /// Return the documentation comment attached to a given declaration, /// without looking into cache. RawComment *getRawCommentForDeclNoCache(const Decl *D) const; @@ -818,6 +840,10 @@ getRawCommentForAnyRedecl(const Decl *D, const Decl **OriginalDecl = nullptr) const; + /// Searches existing comments for doc comments that should be attached to \p + /// Decls. If any doc comment is found, it is parsed. + void searchForDocComments(ArrayRef Decls, const Preprocessor *PP); + /// Return parsed documentation comment attached to a given declaration. /// Returns nullptr if no comment is attached. /// Index: clang/include/clang/AST/RawCommentList.h =================================================================== --- clang/include/clang/AST/RawCommentList.h +++ clang/include/clang/AST/RawCommentList.h @@ -10,8 +10,11 @@ #define LLVM_CLANG_AST_RAWCOMMENTLIST_H #include "clang/Basic/CommentOptions.h" +#include "clang/Basic/SourceLocation.h" #include "clang/Basic/SourceManager.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include namespace clang { @@ -196,17 +199,31 @@ void addComment(const RawComment &RC, const CommentOptions &CommentOpts, llvm::BumpPtrAllocator &Allocator); - ArrayRef getComments() const { - return Comments; - } + /// \returns nullptr in case there are no comments in in \p File. + const std::map *getCommentsInFile(FileID File) const; + + bool empty() const; + + unsigned getCommentBeginLine(RawComment *C, FileID File, + unsigned Offset) const; + unsigned getCommentEndOffset(RawComment *C) const; private: SourceManager &SourceMgr; - std::vector Comments; - - void addDeserializedComments(ArrayRef DeserializedComments); + // mapping: FileId -> comment begin offset -> comment + llvm::DenseMap> OrderedComments; + mutable llvm::DenseMap CommentBeginLine; + mutable llvm::DenseMap CommentEndOffset; + + /// Adds comments that were deserialized from an AST file. + /// \p DeserializedComments is a mapping: FileId -> comment begin offset + /// -> comment + void addDeserializedComments( + const llvm::DenseMap> + &DeserializedComments); friend class ASTReader; + friend class ASTWriter; }; } // end namespace clang Index: clang/lib/AST/ASTContext.cpp =================================================================== --- clang/lib/AST/ASTContext.cpp +++ clang/lib/AST/ASTContext.cpp @@ -98,62 +98,60 @@ Float16Rank, HalfRank, FloatRank, DoubleRank, LongDoubleRank, Float128Rank }; -RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const { +/// \returns location that is relevant when searching for Doc comments related +/// to \p D. +static SourceLocation getDeclLocForCommentSearch(const Decl *D, + SourceManager &SourceMgr) { assert(D); - // If we already tried to load comments but there are none, - // we won't find anything. - if (CommentsLoaded && Comments.getComments().empty()) - return nullptr; - // User can not attach documentation to implicit declarations. if (D->isImplicit()) - return nullptr; + return {}; // User can not attach documentation to implicit instantiations. if (const auto *FD = dyn_cast(D)) { if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation) - return nullptr; + return {}; } if (const auto *VD = dyn_cast(D)) { if (VD->isStaticDataMember() && VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation) - return nullptr; + return {}; } if (const auto *CRD = dyn_cast(D)) { if (CRD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation) - return nullptr; + return {}; } if (const auto *CTSD = dyn_cast(D)) { TemplateSpecializationKind TSK = CTSD->getSpecializationKind(); if (TSK == TSK_ImplicitInstantiation || TSK == TSK_Undeclared) - return nullptr; + return {}; } if (const auto *ED = dyn_cast(D)) { if (ED->getTemplateSpecializationKind() == TSK_ImplicitInstantiation) - return nullptr; + return {}; } if (const auto *TD = dyn_cast(D)) { // When tag declaration (but not definition!) is part of the // decl-specifier-seq of some other declaration, it doesn't get comment if (TD->isEmbeddedInDeclarator() && !TD->isCompleteDefinition()) - return nullptr; + return {}; } // TODO: handle comments for function parameters properly. if (isa(D)) - return nullptr; + return {}; // TODO: we could look up template parameter documentation in the template // documentation. if (isa(D) || isa(D) || isa(D)) - return nullptr; + return {}; // Find declaration location. // For Objective-C declarations we generally don't expect to have multiple @@ -161,20 +159,19 @@ // location". // For all other declarations multiple declarators are used quite frequently, // so we use the location of the identifier as the "declaration location". - SourceLocation DeclLoc; if (isa(D) || isa(D) || isa(D) || isa(D) || isa(D)) - DeclLoc = D->getBeginLoc(); + return D->getBeginLoc(); else { - DeclLoc = D->getLocation(); + const SourceLocation DeclLoc = D->getLocation(); if (DeclLoc.isMacroID()) { if (isa(D)) { // If location of the typedef name is in a macro, it is because being // declared via a macro. Try using declaration's starting location as // the "declaration location". - DeclLoc = D->getBeginLoc(); + return D->getBeginLoc(); } else if (const auto *TD = dyn_cast(D)) { // If location of the tag decl is inside a macro, but the spelling of // the tag name comes from a macro argument, it looks like a special @@ -183,102 +180,73 @@ // attach the comment to the tag decl. if (SourceMgr.isMacroArgExpansion(DeclLoc) && TD->isCompleteDefinition()) - DeclLoc = SourceMgr.getExpansionLoc(DeclLoc); + return SourceMgr.getExpansionLoc(DeclLoc); } } + return DeclLoc; } + return {}; +} + +RawComment *ASTContext::getRawCommentForDeclNoCacheImpl( + const Decl *D, const SourceLocation RepresentativeLocForDecl, + const std::map &CommentsInTheFile) const { // If the declaration doesn't map directly to a location in a file, we // can't find the comment. - if (DeclLoc.isInvalid() || !DeclLoc.isFileID()) + if (RepresentativeLocForDecl.isInvalid() || + !RepresentativeLocForDecl.isFileID()) return nullptr; - if (!CommentsLoaded && ExternalSource) { - ExternalSource->ReadComments(); - -#ifndef NDEBUG - ArrayRef RawComments = Comments.getComments(); - assert(std::is_sorted(RawComments.begin(), RawComments.end(), - BeforeThanCompare(SourceMgr))); -#endif - - CommentsLoaded = true; - } - - ArrayRef RawComments = Comments.getComments(); // If there are no comments anywhere, we won't find anything. - if (RawComments.empty()) + if (CommentsInTheFile.empty()) return nullptr; - // Find the comment that occurs just after this declaration. - ArrayRef::iterator Comment; - { - // When searching for comments during parsing, the comment we are looking - // for is usually among the last two comments we parsed -- check them - // first. - RawComment CommentAtDeclLoc( - SourceMgr, SourceRange(DeclLoc), LangOpts.CommentOpts, false); - BeforeThanCompare Compare(SourceMgr); - ArrayRef::iterator MaybeBeforeDecl = RawComments.end() - 1; - bool Found = Compare(*MaybeBeforeDecl, &CommentAtDeclLoc); - if (!Found && RawComments.size() >= 2) { - MaybeBeforeDecl--; - Found = Compare(*MaybeBeforeDecl, &CommentAtDeclLoc); - } - - if (Found) { - Comment = MaybeBeforeDecl + 1; - assert(Comment == - llvm::lower_bound(RawComments, &CommentAtDeclLoc, Compare)); - } else { - // Slow path. - Comment = llvm::lower_bound(RawComments, &CommentAtDeclLoc, Compare); - } - } - // Decompose the location for the declaration and find the beginning of the // file buffer. - std::pair DeclLocDecomp = SourceMgr.getDecomposedLoc(DeclLoc); + const std::pair DeclLocDecomp = + SourceMgr.getDecomposedLoc(RepresentativeLocForDecl); + + // Slow path. + auto OffsetCommentBehindDecl = + CommentsInTheFile.lower_bound(DeclLocDecomp.second); // First check whether we have a trailing comment. - if (Comment != RawComments.end() && - ((*Comment)->isDocumentation() || LangOpts.CommentOpts.ParseAllComments) - && (*Comment)->isTrailingComment() && - (isa(D) || isa(D) || isa(D) || - isa(D) || isa(D))) { - std::pair CommentBeginDecomp - = SourceMgr.getDecomposedLoc((*Comment)->getSourceRange().getBegin()); - // Check that Doxygen trailing comment comes after the declaration, starts - // on the same line and in the same file as the declaration. - if (DeclLocDecomp.first == CommentBeginDecomp.first && - SourceMgr.getLineNumber(DeclLocDecomp.first, DeclLocDecomp.second) - == SourceMgr.getLineNumber(CommentBeginDecomp.first, - CommentBeginDecomp.second)) { - (**Comment).setAttached(); - return *Comment; + if (OffsetCommentBehindDecl != CommentsInTheFile.end()) { + RawComment *CommentBehindDecl = OffsetCommentBehindDecl->second; + if ((CommentBehindDecl->isDocumentation() || + LangOpts.CommentOpts.ParseAllComments) && + CommentBehindDecl->isTrailingComment() && + (isa(D) || isa(D) || isa(D) || + isa(D) || isa(D))) { + + // Check that Doxygen trailing comment comes after the declaration, starts + // on the same line and in the same file as the declaration. + if (SourceMgr.getLineNumber(DeclLocDecomp.first, DeclLocDecomp.second) == + Comments.getCommentBeginLine(CommentBehindDecl, DeclLocDecomp.first, + OffsetCommentBehindDecl->first)) { + return CommentBehindDecl; + } } } // The comment just after the declaration was not a trailing comment. // Let's look at the previous comment. - if (Comment == RawComments.begin()) + if (OffsetCommentBehindDecl == CommentsInTheFile.begin()) return nullptr; - --Comment; + + auto OffsetCommentBeforeDecl = --OffsetCommentBehindDecl; + RawComment *CommentBeforeDecl = OffsetCommentBeforeDecl->second; // Check that we actually have a non-member Doxygen comment. - if (!((*Comment)->isDocumentation() || + if (!(CommentBeforeDecl->isDocumentation() || LangOpts.CommentOpts.ParseAllComments) || - (*Comment)->isTrailingComment()) + CommentBeforeDecl->isTrailingComment()) return nullptr; // Decompose the end of the comment. - std::pair CommentEndDecomp - = SourceMgr.getDecomposedLoc((*Comment)->getSourceRange().getEnd()); - - // If the comment and the declaration aren't in the same file, then they - // aren't related. - if (DeclLocDecomp.first != CommentEndDecomp.first) - return nullptr; + const unsigned CommentEndOffset = + Comments.getCommentEndOffset(CommentBeforeDecl); // Get the corresponding buffer. bool Invalid = false; @@ -288,16 +256,41 @@ return nullptr; // Extract text between the comment and declaration. - StringRef Text(Buffer + CommentEndDecomp.second, - DeclLocDecomp.second - CommentEndDecomp.second); + StringRef Text(Buffer + CommentEndOffset, + DeclLocDecomp.second - CommentEndOffset); // There should be no other declarations or preprocessor directives between // comment and declaration. if (Text.find_first_of(";{}#@") != StringRef::npos) return nullptr; - (**Comment).setAttached(); - return *Comment; + return CommentBeforeDecl; +} + +RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const { + const SourceLocation DeclLoc = + getDeclLocForCommentSearch(D, SourceMgr); + + // If the declaration doesn't map directly to a location in a file, we + // can't find the comment. + if (DeclLoc.isInvalid() || !DeclLoc.isFileID()) + return nullptr; + + if (ExternalSource && !CommentsLoaded) { + ExternalSource->ReadComments(); + CommentsLoaded = true; + } + + if (Comments.empty()) + return nullptr; + + const FileID File = SourceMgr.getDecomposedLoc(DeclLoc).first; + const auto CommentsInThisFile = Comments.getCommentsInFile(File); + if (!CommentsInThisFile || CommentsInThisFile->empty()) + return nullptr; + + return getRawCommentForDeclNoCacheImpl(D, DeclLoc, + *CommentsInThisFile); } /// If we have a 'templated' declaration for a template, adjust 'D' to @@ -441,6 +434,47 @@ return RC; } +void ASTContext::cacheDocCommentDataForDecl( + const RawComment *Comment, const Decl *const OriginalDeclForComment) const { + assert(OriginalDeclForComment); + assert(!Comment || Comment->isDocumentation() || + LangOpts.CommentOpts.ParseAllComments); + if (Comment) { + RawCommentAndCacheFlags CachedComment; + CachedComment.setKind(RawCommentAndCacheFlags::FromDecl); + CachedComment.setRaw(Comment); + CachedComment.setOriginalDecl(OriginalDeclForComment); + RedeclComments[OriginalDeclForComment] = CachedComment; + } else if (RedeclComments.count(OriginalDeclForComment) == 0) { + RawCommentAndCacheFlags CachedAbsenceOfComment; + CachedAbsenceOfComment.setKind(RawCommentAndCacheFlags::NoCommentInDecl); + CachedAbsenceOfComment.setRaw(nullptr); + CachedAbsenceOfComment.setOriginalDecl(nullptr); + RedeclComments[OriginalDeclForComment] = CachedAbsenceOfComment; + } +} + +void ASTContext::cacheDocCommentDataForRedecls( + const Decl *const D, const RawComment *Comment, + const Decl *const OriginalDeclForComment) const { + assert(D); + assert(!Comment || Comment->isDocumentation() || + LangOpts.CommentOpts.ParseAllComments); + if (D->isInvalidDecl() || OriginalDeclForComment->isInvalidDecl()) + return; + // Update cache for redeclaration without a comment. + RawCommentAndCacheFlags CachedCommentFromOtherRedecl; + CachedCommentFromOtherRedecl.setRaw(Comment); + CachedCommentFromOtherRedecl.setKind(RawCommentAndCacheFlags::FromRedecl); + CachedCommentFromOtherRedecl.setOriginalDecl(OriginalDeclForComment); + + for (const auto Redecl : D->redecls()) { + if (RedeclComments[Redecl].getKind() == + RawCommentAndCacheFlags::NoCommentInDecl) + RedeclComments[Redecl] = CachedCommentFromOtherRedecl; + } +} + static void addRedeclaredMethods(const ObjCMethodDecl *ObjCMethod, SmallVectorImpl &Redeclared) { const DeclContext *DC = ObjCMethod->getDeclContext(); @@ -458,6 +492,45 @@ } } +void ASTContext::searchForDocComments(ArrayRef Decls, + const Preprocessor *PP) { + if (Comments.empty() || Decls.empty()) + return; + + const FileID File = + SourceMgr.getDecomposedLoc((*Decls.begin())->getLocation()).first; + const auto CommentsInThisFile = getRawCommentList().getCommentsInFile(File); + if (!CommentsInThisFile || CommentsInThisFile->empty()) + return; + + for (const Decl *D : Decls) { + assert(D); + if (D->isInvalidDecl()) + continue; + + D = adjustDeclToTemplate(D); + + const SourceLocation DeclLoc = getDeclLocForCommentSearch(D, SourceMgr); + + if (DeclLoc.isInvalid() || !DeclLoc.isFileID()) + continue; + + if (RedeclComments.count(D) > 0) + continue; + + RawComment *const DocComment = + getRawCommentForDeclNoCacheImpl(D, DeclLoc, *CommentsInThisFile); + cacheDocCommentDataForDecl(DocComment, D); + cacheDocCommentDataForRedecls(D, DocComment, D); + if (DocComment) { + comments::FullComment *FC = DocComment->parse(*this, PP, D); + const Decl *Canonical = D->getCanonicalDecl(); + assert(Canonical); + ParsedComments[Canonical] = FC; + } + } +} + comments::FullComment *ASTContext::cloneFullComment(comments::FullComment *FC, const Decl *D) const { auto *ThisDeclInfo = new (*this) comments::DeclInfo; Index: clang/lib/AST/RawCommentList.cpp =================================================================== --- clang/lib/AST/RawCommentList.cpp +++ clang/lib/AST/RawCommentList.cpp @@ -275,27 +275,25 @@ if (RC.isInvalid()) return; - // Check if the comments are not in source order. - while (!Comments.empty() && - !SourceMgr.isBeforeInTranslationUnit(Comments.back()->getBeginLoc(), - RC.getBeginLoc())) { - // If they are, just pop a few last comments that don't fit. - // This happens if an \#include directive contains comments. - Comments.pop_back(); - } - // Ordinary comments are not interesting for us. if (RC.isOrdinary() && !CommentOpts.ParseAllComments) return; + std::pair Loc = + SourceMgr.getDecomposedLoc(RC.getBeginLoc()); + + const FileID CommentFile = Loc.first; + const unsigned CommentOffset = Loc.second; + // If this is the first Doxygen comment, save it (because there isn't // anything to merge it with). - if (Comments.empty()) { - Comments.push_back(new (Allocator) RawComment(RC)); + if (OrderedComments[CommentFile].empty()) { + OrderedComments[CommentFile][CommentOffset] = + new (Allocator) RawComment(RC); return; } - const RawComment &C1 = *Comments.back(); + const RawComment &C1 = *OrderedComments[CommentFile].rbegin()->second; const RawComment &C2 = RC; // Merge comments only if there is only whitespace between them. @@ -318,21 +316,50 @@ onlyWhitespaceBetween(SourceMgr, C1.getEndLoc(), C2.getBeginLoc(), /*MaxNewlinesAllowed=*/1)) { SourceRange MergedRange(C1.getBeginLoc(), C2.getEndLoc()); - *Comments.back() = RawComment(SourceMgr, MergedRange, CommentOpts, true); + *OrderedComments[CommentFile].rbegin()->second = + RawComment(SourceMgr, MergedRange, CommentOpts, true); } else { - Comments.push_back(new (Allocator) RawComment(RC)); + OrderedComments[CommentFile][CommentOffset] = + new (Allocator) RawComment(RC); } } -void RawCommentList::addDeserializedComments(ArrayRef DeserializedComments) { - std::vector MergedComments; - MergedComments.reserve(Comments.size() + DeserializedComments.size()); +const std::map * +RawCommentList::getCommentsInFile(FileID File) const { + auto CommentsInFile = OrderedComments.find(File); + if (CommentsInFile == OrderedComments.end()) + return nullptr; + + return &CommentsInFile->second; +} + +bool RawCommentList::empty() const { return OrderedComments.empty(); } + +unsigned RawCommentList::getCommentBeginLine(RawComment *C, FileID File, + unsigned Offset) const { + auto Cached = CommentBeginLine.find(C); + if (Cached != CommentBeginLine.end()) + return Cached->second; + const unsigned Line = SourceMgr.getLineNumber(File, Offset); + CommentBeginLine[C] = Line; + return Line; +} + +unsigned RawCommentList::getCommentEndOffset(RawComment *C) const { + auto Cached = CommentEndOffset.find(C); + if (Cached != CommentEndOffset.end()) + return Cached->second; + const unsigned Offset = + SourceMgr.getDecomposedLoc(C->getSourceRange().getEnd()).second; + CommentEndOffset[C] = Offset; + return Offset; +} - std::merge(Comments.begin(), Comments.end(), - DeserializedComments.begin(), DeserializedComments.end(), - std::back_inserter(MergedComments), - BeforeThanCompare(SourceMgr)); - std::swap(Comments, MergedComments); +void RawCommentList::addDeserializedComments( + const llvm::DenseMap> + &DeserializedComments) { + for (const auto &F : DeserializedComments) + OrderedComments[F.first].insert(F.second.begin(), F.second.end()); } std::string RawComment::getFormattedText(const SourceManager &SourceMgr, Index: clang/lib/Sema/SemaDecl.cpp =================================================================== --- clang/lib/Sema/SemaDecl.cpp +++ clang/lib/Sema/SemaDecl.cpp @@ -12727,18 +12727,23 @@ } // See if there are any new comments that are not attached to a decl. - ArrayRef Comments = Context.getRawCommentList().getComments(); - if (!Comments.empty() && - !Comments.back()->isAttached()) { - // There is at least one comment that not attached to a decl. + // The location doesn't have to be precise - we care only about the file. + // FIMXE: We assume every Decl in the group is in the same file. + // This is false when preprocessor constructs the group from decls in + // different files (e. g. macros or #include). + const FileID File = + SourceMgr.getDecomposedLoc((*Group.begin())->getLocation()).first; + auto CommentsInThisFile = Context.getRawCommentList().getCommentsInFile(File); + if (CommentsInThisFile && !CommentsInThisFile->empty() && + !CommentsInThisFile->rbegin()->second->isAttached()) { + // There is at least one comment not attached to a decl. // Maybe it should be attached to one of these decls? // // Note that this way we pick up not only comments that precede the // declaration, but also comments that *follow* the declaration -- thanks to // the lookahead in the lexer: we've consumed the semicolon and looked // ahead through comments. - for (unsigned i = 0, e = Group.size(); i != e; ++i) - Context.getCommentForDecl(Group[i], &PP); + Context.searchForDocComments(Group, &getPreprocessor()); } } Index: clang/lib/Serialization/ASTReader.cpp =================================================================== --- clang/lib/Serialization/ASTReader.cpp +++ clang/lib/Serialization/ASTReader.cpp @@ -9718,10 +9718,19 @@ } } NextCursor: - // De-serialized SourceLocations get negative FileIDs for other modules, - // potentially invalidating the original order. Sort it again. - llvm::sort(Comments, BeforeThanCompare(SourceMgr)); - Context.Comments.addDeserializedComments(Comments); + llvm::DenseMap> + FileToOffsetToComment; + for (RawComment *C : Comments) { + SourceLocation CommentLoc = C->getBeginLoc(); + if (CommentLoc.isValid()) { + std::pair Loc = + SourceMgr.getDecomposedLoc(CommentLoc); + if (Loc.first.isValid()) + FileToOffsetToComment[Loc.first][Loc.second] = C; + } + } + + Context.Comments.addDeserializedComments(FileToOffsetToComment); } } Index: clang/lib/Serialization/ASTWriter.cpp =================================================================== --- clang/lib/Serialization/ASTWriter.cpp +++ clang/lib/Serialization/ASTWriter.cpp @@ -3266,15 +3266,17 @@ auto _ = llvm::make_scope_exit([this] { Stream.ExitBlock(); }); if (!PP->getPreprocessorOpts().WriteCommentListToPCH) return; - ArrayRef RawComments = Context->Comments.getComments(); RecordData Record; - for (const auto *I : RawComments) { - Record.clear(); - AddSourceRange(I->getSourceRange(), Record); - Record.push_back(I->getKind()); - Record.push_back(I->isTrailingComment()); - Record.push_back(I->isAlmostTrailingComment()); - Stream.EmitRecord(COMMENTS_RAW_COMMENT, Record); + for (const auto &FO : Context->Comments.OrderedComments) { + for (const auto &OC : FO.second) { + const RawComment *I = OC.second; + Record.clear(); + AddSourceRange(I->getSourceRange(), Record); + Record.push_back(I->getKind()); + Record.push_back(I->isTrailingComment()); + Record.push_back(I->isAlmostTrailingComment()); + Stream.EmitRecord(COMMENTS_RAW_COMMENT, Record); + } } }