[clangd] Add doxygen parsing for Hover

[tom-anders]

Summary

This adds a new class SymbolDocumentation that does basic doxygen
parsing. Currently, it's only consumed by Hover, but in future
patches we'll probably also want to use it in CodeComplete and
SignatureHelp.

[tom-anders]

Ping

[logankaser]

Is there anything I can do as a random member of the public that wants this and knows C++?
My organization is trying to move to clangd for the default config, and the lack of docstring parsing is common feedback.

[nridge]

In D131853#3792985, @logankaser wrote:

Is there anything I can do as a random member of the public that wants this and knows C++?

Maybe apply the patch locally, use it for a bit, and provide feedback?

I haven't forgotten about this review and I hope to get to it as time permits.

[tom-anders]

In D131853#3793831, @nridge wrote:

In D131853#3792985, @logankaser wrote:

Is there anything I can do as a random member of the public that wants this and knows C++?

Maybe apply the patch locally, use it for a bit, and provide feedback?

I'm sure that would help!

I haven't forgotten about this review and I hope to get to it as time permits.

Would it maybe also help to split this patch into 2 or 3 smaller patches?

[tom-anders]

Split up into 3 smaller patches: https://reviews.llvm.org/D134130, https://reviews.llvm.org/D134131 and https://reviews.llvm.org/D134132

clang-tools-extra/clangd/CMakeLists.txt

add_clang_library(clangDaemon
   Quality.cpp
   ParsedAST.cpp
   Preamble.cpp
   RIFF.cpp
   Selection.cpp
   SemanticHighlighting.cpp
   SemanticSelection.cpp
   SourceCode.cpp
+  SymbolDocumentation.cpp
   QueryDriverDatabase.cpp
   TidyProvider.cpp
   TUScheduler.cpp
   URI.cpp
   XRefs.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp
 
   index/Background.cpp

clang-tools-extra/clangd/CodeComplete.cpp

void add(const CompletionCandidate &C, CodeCompletionString *SemaCCS) {
     if (!Completion.Documentation) {
       auto SetDoc = [&](llvm::StringRef Doc) {
         if (!Doc.empty()) {
           Completion.Documentation.emplace();
           parseDocumentation(Doc, *Completion.Documentation);
         }
       };
       if (C.IndexResult) {
-        SetDoc(C.IndexResult->Documentation);
+        SetDoc(C.IndexResult->Documentation.CommentText);
       } else if (C.SemaResult) {
-        const auto DocComment = getDocComment(*ASTCtx, *C.SemaResult,
+        const auto DocComment = getDocumentation(*ASTCtx, *C.SemaResult,
                                                  /*CommentsFromHeaders=*/false);
-        SetDoc(formatDocumentation(*SemaCCS, DocComment));
+        SetDoc(formatDocumentation(*SemaCCS, DocComment.CommentText));
       }
     }
     if (Completion.Deprecated) {
       if (C.SemaResult)
         Completion.Deprecated &=
             C.SemaResult->Availability == CXAvailability_Deprecated;
       if (C.IndexResult)
         Completion.Deprecated &=
for (unsigned I = 0; I < NumCandidates; ++I) {
 
       const auto *CCS = Candidate.CreateSignatureString(
           CurrentArg, S, *Allocator, CCTUInfo,
           /*IncludeBriefComments=*/true, Braced);
       assert(CCS && "Expected the CodeCompletionString to be non-null");
       ScoredSignatures.push_back(processOverloadCandidate(
           Candidate, *CCS,
           Candidate.getFunction()
-              ? getDeclComment(S.getASTContext(), *Candidate.getFunction())
-              : ""));
+              ? getDeclDocumentation(S.getASTContext(),
+                                     *Candidate.getFunction())
+              : SymbolDocumentationOwned{}));
     }
 
     // Sema does not load the docs from the preamble, so we need to fetch extra
     // docs from the index instead.
     llvm::DenseMap<SymbolID, std::string> FetchedDocs;
     if (Index) {
       LookupRequest IndexRequest;
       for (const auto &S : ScoredSignatures) {
         if (!S.IDForDoc)
           continue;
         IndexRequest.IDs.insert(S.IDForDoc);
       }
       Index->lookup(IndexRequest, [&](const Symbol &S) {
         if (!S.Documentation.empty())
-          FetchedDocs[S.ID] = std::string(S.Documentation);
+          FetchedDocs[S.ID] = std::string(S.Documentation.CommentText);
       });
       vlog("SigHelp: requested docs for {0} symbols from the index, got {1} "
            "symbols with non-empty docs in the response",
            IndexRequest.IDs.size(), FetchedDocs.size());
     }
 
     llvm::sort(ScoredSignatures, [](const ScoredSignature &L,
                                     const ScoredSignature &R) {
for (const auto &Chunk : CCS) {
         Signature.label += Chunk.Text;
         break;
       }
     }
   }
 
   // FIXME(ioeric): consider moving CodeCompletionString logic here to
   // CompletionString.h.
-  ScoredSignature processOverloadCandidate(const OverloadCandidate &Candidate,
-                                           const CodeCompletionString &CCS,
-                                           llvm::StringRef DocComment) const {
+  ScoredSignature
+  processOverloadCandidate(const OverloadCandidate &Candidate,
+                           const SymbolDocumentationOwned &DocComment) const {
     SignatureInformation Signature;
     SignatureQualitySignals Signal;
     const char *ReturnType = nullptr;
 
     markup::Document OverloadComment;
-    parseDocumentation(formatDocumentation(CCS, DocComment), OverloadComment);
+    parseDocumentation(formatDocumentation(CCS, DocComment.CommentText),
+                       OverloadComment);
     Signature.documentation = renderDoc(OverloadComment, DocumentationFormat);
     Signal.Kind = Candidate.getKind();
 
     for (const auto &Chunk : CCS) {
       switch (Chunk.Kind) {
       case CodeCompletionString::CK_ResultType:
         // A piece of text that describes the type of an entity or,
         // for functions and methods, the return type.

clang-tools-extra/clangd/CodeCompletionStrings.h

 //===--- CodeCompletionStrings.h ---------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Functions for retrieving code completion information from
 // `CodeCompletionString`.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETIONSTRINGS_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CODECOMPLETIONSTRINGS_H
 
+#include "clang/AST/CommentVisitor.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 
+#include "SymbolDocumentation.h"
+
 namespace clang {
 class ASTContext;
 
 namespace clangd {
 
-/// Gets a minimally formatted documentation comment of \p Result, with comment
-/// markers stripped. See clang::RawComment::getFormattedText() for the detailed
-/// explanation of how the comment text is transformed.
-/// Returns empty string when no comment is available.
+/// Gets the parsed doxygen documentation of \p Result.
+/// Returns an empty SymbolDocumentationOwned when no comment is available.
 /// If \p CommentsFromHeaders parameter is set, only comments from the main
 /// file will be returned. It is used to workaround crashes when parsing
 /// comments in the stale headers, coming from completion preamble.
-std::string getDocComment(const ASTContext &Ctx,
+SymbolDocumentationOwned getDocumentation(const ASTContext &Ctx,
                                           const CodeCompletionResult &Result,
                                           bool CommentsFromHeaders);
 
-/// Similar to getDocComment, but returns the comment for a NamedDecl.
-std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &D);
+/// Similar to getDocumentation, but returns the comment for a NamedDecl.
+SymbolDocumentationOwned getDeclDocumentation(const ASTContext &Ctx,
+                                              const NamedDecl &D);
 
 /// Formats the signature for an item, as a display string and snippet.
 /// e.g. for const_reference std::vector<T>::at(size_type) const, this returns:
 ///   *Signature = "(size_type) const"
 ///   *Snippet = "(${1:size_type})"
 /// If set, RequiredQualifiers is the text that must be typed before the name.
 /// e.g "Base::" when calling a base class member function that's hidden.
 ///

clang-tools-extra/clangd/CodeCompletionStrings.cpp

bool looksLikeDocComment(llvm::StringRef CommentText) {
   //   =================
   //   -----------------
   //   *****************
   return CommentText.find_first_not_of("/*-= \t\r\n") != llvm::StringRef::npos;
 }
 
 } // namespace
 
-std::string getDocComment(const ASTContext &Ctx,
+SymbolDocumentationOwned getDocumentation(const ASTContext &Ctx,
                                           const CodeCompletionResult &Result,
                                           bool CommentsFromHeaders) {
+  // FIXME: CommentsFromHeaders seems to be unused? Is this a bug?
+
   // FIXME: clang's completion also returns documentation for RK_Pattern if they
   // contain a pattern for ObjC properties. Unfortunately, there is no API to
   // get this declaration, so we don't show documentation in that case.
   if (Result.Kind != CodeCompletionResult::RK_Declaration)
-    return "";
-  return Result.getDeclaration() ? getDeclComment(Ctx, *Result.getDeclaration())
-                                 : "";
+    return {};
+  return Result.getDeclaration()
+             ? getDeclDocumentation(Ctx, *Result.getDeclaration())
+             : SymbolDocumentationOwned{};
 }
 
-std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) {
+SymbolDocumentationOwned getDeclDocumentation(const ASTContext &Ctx,
+                                              const NamedDecl &Decl) {
   if (isa<NamespaceDecl>(Decl)) {
     // Namespaces often have too many redecls for any particular redecl comment
     // to be useful. Moreover, we often confuse file headers or generated
     // comments with namespace comments. Therefore we choose to just ignore
     // the comments for namespaces.
-    return "";
+    return {};
   }
   const RawComment *RC = getCompletionComment(Ctx, &Decl);
   if (!RC)
-    return "";
+    return {};
   // Sanity check that the comment does not come from the PCH. We choose to not
   // write them into PCH, because they are racy and slow to load.
   assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
-  std::string Doc =
-      RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
-  if (!looksLikeDocComment(Doc))
-    return "";
-  // Clang requires source to be UTF-8, but doesn't enforce this in comments.
-  if (!llvm::json::isUTF8(Doc))
-    Doc = llvm::json::fixUTF8(Doc);
+
+  SymbolDocumentationOwned Doc = parseDoxygenComment(*RC, Ctx, &Decl);
+
+  if (!looksLikeDocComment(Doc.CommentText))
+    return {};
+
   return Doc;
 }
 
 void getSignature(const CodeCompletionString &CCS, std::string *Signature,
                   std::string *Snippet, std::string *RequiredQualifiers,
                   bool CompletingPattern) {
   // Placeholder with this index will be ${0:…} to mark final cursor position.
   // Usually we do not add $0, so the cursor is placed at end of completed text.

clang-tools-extra/clangd/Hover.h

 //===--- Hover.h - Information about code at the cursor location -*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_HOVER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_HOVER_H
 
 #include "ParsedAST.h"
 #include "Protocol.h"
+#include "SymbolDocumentation.h"
 #include "support/Markup.h"
 #include "clang/Index/IndexSymbol.h"
 
 namespace clang {
 namespace clangd {
 
 /// Contains detailed information about a Symbol. Especially useful when
 /// generating hover responses. It can be rendered as a hover panel, or
struct HoverInfo {
   llvm::Optional<std::string> NamespaceScope;
   /// Remaining named contexts in symbol's qualified name, empty string means
   /// symbol is not local.
   std::string LocalScope;
   /// Name of the symbol, does not contain any "::".
   std::string Name;
   llvm::Optional<Range> SymRange;
   index::SymbolKind Kind = index::SymbolKind::Unknown;
-  std::string Documentation;
+  SymbolDocumentationOwned Documentation;
   /// Source code containing the definition of the symbol.
   std::string Definition;
   const char *DefinitionLanguage = "cpp";
   /// Access specifier for declarations inside class/struct/unions, empty for
   /// others.
   std::string AccessSpecifier;
   /// Printable variable type.
   /// Set only for variables.

clang-tools-extra/clangd/Hover.cpp

if (!SymbolCollector::shouldCollectSymbol(ND, ND.getASTContext(),
                                             /*IsMainFileOnly=*/false))
     return;
   auto ID = getSymbolID(&ND);
   if (!ID)
     return;
   LookupRequest Req;
   Req.IDs.insert(ID);
   Index->lookup(Req, [&](const Symbol &S) {
-    Hover.Documentation = std::string(S.Documentation);
+    Hover.Documentation = S.Documentation.toOwned();
   });
 }
 
 // Default argument might exist but be unavailable, in the case of unparsed
 // arguments for example. This function returns the default argument if it is
 // available.
 const Expr *getDefaultArg(const ParmVarDecl *PVD) {
   // Default argument can be unparsed or uninstantiated. For the former we
HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP,
   if (!HI.NamespaceScope->empty())
     HI.NamespaceScope->append("::");
   HI.LocalScope = getLocalScope(D);
   if (!HI.LocalScope.empty())
     HI.LocalScope.append("::");
 
   HI.Name = printName(Ctx, *D);
   const auto *CommentD = getDeclForComment(D);
-  HI.Documentation = getDeclComment(Ctx, *CommentD);
+  HI.Documentation = getDeclDocumentation(Ctx, *CommentD);
   enhanceFromIndex(HI, *CommentD, Index);
   if (HI.Documentation.empty())
-    HI.Documentation = synthesizeDocumentation(D);
+    HI.Documentation =
+        SymbolDocumentationOwned::descriptionOnly(synthesizeDocumentation(D));
 
   HI.Kind = index::getSymbolInfo(D).Kind;
 
   // Fill in template params.
   if (const TemplateDecl *TD = D->getDescribedTemplate()) {
     HI.TemplateParameters =
         fetchTemplateParameters(TD->getTemplateParameters(), PP);
     D = TD;
HoverInfo getDeducedTypeHoverContents(QualType QT, const syntax::Token &Tok,
 
   if (QT->isUndeducedAutoType()) {
     HI.Definition = "/* not deduced */";
   } else {
     HI.Definition = typeAsDefinition(printType(QT, ASTCtx, PP));
 
     if (const auto *D = QT->getAsTagDecl()) {
       const auto *CommentD = getDeclForComment(D);
-      HI.Documentation = getDeclComment(ASTCtx, *CommentD);
+      HI.Documentation = getDeclDocumentation(ASTCtx, *CommentD);
       enhanceFromIndex(HI, *CommentD, Index);
     }
   }
 
   return HI;
 }
 
 bool isLiteral(const Expr *E) {
llvm::Optional<HoverInfo> getHoverContents(const Attr *A, ParsedAST &AST) {
   HoverInfo HI;
   HI.Name = A->getSpelling();
   if (A->hasScope())
     HI.LocalScope = A->getScopeName()->getName().str();
   {
     llvm::raw_string_ostream OS(HI.Definition);
     A->printPretty(OS, AST.getASTContext().getPrintingPolicy());
   }
-  HI.Documentation = Attr::getDocumentation(A->getKind()).str();
+  HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+      Attr::getDocumentation(A->getKind()).str());
   return HI;
 }
 
 bool isParagraphBreak(llvm::StringRef Rest) {
   return Rest.ltrim(" \t").startswith("\n");
 }
 
 bool punctuationIndicatesLineBreak(llvm::StringRef Line) {
markup::Document HoverInfo::present() const {
   markup::Paragraph &Header = Output.addHeading(3);
   if (Kind != index::SymbolKind::Unknown)
     Header.appendText(index::getSymbolKindString(Kind)).appendSpace();
   assert(!Name.empty() && "hover triggered on a nameless symbol");
   Header.appendCode(Name);
 
   // Put a linebreak after header to increase readability.
   Output.addRuler();
+
+  if (!Documentation.Brief.empty())
+    parseDocumentation(Documentation.Brief, Output);
+
   // Print Types on their own lines to reduce chances of getting line-wrapped by
   // editor, as they might be long.
   if (ReturnType) {
     // For functions we display signature in a list form, e.g.:
     // → `x`
     // Parameters:
     // - `bool param1`
     // - `int param2 = 5`
-    Output.addParagraph().appendText("→ ").appendCode(
+    auto &P = Output.addParagraph().appendText("→ ").appendCode(
         llvm::to_string(*ReturnType));
-  }
 
+    if (!Documentation.Returns.empty())
+      P.appendText(": ").appendText(Documentation.Returns);
+  }
   if (Parameters && !Parameters->empty()) {
     Output.addParagraph().appendText("Parameters: ");
     markup::BulletList &L = Output.addBulletList();
-    for (const auto &Param : *Parameters)
-      L.addItem().addParagraph().appendCode(llvm::to_string(Param));
+
+    llvm::SmallVector<ParameterDocumentationOwned> ParamDocs =
+        Documentation.Parameters;
+
+    for (const auto &Param : *Parameters) {
+      auto &Paragraph = L.addItem().addParagraph();
+      Paragraph.appendCode(llvm::to_string(Param));
+
+      if (Param.Name.has_value()) {
+        auto ParamDoc = std::find_if(ParamDocs.begin(), ParamDocs.end(),
+                                     [Param](const auto &ParamDoc) {
+                                       return Param.Name == ParamDoc.Name;
+                                     });
+        if (ParamDoc != ParamDocs.end()) {
+          Paragraph.appendText(": ").appendText(ParamDoc->Description);
+          ParamDocs.erase(ParamDoc);
+        }
+      }
+    }
+
+    // We erased all parameters that matched, but some may still be left,
+    // usually typos. Let's also print them here.
+    for (const auto &ParamDoc : ParamDocs) {
+      L.addItem()
+          .addParagraph()
+          .appendCode(ParamDoc.Name)
+          .appendText(": ")
+          .appendText(ParamDoc.Description);
+    }
   }
 
   // Don't print Type after Parameters or ReturnType as this will just duplicate
   // the information
   if (Type && !ReturnType && !Parameters)
     Output.addParagraph().appendText("Type: ").appendCode(
         llvm::to_string(*Type));
 
if (CalleeArgInfo) {
     }
     if (CalleeArgInfo->Name)
       OS << "as " << CalleeArgInfo->Name;
     if (CallPassType->Converted && CalleeArgInfo->Type)
       OS << " (converted to " << CalleeArgInfo->Type->Type << ")";
     Output.addParagraph().appendText(OS.str());
   }
 
-  if (!Documentation.empty())
-    parseDocumentation(Documentation, Output);
+  if (!Documentation.Description.empty())
+    parseDocumentation(Documentation.Description, Output);
+
+  if (!Documentation.Warnings.empty()) {
+    Output.addRuler();
+    Output.addParagraph()
+        .appendText("Warning")
+        .appendText(Documentation.Warnings.size() > 1 ? "s" : "")
+        .appendText(": ");
+    markup::BulletList &L = Output.addBulletList();
+    for (const auto &Warning : Documentation.Warnings)
+      parseDocumentation(Warning, L.addItem());
+  }
+
+  if (!Documentation.Notes.empty()) {
+    Output.addRuler();
+    Output.addParagraph()
+        .appendText("Note")
+        .appendText(Documentation.Notes.size() > 1 ? "s" : "")
+        .appendText(": ");
+    markup::BulletList &L = Output.addBulletList();
+    for (const auto &Note : Documentation.Notes)
+      parseDocumentation(Note, L.addItem());
+  }
 
   if (!Definition.empty()) {
     Output.addRuler();
     std::string ScopeComment;
     // Drop trailing "::".
     if (!LocalScope.empty()) {
       // Container name, e.g. class, method, function.
       // We might want to propagate some info about container type to print

clang-tools-extra/clangd/SymbolDocumentation.h

+//===--- SymbolDocumentation.h ==---------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Class to parse doxygen comments into a flat structure for consumption
+// in e.g. Hover and Code Completion
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Comment.h"
+#include "clang/AST/CommentVisitor.h"
+
+namespace clang {
+namespace clangd {
+
+template <class String> struct ParameterDocumentation {
+  String Name;
+  String Description;
+
+  ParameterDocumentation<llvm::StringRef> toRef() const;
+  ParameterDocumentation<std::string> toOwned() const;
+};
+
+using ParameterDocumentationRef = ParameterDocumentation<llvm::StringRef>;
+using ParameterDocumentationOwned = ParameterDocumentation<std::string>;
+
+/// @brief Represents a parsed doxygen comment.
+/// @details Currently there's special handling for the "brief", "param"
+/// "returns", "note" and "warning" commands. The content of all other
+/// paragraphs will be appended to the #Description field.
+/// If you're only interested in the full comment, but with comment
+/// markers stripped, use the #CommentText field.
+/// \tparam String When built from a declaration, we're building the strings
+/// by ourselves, so in this case String==std::string.
+/// However, when storing the contents of this class in the index, we need to
+/// use llvm::StringRef. To connvert between std::string and llvm::StringRef
+/// versions of this class, use toRef() and toOwned().
+template <class String> class SymbolDocumentation {
+public:
+  friend class CommentToSymbolDocumentation;
+
+  static SymbolDocumentation<String> descriptionOnly(String &&Description) {
+    SymbolDocumentation<String> Doc;
+    Doc.Description = Description;
+    Doc.CommentText = Description;
+    return Doc;
+  }
+
+  /// Constructs with all fields as empty strings/vectors.
+  SymbolDocumentation() = default;
+
+  SymbolDocumentation<llvm::StringRef> toRef() const;
+  SymbolDocumentation<std::string> toOwned() const;
+
+  bool empty() const { return CommentText.empty(); }
+
+  /// Paragraph of the "brief" command.
+  String Brief;
+
+  /// Paragraph of the "return" command.
+  String Returns;
+
+  /// Paragraph(s) of the "note" command(s)
+  llvm::SmallVector<String, 1> Notes;
+  /// Paragraph(s) of the "warning" command(s)
+  llvm::SmallVector<String, 1> Warnings;
+
+  /// Parsed paragaph(s) of the "param" comamnd(s)
+  llvm::SmallVector<ParameterDocumentation<String>> Parameters;
+
+  /// All the paragraphs we don't have any special handling for,
+  /// e.g. "details".
+  String Description;
+
+  /// The full documentation comment with comment markers stripped.
+  /// See clang::RawComment::getFormattedText() for the detailed
+  /// explanation of how the comment text is transformed.
+  String CommentText;
+};
+
+using SymbolDocumentationOwned = SymbolDocumentation<std::string>;
+using SymbolDocumentationRef = SymbolDocumentation<llvm::StringRef>;
+
+/// @param RC the comment to parse
+/// @param D the declaration that \p RC belongs to
+/// @return parsed doxgen documentation.
+SymbolDocumentationOwned
+parseDoxygenComment(const RawComment &RC, const ASTContext &Ctx, const Decl *D);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H

clang-tools-extra/clangd/SymbolDocumentation.cpp

+//===--- SymbolDocumentation.cpp ==-------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolDocumentation.h"
+#include "clang/AST/CommentVisitor.h"
+#include "llvm/Support/JSON.h"
+
+namespace clang {
+namespace clangd {
+
+void ensureUTF8(std::string &Str) {
+  if (!llvm::json::isUTF8(Str))
+    Str = llvm::json::fixUTF8(Str);
+}
+
+class BlockCommentToString
+    : public comments::ConstCommentVisitor<BlockCommentToString> {
+public:
+  BlockCommentToString(std::string &Out, const ASTContext &Ctx)
+      : Out(Out), Ctx(Ctx) {}
+
+  void visitParagraphComment(const comments::ParagraphComment *C) {
+    for (const auto *Child = C->child_begin(); Child != C->child_end();
+         ++Child) {
+      visit(*Child);
+    }
+  }
+
+  void visitBlockCommandComment(const comments::BlockCommandComment *B) {
+    Out << (B->getCommandMarker() == (comments::CommandMarkerKind::CMK_At)
+                ? '@'
+                : '\\')
+        << B->getCommandName(Ctx.getCommentCommandTraits());
+
+    visit(B->getParagraph());
+  }
+
+  void visitTextComment(const comments::TextComment *C) {
+    // If this is the very first node, the paragraph has no doxygen command,
+    // so there will be a leading space -> Trim it
+    // Otherwise just trim trailing space
+    if (Out.str().empty())
+      Out << C->getText().trim();
+    else
+      Out << C->getText().rtrim();
+  }
+
+  void visitInlineCommandComment(const comments::InlineCommandComment *C) {
+    const std::string SurroundWith = [C] {
+      switch (C->getRenderKind()) {
+      case comments::InlineCommandComment::RenderKind::RenderMonospaced:
+        return "`";
+      case comments::InlineCommandComment::RenderKind::RenderBold:
+        return "**";
+      case comments::InlineCommandComment::RenderKind::RenderEmphasized:
+        return "*";
+      default:
+        return "";
+      }
+    }();
+
+    Out << " " << SurroundWith;
+    for (unsigned I = 0; I < C->getNumArgs(); ++I) {
+      Out << C->getArgText(I);
+    }
+    Out << SurroundWith;
+  }
+
+private:
+  llvm::raw_string_ostream Out;
+  const ASTContext &Ctx;
+};
+
+class CommentToSymbolDocumentation
+    : public comments::ConstCommentVisitor<CommentToSymbolDocumentation> {
+public:
+  CommentToSymbolDocumentation(const RawComment &RC, const ASTContext &Ctx,
+                               const Decl *D, SymbolDocumentationOwned &Doc)
+      : FullComment(RC.parse(Ctx, nullptr, D)), Output(Doc), Ctx(Ctx) {
+
+    Doc.CommentText =
+        RC.getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+
+    for (auto *Block : FullComment->getBlocks()) {
+      visit(Block);
+    }
+  }
+
+  void visitBlockCommandComment(const comments::BlockCommandComment *B) {
+    const llvm::StringRef CommandName =
+        B->getCommandName(Ctx.getCommentCommandTraits());
+
+    // Visit B->getParagraph() for commands that we have special fields for,
+    // so that the command name won't be included in the string.
+    // Otherwise, we want to keep the command name, so visit B itself.
+    if (CommandName == "brief") {
+      BlockCommentToString(Output.Brief, Ctx).visit(B->getParagraph());
+    } else if (CommandName == "return") {
+      BlockCommentToString(Output.Returns, Ctx).visit(B->getParagraph());
+    } else if (CommandName == "warning") {
+      BlockCommentToString(Output.Warnings.emplace_back(), Ctx)
+          .visit(B->getParagraph());
+    } else if (CommandName == "note") {
+      BlockCommentToString(Output.Notes.emplace_back(), Ctx)
+          .visit(B->getParagraph());
+    } else {
+      if (!Output.Description.empty())
+        Output.Description += "\n\n";
+
+      BlockCommentToString(Output.Description, Ctx).visit(B);
+    }
+  }
+
+  void visitParagraphComment(const comments::ParagraphComment *P) {
+    BlockCommentToString(Output.Description, Ctx).visit(P);
+  }
+
+  void visitParamCommandComment(const comments::ParamCommandComment *P) {
+    if (P->hasParamName() && P->hasNonWhitespaceParagraph()) {
+      ParameterDocumentationOwned Doc;
+      Doc.Name = P->getParamNameAsWritten().str();
+      BlockCommentToString(Doc.Description, Ctx).visit(P->getParagraph());
+      Output.Parameters.push_back(std::move(Doc));
+    }
+  }
+
+private:
+  comments::FullComment *FullComment;
+  SymbolDocumentationOwned &Output;
+  const ASTContext &Ctx;
+};
+
+SymbolDocumentationOwned parseDoxygenComment(const RawComment &RC,
+                                             const ASTContext &Ctx,
+                                             const Decl *D) {
+  SymbolDocumentationOwned Doc;
+  CommentToSymbolDocumentation(RC, Ctx, D, Doc);
+
+  // Clang requires source to be UTF-8, but doesn't enforce this in comments.
+  ensureUTF8(Doc.Brief);
+  ensureUTF8(Doc.Returns);
+  for (auto &Note : Doc.Notes)
+    ensureUTF8(Note);
+  for (auto &Warning : Doc.Warnings)
+    ensureUTF8(Warning);
+  for (auto &Param : Doc.Parameters) {
+    ensureUTF8(Param.Name);
+    ensureUTF8(Param.Description);
+  }
+  ensureUTF8(Doc.Description);
+  ensureUTF8(Doc.CommentText);
+
+  return Doc;
+}
+
+template struct ParameterDocumentation<std::string>;
+template struct ParameterDocumentation<llvm::StringRef>;
+
+template <class S1, class S2>
+SymbolDocumentation<S1> convert(const SymbolDocumentation<S2> &In) {
+  SymbolDocumentation<S1> Doc;
+
+  Doc.Brief = In.Brief;
+  Doc.Returns = In.Returns;
+
+  Doc.Notes.reserve(In.Notes.size());
+  for (const auto &Note : In.Notes) {
+    Doc.Notes.emplace_back(Note);
+  }
+
+  Doc.Warnings.reserve(In.Warnings.size());
+  for (const auto &Warning : In.Warnings) {
+    Doc.Warnings.emplace_back(Warning);
+  }
+
+  Doc.Parameters.reserve(In.Parameters.size());
+  for (const auto &ParamDoc : In.Parameters) {
+    Doc.Parameters.emplace_back(ParameterDocumentation<S1>{
+        S1(ParamDoc.Name), S1(ParamDoc.Description)});
+  }
+
+  Doc.Description = In.Description;
+  Doc.CommentText = In.CommentText;
+
+  return Doc;
+}
+
+template <> SymbolDocumentationRef SymbolDocumentationOwned::toRef() const {
+  return convert<llvm::StringRef>(*this);
+}
+
+template <> SymbolDocumentationOwned SymbolDocumentationRef::toOwned() const {
+  return convert<std::string>(*this);
+}
+
+template class SymbolDocumentation<std::string>;
+template class SymbolDocumentation<llvm::StringRef>;
+
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/index/Merge.cpp

if (prefer(O.CanonicalDeclaration, S.CanonicalDeclaration))
     S.CanonicalDeclaration = O.CanonicalDeclaration;
   if (prefer(O.Definition, S.Definition))
     S.Definition = O.Definition;
   S.References += O.References;
   if (S.Signature == "")
     S.Signature = O.Signature;
   if (S.CompletionSnippetSuffix == "")
     S.CompletionSnippetSuffix = O.CompletionSnippetSuffix;
-  if (S.Documentation == "") {
+  if (S.Documentation.empty()) {
     // Don't accept documentation from bare forward class declarations, if there
     // is a definition and it didn't provide one. S is often an undocumented
     // class, and O is a non-canonical forward decl preceded by an irrelevant
     // comment.
     bool IsClass = S.SymInfo.Kind == index::SymbolKind::Class ||
                    S.SymInfo.Kind == index::SymbolKind::Struct ||
                    S.SymInfo.Kind == index::SymbolKind::Union;
     if (!IsClass || !S.Definition)

clang-tools-extra/clangd/index/Serialization.cpp

SymbolLocation readLocation(Reader &Data,
   Loc.FileURI = Data.consumeString(Strings).data();
   for (auto *Endpoint : {&Loc.Start, &Loc.End}) {
     Endpoint->setLine(Data.consumeVar());
     Endpoint->setColumn(Data.consumeVar());
   }
   return Loc;
 }
 
+void writeSymbolDocumentation(const SymbolDocumentationRef &Doc,
+                              const StringTableOut &Strings,
+                              llvm::raw_ostream &OS) {
+  writeVar(Strings.index(Doc.Brief), OS);
+  writeVar(Strings.index(Doc.Returns), OS);
+
+  writeVar(Doc.Notes.size(), OS);
+  for (const auto &Note : Doc.Notes)
+    writeVar(Strings.index(Note), OS);
+
+  writeVar(Doc.Warnings.size(), OS);
+  for (const auto &Warning : Doc.Warnings)
+    writeVar(Strings.index(Warning), OS);
+
+  writeVar(Doc.Parameters.size(), OS);
+  for (const auto &ParamDoc : Doc.Parameters) {
+    writeVar(Strings.index(ParamDoc.Name), OS);
+    writeVar(Strings.index(ParamDoc.Description), OS);
+  }
+
+  writeVar(Strings.index(Doc.Description), OS);
+  writeVar(Strings.index(Doc.CommentText), OS);
+}
+
+SymbolDocumentationRef
+readSymbolDocumentation(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
+  SymbolDocumentationRef Doc;
+  Doc.Brief = Data.consumeString(Strings);
+  Doc.Returns = Data.consumeString(Strings);
+
+  if (!Data.consumeSize(Doc.Notes))
+    return Doc;
+  for (auto &Note : Doc.Notes)
+    Note = Data.consumeString(Strings);
+
+  if (!Data.consumeSize(Doc.Warnings))
+    return Doc;
+  for (auto &Warning : Doc.Warnings)
+    Warning = Data.consumeString(Strings);
+
+  if (!Data.consumeSize(Doc.Parameters))
+    return Doc;
+  for (auto &ParamDoc : Doc.Parameters)
+    ParamDoc = {Data.consumeString(Strings), Data.consumeString(Strings)};
+
+  Doc.Description = Data.consumeString(Strings);
+  Doc.CommentText = Data.consumeString(Strings);
+
+  return Doc;
+}
+
 IncludeGraphNode readIncludeGraphNode(Reader &Data,
                                       llvm::ArrayRef<llvm::StringRef> Strings) {
   IncludeGraphNode IGN;
   IGN.Flags = static_cast<IncludeGraphNode::SourceFlag>(Data.consume8());
   IGN.URI = Data.consumeString(Strings);
   llvm::StringRef Digest = Data.consume(IGN.Digest.size());
   std::copy(Digest.bytes_begin(), Digest.bytes_end(), IGN.Digest.begin());
   if (!Data.consumeSize(IGN.DirectIncludes))
void writeSymbol(const Symbol &Sym, const StringTableOut &Strings,
   writeVar(Strings.index(Sym.Scope), OS);
   writeVar(Strings.index(Sym.TemplateSpecializationArgs), OS);
   writeLocation(Sym.Definition, Strings, OS);
   writeLocation(Sym.CanonicalDeclaration, Strings, OS);
   writeVar(Sym.References, OS);
   OS.write(static_cast<uint8_t>(Sym.Flags));
   writeVar(Strings.index(Sym.Signature), OS);
   writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);
-  writeVar(Strings.index(Sym.Documentation), OS);
+  writeSymbolDocumentation(Sym.Documentation, Strings, OS);
   writeVar(Strings.index(Sym.ReturnType), OS);
   writeVar(Strings.index(Sym.Type), OS);
 
   auto WriteInclude = [&](const Symbol::IncludeHeaderWithReferences &Include) {
     writeVar(Strings.index(Include.IncludeHeader), OS);
     writeVar(Include.References, OS);
   };
   writeVar(Sym.IncludeHeaders.size(), OS);
Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings,
   Sym.TemplateSpecializationArgs = Data.consumeString(Strings);
   Sym.Definition = readLocation(Data, Strings);
   Sym.CanonicalDeclaration = readLocation(Data, Strings);
   Sym.References = Data.consumeVar();
   Sym.Flags = static_cast<Symbol::SymbolFlag>(Data.consume8());
   Sym.Origin = Origin;
   Sym.Signature = Data.consumeString(Strings);
   Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
-  Sym.Documentation = Data.consumeString(Strings);
+  Sym.Documentation = readSymbolDocumentation(Data, Strings);
   Sym.ReturnType = Data.consumeString(Strings);
   Sym.Type = Data.consumeString(Strings);
   if (!Data.consumeSize(Sym.IncludeHeaders))
     return Sym;
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);
     I.References = Data.consumeVar();
   }

 //   - srcs: information related to include graph
 //   - stri: string table
 //   - symb: symbols
 //   - refs: references to symbols
 
 // The current versioning scheme is simple - non-current versions are rejected.
 // If you make a breaking change, bump this version number to invalidate stored
 // data. Later we may want to support some backward compatibility.
-constexpr static uint32_t Version = 17;
+constexpr static uint32_t Version = 18;
 
 llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data,
                                      SymbolOrigin Origin) {
   auto RIFF = riff::readFile(Data);
   if (!RIFF)
     return RIFF.takeError();
   if (RIFF->Type != riff::fourCC("CdIx"))
     return error("wrong RIFF filetype: {0}", riff::fourCCStr(RIFF->Type));

clang-tools-extra/clangd/index/Symbol.h

 //===--- Symbol.h ------------------------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_H
 
+#include "SymbolDocumentation.h"
 #include "index/SymbolID.h"
 #include "index/SymbolLocation.h"
 #include "index/SymbolOrigin.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/StringSaver.h"
 
 namespace clang {
struct Symbol {
   /// non-specializations. Example: "<int, bool, 3>"
   llvm::StringRef TemplateSpecializationArgs;
   /// What to insert when completing this symbol, after the symbol name.
   /// This is in LSP snippet syntax (e.g. "({$0})" for a no-args function).
   /// (When snippets are disabled, the symbol name alone is used).
   /// Only set when the symbol is indexed for completion.
   llvm::StringRef CompletionSnippetSuffix;
   /// Documentation including comment for the symbol declaration.
-  llvm::StringRef Documentation;
+  SymbolDocumentationRef Documentation;
   /// Type when this symbol is used in an expression. (Short display form).
   /// e.g. return type of a function, or type of a variable.
   /// Only set when the symbol is indexed for completion.
   llvm::StringRef ReturnType;
 
   /// Raw representation of the OpaqueType of the symbol, used for scoring
   /// purposes.
   /// Only set when the symbol is indexed for completion.

 /// Invokes Callback with each StringRef& contained in the Symbol.
 /// Useful for deduplicating backing strings.
 template <typename Callback> void visitStrings(Symbol &S, const Callback &CB) {
   CB(S.Name);
   CB(S.Scope);
   CB(S.TemplateSpecializationArgs);
   CB(S.Signature);
   CB(S.CompletionSnippetSuffix);
-  CB(S.Documentation);
+
+  CB(S.Documentation.Brief);
+  CB(S.Documentation.Returns);
+  for (auto &Note : S.Documentation.Notes)
+    CB(Note);
+  for (auto &Warning : S.Documentation.Warnings)
+    CB(Warning);
+  for (auto &ParamDoc : S.Documentation.Parameters) {
+    CB(ParamDoc.Name);
+    CB(ParamDoc.Description);
+  }
+  CB(S.Documentation.Description);
+  CB(S.Documentation.CommentText);
+
   CB(S.ReturnType);
   CB(S.Type);
   auto RawCharPointerCB = [&CB](const char *&P) {
     llvm::StringRef S(P);
     CB(S);
     assert(!S.data()[S.size()] && "Visited StringRef must be null-terminated");
     P = S.data();
   };

clang-tools-extra/clangd/index/SymbolCollector.cpp

const Symbol *SymbolCollector::addDeclaration(const NamedDecl &ND, SymbolID ID,
   // FIXME: we may want to choose a different redecl, or combine from several.
   assert(ASTCtx && PP && "ASTContext and Preprocessor must be set.");
   // We use the primary template, as clang does during code completion.
   CodeCompletionResult SymbolCompletion(&getTemplateOrThis(ND), 0);
   const auto *CCS = SymbolCompletion.CreateCodeCompletionString(
       *ASTCtx, *PP, CodeCompletionContext::CCC_Symbol, *CompletionAllocator,
       *CompletionTUInfo,
       /*IncludeBriefComments*/ false);
-  std::string Documentation =
-      formatDocumentation(*CCS, getDocComment(Ctx, SymbolCompletion,
-                                              /*CommentsFromHeaders=*/true));
+
+  const SymbolDocumentationOwned Documentation =
+      getDocumentation(Ctx, SymbolCompletion, /*CommentsFromHeaders=*/true);
+
   if (!(S.Flags & Symbol::IndexedForCodeCompletion)) {
     if (Opts.StoreAllDocumentation)
-      S.Documentation = Documentation;
+      S.Documentation = Documentation.toRef();
     Symbols.insert(S);
     return Symbols.find(S.ID);
   }
-  S.Documentation = Documentation;
+  S.Documentation = Documentation.toRef();
   std::string Signature;
   std::string SnippetSuffix;
   getSignature(*CCS, &Signature, &SnippetSuffix);
   S.Signature = Signature;
   S.CompletionSnippetSuffix = SnippetSuffix;
   std::string ReturnType = getReturnType(*CCS);
   S.ReturnType = ReturnType;
 

clang-tools-extra/clangd/index/YAMLSerialization.cpp

 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Symbol::IncludeHeaderWithReferences)
 LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::Ref)
+LLVM_YAML_IS_SEQUENCE_VECTOR(clang::clangd::ParameterDocumentationRef)
 
 namespace {
 using RefBundle =
     std::pair<clang::clangd::SymbolID, std::vector<clang::clangd::Ref>>;
 // This is a pale imitation of std::variant<Symbol, RefBundle, Relation>
 struct VariantEntry {
   llvm::Optional<clang::clangd::Symbol> Symbol;
   llvm::Optional<RefBundle> Refs;

 
 } // namespace
 namespace llvm {
 namespace yaml {
 
 using clang::clangd::FileDigest;
 using clang::clangd::IncludeGraph;
 using clang::clangd::IncludeGraphNode;
+using clang::clangd::ParameterDocumentationRef;
 using clang::clangd::Ref;
 using clang::clangd::RefKind;
 using clang::clangd::Relation;
 using clang::clangd::RelationKind;
 using clang::clangd::Symbol;
+using clang::clangd::SymbolDocumentationRef;
 using clang::clangd::SymbolID;
 using clang::clangd::SymbolLocation;
 using clang::index::SymbolInfo;
 using clang::index::SymbolKind;
 using clang::index::SymbolLanguage;
 using clang::tooling::CompileCommand;
 
 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.

 struct MappingTraits<clang::clangd::Symbol::IncludeHeaderWithReferences> {
   static void mapping(IO &IO,
                       clang::clangd::Symbol::IncludeHeaderWithReferences &Inc) {
     IO.mapRequired("Header", Inc.IncludeHeader);
     IO.mapRequired("References", Inc.References);
   }
 };
 
+template <> struct MappingTraits<ParameterDocumentationRef> {
+  static void mapping(IO &IO, ParameterDocumentationRef &P) {
+    IO.mapRequired("Name", P.Name);
+    IO.mapRequired("Description", P.Description);
+  }
+};
+
+template <> struct MappingTraits<SymbolDocumentationRef> {
+  static void mapping(IO &IO, SymbolDocumentationRef &Doc) {
+    IO.mapOptional("Brief", Doc.Brief);
+    IO.mapOptional("Returns", Doc.Returns);
+
+    IO.mapOptional("Notes", Doc.Notes);
+    IO.mapOptional("Warnings", Doc.Warnings);
+
+    IO.mapOptional("Parameters", Doc.Parameters);
+
+    IO.mapOptional("Description", Doc.Description);
+    IO.mapOptional("CommentText", Doc.CommentText);
+  }
+};
+
 template <> struct MappingTraits<Symbol> {
   static void mapping(IO &IO, Symbol &Sym) {
     MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(IO, Sym.ID);
     MappingNormalization<NormalizedSymbolFlag, Symbol::SymbolFlag> NSymbolFlag(
         IO, Sym.Flags);
     IO.mapRequired("ID", NSymbolID->HexString);
     IO.mapRequired("Name", Sym.Name);
     IO.mapRequired("Scope", Sym.Scope);

clang-tools-extra/clangd/test/index-serialization/Inputs/sample.cpp

 // Include a file to ensure we have multiple sources.
 #include "sample.h"
 
 // This introduces a symbol, a reference and a relation.
 struct Bar : public Foo {
-  // This introduces an OverriddenBy relation by implementing Foo::Func.
+  /// \brief This introduces an OverriddenBy relation by implementing Foo::Func.
+  /// \details And it also introduces some doxygen!
+  /// \param foo bar
+  /// \warning !!!
+  /// \note a note
+  /// \return nothing
   void Func() override {}
 };

clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

TEST(SignatureHelpTest, InstantiatedSignatures) {
   )cpp";
 
   EXPECT_THAT(signatures(Sig3).signatures,
               ElementsAre(sig("foo([[T]], [[U]]) -> void")));
 }
 
 TEST(SignatureHelpTest, IndexDocumentation) {
   Symbol Foo0 = sym("foo", index::SymbolKind::Function, "@F@\\0#");
-  Foo0.Documentation = "doc from the index";
+  Foo0.Documentation.CommentText = "doc from the index";
   Symbol Foo1 = sym("foo", index::SymbolKind::Function, "@F@\\0#I#");
-  Foo1.Documentation = "doc from the index";
+  Foo1.Documentation.CommentText = "doc from the index";
   Symbol Foo2 = sym("foo", index::SymbolKind::Function, "@F@\\0#I#I#");
 
   StringRef Sig0 = R"cpp(
     int foo();
     int foo(double);
 
     void test() {
       foo(^);

clang-tools-extra/clangd/unittests/CodeCompletionStringsTests.cpp

 //===-- CodeCompletionStringsTests.cpp --------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CodeCompletionStrings.h"
+#include "SymbolDocumentationMatchers.h"
 #include "TestTU.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 namespace clang {
 namespace clangd {
 namespace {

 
 TEST_F(CompletionStringTest, DocumentationWithAnnotation) {
   Builder.addBriefComment("This is ignored");
   Builder.AddAnnotation("Ano");
   EXPECT_EQ(formatDocumentation(*Builder.TakeString(), "Is this brief?"),
             "Annotation: Ano\n\nIs this brief?");
 }
 
-TEST_F(CompletionStringTest, GetDeclCommentBadUTF8) {
+TEST_F(CompletionStringTest, GetDeclDocumentationBadUTF8) {
   // <ff> is not a valid byte here, should be replaced by encoded <U+FFFD>.
-  auto TU = TestTU::withCode("/*x\xffy*/ struct X;");
+  const std::string Code = llvm::formatv(R"cpp(
+  /// \brief {0}
+  /// \details {0}
+  /// \param {0} {0}
+  /// \warning {0}
+  /// \note {0}
+  /// \return {0}
+  struct X;
+  )cpp",
+                                         "x\xffy");
+
+  auto TU = TestTU::withCode(Code);
+  auto AST = TU.build();
+
+  const std::string Utf8Replacement = "x\xef\xbf\xbdy";
+  SymbolDocumentationOwned ExpectedDoc;
+  ExpectedDoc.Brief = Utf8Replacement;
+  ExpectedDoc.Returns = Utf8Replacement;
+  ExpectedDoc.Parameters = {{Utf8Replacement, Utf8Replacement}};
+  ExpectedDoc.Notes = {Utf8Replacement};
+  ExpectedDoc.Warnings = {Utf8Replacement};
+  ExpectedDoc.Description = {"\\details " + Utf8Replacement};
+  ExpectedDoc.CommentText = llvm::formatv(R"(\brief {0}
+\details {0}
+\param {0} {0}
+\warning {0}
+\note {0}
+\return {0})", Utf8Replacement);
+
+  EXPECT_THAT(getDeclDocumentation(AST.getASTContext(), findDecl(AST, "X")),
+              matchesDoc(ExpectedDoc));
+}
+
+TEST_F(CompletionStringTest, DoxygenParsing) {
+  struct {
+    const char *const Code;
+    const std::function<void(SymbolDocumentationOwned &)> ExpectedBuilder;
+  } Cases[] = {
+      {R"cpp(
+    // Hello world
+    void foo();
+    )cpp",
+       [](SymbolDocumentationOwned &Doc) { Doc.Description = "Hello world"; }},
+      {R"cpp(
+    /*! 
+     * \brief brief
+     * \details details
+     */
+    void foo();
+    )cpp",
+       [](SymbolDocumentationOwned &Doc) {
+         Doc.Brief = "brief";
+         Doc.Description = "\\details details";
+       }},
+      {R"cpp(
+    /** 
+     * @brief brief
+     * @details details
+     * @see somewhere else
+     */
+    void foo();
+    )cpp",
+       [](SymbolDocumentationOwned &Doc) {
+         Doc.Brief = "brief";
+         Doc.Description = "@details details\n\n@see somewhere else";
+       }},
+      {R"cpp(
+    /*! 
+     * @brief brief
+     * @details details
+     * @param foo foodoc
+     * @throws ball at hoop
+     * @note note1
+     * @warning warning1
+     * @note note2
+     * @warning warning2
+     * @param bar bardoc
+     * @return something
+     */
+    void foo();
+    )cpp",
+       [](SymbolDocumentationOwned &Doc) {
+         Doc.Brief = "brief";
+         Doc.Description = "@details details\n\n@throws ball at hoop";
+         Doc.Parameters = {{"foo", "foodoc"}, {"bar", "bardoc"}};
+         Doc.Warnings = {"warning1", "warning2"};
+         Doc.Notes = {"note1", "note2"};
+         Doc.Returns = "something";
+       }},
+      {R"cpp(
+    /// @brief Here's \b bold \e italic and \p code
+    int foo;
+    )cpp",
+       [](SymbolDocumentationOwned &Doc) {
+         Doc.Brief = "Here's **bold** *italic* and `code`";
+       }}};
+
+  for (const auto &Case : Cases) {
+    SCOPED_TRACE(Case.Code);
+
+    auto TU = TestTU::withCode(Case.Code);
     auto AST = TU.build();
-  EXPECT_EQ("x\xef\xbf\xbdy",
-            getDeclComment(AST.getASTContext(), findDecl(AST, "X")));
+    auto &Ctx = AST.getASTContext();
+    const auto &Decl = findDecl(AST, "foo");
+
+    SymbolDocumentationOwned ExpectedDoc;
+    ExpectedDoc.CommentText =
+        getCompletionComment(Ctx, &Decl)
+            ->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+    Case.ExpectedBuilder(ExpectedDoc);
+
+    EXPECT_THAT(getDeclDocumentation(Ctx, Decl), matchesDoc(ExpectedDoc));
+  }
 }
 
 TEST_F(CompletionStringTest, MultipleAnnotations) {
   Builder.AddAnnotation("Ano1");
   Builder.AddAnnotation("Ano2");
   Builder.AddAnnotation("Ano3");
 
   EXPECT_EQ(formatDocumentation(*Builder.TakeString(), ""),

clang-tools-extra/clangd/unittests/HoverTests.cpp

 //===-- HoverTests.cpp ----------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "AST.h"
 #include "Annotations.h"
 #include "Config.h"
 #include "Hover.h"
+#include "SymbolDocumentationMatchers.h"
 #include "TestIndex.h"
 #include "TestTU.h"
 #include "index/MemIndex.h"
 #include "clang/AST/Attr.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/StringRef.h"
 
const std::function<void(HoverInfo &)> ExpectedBuilder;
       {R"cpp(
           // Best foo ever.
           void [[fo^o]]() {}
           )cpp",
        [](HoverInfo &HI) {
          HI.NamespaceScope = "";
          HI.Name = "foo";
          HI.Kind = index::SymbolKind::Function;
-         HI.Documentation = "Best foo ever.";
+         HI.Documentation =
+             SymbolDocumentationOwned::descriptionOnly("Best foo ever.");
          HI.Definition = "void foo()";
          HI.ReturnType = "void";
          HI.Type = "void ()";
          HI.Parameters.emplace();
        }},
       // Inside namespace
       {R"cpp(
           namespace ns1 { namespace ns2 {
             /// Best foo ever.
             void [[fo^o]]() {}
           }}
           )cpp",
        [](HoverInfo &HI) {
          HI.NamespaceScope = "ns1::ns2::";
          HI.Name = "foo";
          HI.Kind = index::SymbolKind::Function;
-         HI.Documentation = "Best foo ever.";
+         HI.Documentation =
+             SymbolDocumentationOwned::descriptionOnly("Best foo ever.");
          HI.Definition = "void foo()";
          HI.ReturnType = "void";
          HI.Type = "void ()";
          HI.Parameters.emplace();
        }},
       // Field
       {R"cpp(
           namespace ns1 { namespace ns2 {
class Foo final {})cpp";
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "Foo<int *>";
          HI.Kind = index::SymbolKind::Class;
          HI.NamespaceScope = "";
          HI.Definition = "template <> class Foo<int *>";
          // FIXME: Maybe force instantiation to make use of real template
          // pattern.
-         HI.Documentation = "comment from primary";
+         HI.Documentation =
+             SymbolDocumentationOwned::descriptionOnly("comment from primary");
        }},
       {// Template Type Parameter
        R"cpp(
           template <typename [[^T]] = int> void foo();
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "T";
          HI.Kind = index::SymbolKind::TemplateTypeParm;
class Foo final {})cpp";
           struct X { int Y; float [[^y]]() { return Y; } };
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "y";
          HI.Kind = index::SymbolKind::InstanceMethod;
          HI.NamespaceScope = "";
          HI.Definition = "float y()";
          HI.LocalScope = "X::";
-         HI.Documentation = "Trivial accessor for `Y`.";
+         HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+             "Trivial accessor for `Y`.");
          HI.Type = "float ()";
          HI.ReturnType = "float";
          HI.Parameters.emplace();
          HI.AccessSpecifier = "public";
        }},
       {// Setter
        R"cpp(
           struct X { int Y; void [[^setY]](float v) { Y = v; } };
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "setY";
          HI.Kind = index::SymbolKind::InstanceMethod;
          HI.NamespaceScope = "";
          HI.Definition = "void setY(float v)";
          HI.LocalScope = "X::";
-         HI.Documentation = "Trivial setter for `Y`.";
+         HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+             "Trivial setter for `Y`.");
          HI.Type = "void (float)";
          HI.ReturnType = "void";
          HI.Parameters.emplace();
          HI.Parameters->emplace_back();
          HI.Parameters->back().Type = "float";
          HI.Parameters->back().Name = "v";
          HI.AccessSpecifier = "public";
        }},
       {// Setter (builder)
        R"cpp(
           struct X { int Y; X& [[^setY]](float v) { Y = v; return *this; } };
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "setY";
          HI.Kind = index::SymbolKind::InstanceMethod;
          HI.NamespaceScope = "";
          HI.Definition = "X &setY(float v)";
          HI.LocalScope = "X::";
-         HI.Documentation = "Trivial setter for `Y`.";
+         HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+             "Trivial setter for `Y`.");
          HI.Type = "X &(float)";
          HI.ReturnType = "X &";
          HI.Parameters.emplace();
          HI.Parameters->emplace_back();
          HI.Parameters->back().Type = "float";
          HI.Parameters->back().Name = "v";
          HI.AccessSpecifier = "public";
        }},
       {// Setter (move)
        R"cpp(
           namespace std { template<typename T> T&& move(T&& t); }
           struct X { int Y; void [[^setY]](float v) { Y = std::move(v); } };
           )cpp",
        [](HoverInfo &HI) {
          HI.Name = "setY";
          HI.Kind = index::SymbolKind::InstanceMethod;
          HI.NamespaceScope = "";
          HI.Definition = "void setY(float v)";
          HI.LocalScope = "X::";
-         HI.Documentation = "Trivial setter for `Y`.";
+         HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+             "Trivial setter for `Y`.");
          HI.Type = "void (float)";
          HI.ReturnType = "void";
          HI.Parameters.emplace();
          HI.Parameters->emplace_back();
          HI.Parameters->back().Type = "float";
          HI.Parameters->back().Name = "v";
          HI.AccessSpecifier = "public";
        }},
for (const auto &Case : Cases) {
     HoverInfo Expected;
     Expected.SymRange = T.range();
     Case.ExpectedBuilder(Expected);
 
     EXPECT_EQ(H->NamespaceScope, Expected.NamespaceScope);
     EXPECT_EQ(H->LocalScope, Expected.LocalScope);
     EXPECT_EQ(H->Name, Expected.Name);
     EXPECT_EQ(H->Kind, Expected.Kind);
-    EXPECT_EQ(H->Documentation, Expected.Documentation);
+    ASSERT_THAT(H->Documentation, matchesDoc(Expected.Documentation));
     EXPECT_EQ(H->Definition, Expected.Definition);
     EXPECT_EQ(H->Type, Expected.Type);
     EXPECT_EQ(H->ReturnType, Expected.ReturnType);
     EXPECT_EQ(H->Parameters, Expected.Parameters);
     EXPECT_EQ(H->TemplateParameters, Expected.TemplateParameters);
     EXPECT_EQ(H->SymRange, Expected.SymRange);
     EXPECT_EQ(H->Value, Expected.Value);
     EXPECT_EQ(H->Size, Expected.Size);
const std::function<void(HoverInfo &)> ExpectedBuilder;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "foo";
             HI.Kind = index::SymbolKind::Function;
             HI.NamespaceScope = "";
             HI.Type = "void (int)";
             HI.Definition = "void foo(int)";
-            HI.Documentation = "Function definition via pointer";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "Function definition via pointer");
             HI.ReturnType = "void";
             HI.Parameters = {
                 {{"int"}, llvm::None, llvm::None},
             };
           }},
       {
           R"cpp(// Function declaration via call
             int foo(int);
             int main() {
               return ^[[foo]](42);
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "foo";
             HI.Kind = index::SymbolKind::Function;
             HI.NamespaceScope = "";
             HI.Type = "int (int)";
             HI.Definition = "int foo(int)";
-            HI.Documentation = "Function declaration via call";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "Function declaration via call");
             HI.ReturnType = "int";
             HI.Parameters = {
                 {{"int"}, llvm::None, llvm::None},
             };
           }},
       {
           R"cpp(// Field
             struct Foo { int x; };
const std::function<void(HoverInfo &)> ExpectedBuilder;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "Foo";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.NamespaceScope = "";
             HI.Definition = "typedef int Foo";
             HI.Type = "int";
-            HI.Documentation = "Typedef";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("Typedef");
           }},
       {
           R"cpp(// Typedef with embedded definition
             typedef struct Bar {} Foo;
             int main() {
               ^[[Foo]] bar;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "Foo";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.NamespaceScope = "";
             HI.Definition = "typedef struct Bar Foo";
             HI.Type = "struct Bar";
-            HI.Documentation = "Typedef with embedded definition";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "Typedef with embedded definition");
           }},
       {
           R"cpp(// Namespace
             namespace ns {
             struct Foo { static void bar(); };
             } // namespace ns
             int main() { ^[[ns]]::Foo::bar(); }
           )cpp",
const std::function<void(HoverInfo &)> ExpectedBuilder;
             class Foo {};
             [[F^oo]]* foo();
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "Foo";
             HI.Kind = index::SymbolKind::Class;
             HI.NamespaceScope = "";
             HI.Definition = "class Foo {}";
-            HI.Documentation = "Forward class declaration";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "Forward class declaration");
           }},
       {
-          R"cpp(// Function declaration
+          R"cpp(
+            /// \brief Function declaration
+            /// \details Some details
+            /// \throws std::runtime_error sometimes
+            /// \param x doc for x
+            /// \warning Watch out!
+            /// \note note1 \note note2
+            /// \return Nothing
             void foo();
             void g() { [[f^oo]](); }
             void foo() {}
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "foo";
             HI.Kind = index::SymbolKind::Function;
             HI.NamespaceScope = "";
             HI.Type = "void ()";
             HI.Definition = "void foo()";
-            HI.Documentation = "Function declaration";
+            HI.Documentation.Brief = "Function declaration";
+            HI.Documentation.Description = "\\details Some details\n\n\\throws "
+                                           "std::runtime_error sometimes";
+            HI.Documentation.Parameters = {
+                {"x", "doc for x"},
+            };
+            HI.Documentation.Returns = "Nothing";
+            HI.Documentation.Notes = {"note1", "note2"};
+            HI.Documentation.Warnings = {"Watch out!"};
+            HI.Documentation.CommentText = R"(\brief Function declaration
+\details Some details
+\throws std::runtime_error sometimes
+\param x doc for x
+\warning Watch out!
+\note note1 \note note2
+\return Nothing)";
             HI.ReturnType = "void";
             HI.Parameters = std::vector<HoverInfo::Param>{};
           }},
       {
           R"cpp(// Enum declaration
             enum Hello {
               ONE, TWO, THREE,
             };
             void foo() {
               [[Hel^lo]] hello = ONE;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "Hello";
             HI.Kind = index::SymbolKind::Enum;
             HI.NamespaceScope = "";
             HI.Definition = "enum Hello {}";
-            HI.Documentation = "Enum declaration";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("Enum declaration");
           }},
       {
           R"cpp(// Enumerator
             enum Hello {
               ONE, TWO, THREE,
             };
             void foo() {
               Hello hello = [[O^NE]];
\return Nothing)";
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "hey";
             HI.Kind = index::SymbolKind::Variable;
             HI.NamespaceScope = "";
             HI.Type = "int";
             HI.Definition = "static int hey = 10";
-            HI.Documentation = "Global variable";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("Global variable");
             // FIXME: Value shouldn't be set in this case
             HI.Value = "10 (0xa)";
           }},
       {
           R"cpp(// Global variable in namespace
             namespace ns1 {
               static long long hey = -36637162602497;
             }
\return Nothing)";
             void g() { auto x = [[f^oo]]<int>(); }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "foo";
             HI.Kind = index::SymbolKind::Function;
             HI.NamespaceScope = "";
             HI.Type = "int ()";
             HI.Definition = "template <> int foo<int>()";
-            HI.Documentation = "Templated function";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("Templated function");
             HI.ReturnType = "int";
             HI.Parameters = std::vector<HoverInfo::Param>{};
             // FIXME: We should populate template parameters with arguments in
             // case of instantiations.
           }},
       {
           R"cpp(// Anonymous union
             struct outer {
\return Nothing)";
           [](HoverInfo &HI) {
             HI.Name = "indexSymbol";
             HI.Kind = index::SymbolKind::Function;
             HI.NamespaceScope = "";
             HI.Type = "void ()";
             HI.Definition = "void indexSymbol()";
             HI.ReturnType = "void";
             HI.Parameters = std::vector<HoverInfo::Param>{};
-            HI.Documentation = "comment from index";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("comment from index");
           }},
       {
           R"cpp(// Simple initialization with auto
             void foo() {
               ^[[auto]] i = 1;
             }
           )cpp",
           [](HoverInfo &HI) {
\return Nothing)";
             ^[[auto]] test() -> decltype(Bar()) {
               return Bar();
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "auto function return with trailing type";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "auto function return with trailing type");
           }},
       {
           R"cpp(// trailing return type
             struct Bar {};
             auto test() -> ^[[decltype]](Bar()) {
               return Bar();
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "decltype";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "trailing return type";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "trailing return type");
           }},
       {
           R"cpp(// auto in function return
             struct Bar {};
             ^[[auto]] test() {
               return Bar();
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "auto in function return";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "auto in function return");
           }},
       {
           R"cpp(// auto& in function return
             struct Bar {};
             ^[[auto]]& test() {
               static Bar x;
               return x;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "auto& in function return";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "auto& in function return");
           }},
       {
           R"cpp(// auto* in function return
             struct Bar {};
             ^[[auto]]* test() {
               Bar* bar;
               return bar;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "auto* in function return";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "auto* in function return");
           }},
       {
           R"cpp(// const auto& in function return
             struct Bar {};
             const ^[[auto]]& test() {
               static Bar x;
               return x;
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "const auto& in function return";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "const auto& in function return");
           }},
       {
           R"cpp(// decltype(auto) in function return
             struct Bar {};
             ^[[decltype]](auto) test() {
               return Bar();
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "decltype";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation = "decltype(auto) in function return";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "decltype(auto) in function return");
           }},
       {
           R"cpp(// decltype(auto) reference in function return
             ^[[decltype]](auto) test() {
               static int a;
               return (a);
             }
           )cpp",
\return Nothing)";
             void foo() {
               ^[[decltype]](test()) i = test();
             }
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "decltype";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "Bar";
-            HI.Documentation =
-                "decltype of function with trailing return type.";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "decltype of function with trailing return type.");
           }},
       {
           R"cpp(// decltype of var with decltype.
             void foo() {
               int I = 0;
               decltype(I) J = I;
               ^[[decltype]](J) K = J;
             }
\return Nothing)";
           struct cls {};
           typedef cls cls_type;
           ^[[auto]] y = cls_type();
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "cls_type // aka: cls";
-            HI.Documentation = "auto on alias";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("auto on alias");
           }},
       {
           R"cpp(// auto on alias
           template <class>
           struct templ {};
           ^[[auto]] z = templ<int>();
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "auto";
             HI.Kind = index::SymbolKind::TypeAlias;
             HI.Definition = "templ<int>";
-            HI.Documentation = "auto on alias";
+            HI.Documentation =
+                SymbolDocumentationOwned::descriptionOnly("auto on alias");
           }},
       {
           R"cpp(// Undeduced auto declaration
             template<typename T>
             void foo() {
               ^[[auto]] x = T();
             }
           )cpp",
\return Nothing)";
           template <class T> struct cls {};
           [[cl^s]]<cls<cls<int>>> foo;
           )cpp",
           [](HoverInfo &HI) {
             HI.Definition = "template <> struct cls<cls<cls<int>>> {}";
             HI.Kind = index::SymbolKind::Struct;
             HI.NamespaceScope = "";
             HI.Name = "cls<cls<cls<int>>>";
-            HI.Documentation = "type of nested templates.";
+            HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+                "type of nested templates.");
           }},
       {
           R"cpp(// type with decltype
           int a;
           decltype(a) [[b^]] = a;)cpp",
           [](HoverInfo &HI) {
             HI.Definition = "decltype(a) b = a";
             HI.Kind = index::SymbolKind::Variable;
\return Nothing)";
        }},
       {R"cpp(
          void foo(int * __attribute__(([[non^null]], noescape)) );
          )cpp",
        [](HoverInfo &HI) {
          HI.Name = "nonnull";
          HI.Kind = index::SymbolKind::Unknown; // FIXME: no suitable value
          HI.Definition = "__attribute__((nonnull))";
-         HI.Documentation = Attr::getDocumentation(attr::NonNull).str();
+         HI.Documentation = SymbolDocumentationOwned::descriptionOnly(
+             Attr::getDocumentation(attr::NonNull).str());
        }},
   };
 
   // Create a tiny index, so tests above can verify documentation is fetched.
   Symbol IndexSym = func("indexSymbol");
-  IndexSym.Documentation = "comment from index";
+  IndexSym.Documentation =
+      SymbolDocumentationRef::descriptionOnly("comment from index");
   SymbolSlab::Builder Symbols;
   Symbols.insert(IndexSym);
   auto Index =
       MemIndex::build(std::move(Symbols).build(), RefSlab(), RelationSlab());
 
   for (const auto &Case : Cases) {
     SCOPED_TRACE(Case.Code);
 
for (const auto &Case : Cases) {
     Expected.SymRange = T.range();
     Case.ExpectedBuilder(Expected);
 
     SCOPED_TRACE(H->present().asPlainText());
     EXPECT_EQ(H->NamespaceScope, Expected.NamespaceScope);
     EXPECT_EQ(H->LocalScope, Expected.LocalScope);
     EXPECT_EQ(H->Name, Expected.Name);
     EXPECT_EQ(H->Kind, Expected.Kind);
-    EXPECT_EQ(H->Documentation, Expected.Documentation);
+    ASSERT_THAT(H->Documentation, matchesDoc(Expected.Documentation));
     EXPECT_EQ(H->Definition, Expected.Definition);
     EXPECT_EQ(H->Type, Expected.Type);
     EXPECT_EQ(H->ReturnType, Expected.ReturnType);
     EXPECT_EQ(H->Parameters, Expected.Parameters);
     EXPECT_EQ(H->TemplateParameters, Expected.TemplateParameters);
     EXPECT_EQ(H->SymRange, Expected.SymRange);
     EXPECT_EQ(H->Value, Expected.Value);
   }
 }
 
 TEST(Hover, DocsFromIndex) {
   Annotations T(R"cpp(
   template <typename T> class X {};
   void foo() {
     auto t = X<int>();
     X^<int> w;
     (void)w;
   })cpp");
 
   TestTU TU = TestTU::withCode(T.code());
   auto AST = TU.build();
   Symbol IndexSym;
   IndexSym.ID = getSymbolID(&findDecl(AST, "X"));
-  IndexSym.Documentation = "comment from index";
+  IndexSym.Documentation =
+      SymbolDocumentationRef::descriptionOnly("comment from index");
   SymbolSlab::Builder Symbols;
   Symbols.insert(IndexSym);
   auto Index =
       MemIndex::build(std::move(Symbols).build(), RefSlab(), RelationSlab());
 
   for (const auto &P : T.points()) {
     auto H = getHover(AST, P, format::getLLVMStyle(), Index.get());
     ASSERT_TRUE(H);
-    EXPECT_EQ(H->Documentation, IndexSym.Documentation);
+    ASSERT_THAT(H->Documentation.toRef(), matchesDoc(IndexSym.Documentation));
   }
 }
 
 TEST(Hover, DocsFromAST) {
   Annotations T(R"cpp(
   // doc
   template <typename T> class X {};
   // doc
   template <typename T> void bar() {}
   // doc
   template <typename T> T baz;
   void foo() {
     au^to t = X<int>();
     X^<int>();
     b^ar<int>();
     au^to T = ba^z<X<int>>;
     ba^z<int> = 0;
   })cpp");
 
   TestTU TU = TestTU::withCode(T.code());
   auto AST = TU.build();
   for (const auto &P : T.points()) {
     auto H = getHover(AST, P, format::getLLVMStyle(), nullptr);
     ASSERT_TRUE(H);
-    EXPECT_EQ(H->Documentation, "doc");
+    ASSERT_THAT(H->Documentation,
+                matchesDoc(SymbolDocumentationOwned::descriptionOnly("doc")));
   }
 }
 
 TEST(Hover, NoCrash) {
   Annotations T(R"cpp(
     /* error-ok */
     template<typename T> T foo(T);
 
TEST(Hover, DocsFromMostSpecial) {
   })cpp");
 
   TestTU TU = TestTU::withCode(T.code());
   auto AST = TU.build();
   for (const auto *Comment : {"doc1", "doc2", "doc3"}) {
     for (const auto &P : T.points(Comment)) {
       auto H = getHover(AST, P, format::getLLVMStyle(), nullptr);
       ASSERT_TRUE(H);
-      EXPECT_EQ(H->Documentation, Comment);
+      ASSERT_THAT(
+          H->Documentation,
+          matchesDoc(SymbolDocumentationOwned::descriptionOnly(Comment)));
     }
   }
 }
 
 TEST(Hover, Present) {
   struct {
     const std::function<void(HoverInfo &)> Builder;
     llvm::StringRef ExpectedRender;
llvm::StringRef ExpectedRender;
       {
           [](HoverInfo &HI) {
             HI.Kind = index::SymbolKind::Class;
             HI.Size = 10;
             HI.TemplateParameters = {
                 {{"typename"}, std::string("T"), llvm::None},
                 {{"typename"}, std::string("C"), std::string("bool")},
             };
-            HI.Documentation = "documentation";
+            HI.Documentation.Brief = "brief";
+            HI.Documentation.Description = "details";
+            HI.Documentation.Parameters = {
+                {"Parameters", "should be ignored for classes"}};
+            HI.Documentation.Returns = "Returns should be ignored for classes";
+            HI.Documentation.Notes = {"note1", "note2"};
+            HI.Documentation.Warnings = {"warning1", "warning2"};
+            HI.Documentation.CommentText = "Not used for Hover presentation";
             HI.Definition =
                 "template <typename T, typename C = bool> class Foo {}";
             HI.Name = "foo";
             HI.NamespaceScope.emplace();
           },
           R"(class foo
 
+brief
 Size: 10 bytes
-documentation
+details
+
+Warnings:
+- warning1
+- warning2
+
+Notes:
+- note1
+- note2
 
 template <typename T, typename C = bool> class Foo {})",
       },
       {
           [](HoverInfo &HI) {
             HI.Kind = index::SymbolKind::Function;
             HI.Name = "foo";
             HI.Type = {"type", "c_type"};
             HI.ReturnType = {"ret_type", "can_ret_type"};
             HI.Parameters.emplace();
             HoverInfo::Param P;
             HI.Parameters->push_back(P);
             P.Type = {"type", "can_type"};
             HI.Parameters->push_back(P);
             P.Name = "foo";
             HI.Parameters->push_back(P);
             P.Default = "default";
             HI.Parameters->push_back(P);
+            HI.Documentation.Brief = "brief";
+            HI.Documentation.Description = "details";
+            HI.Documentation.Parameters = {
+                {"foo", "param doc"},
+                {"bar", "doc for parameter not in the signature"}};
+            HI.Documentation.Returns = "doc for return";
+            HI.Documentation.Notes = {"note1", "note2"};
+            HI.Documentation.Warnings = {"warning1", "warning2"};
+            HI.Documentation.CommentText = "Not used for Hover presentation";
             HI.NamespaceScope = "ns::";
             HI.Definition = "ret_type foo(params) {}";
           },
           "function foo\n"
           "\n"
-          "→ ret_type (aka can_ret_type)\n"
+          "brief\n"
+          "→ ret_type (aka can_ret_type): doc for return\n"
           "Parameters:\n"
           "- \n"
           "- type (aka can_type)\n"
-          "- type foo (aka can_type)\n"
+          "- type foo (aka can_type): param doc\n"
           "- type foo = default (aka can_type)\n"
+          "- bar: doc for parameter not in the signature\n"
+          "details\n"
+          "\n"
+          "Warnings:\n"
+          "- warning1\n"
+          "- warning2\n"
+          "\n"
+          "Notes:\n"
+          "- note1\n"
+          "- note2\n"
           "\n"
           "// In namespace ns\n"
           "ret_type foo(params) {}",
       },
       {
           [](HoverInfo &HI) {
             HI.Kind = index::SymbolKind::Field;
             HI.LocalScope = "test::Bar::";
TEST(Hover, SpaceshipTemplateNoCrash) {
   };
   constexpr strong_ordering strong_ordering::equal = {0};
   constexpr strong_ordering strong_ordering::greater = {1};
   constexpr strong_ordering strong_ordering::less = {-1};
   }
 
   template <typename T>
   struct S {
-    // Foo bar baz
+    /// Foo bar baz
     friend auto operator<=>(S, S) = default;
   };
   static_assert(S<void>() =^= S<void>());
     )cpp");
 
   TestTU TU = TestTU::withCode(T.code());
   TU.ExtraArgs.push_back("-std=c++20");
   auto AST = TU.build();
   auto HI = getHover(AST, T.point(), format::getLLVMStyle(), nullptr);
-  EXPECT_EQ(HI->Documentation, "Foo bar baz");
+
+  ASSERT_THAT(
+      HI->Documentation,
+      matchesDoc(SymbolDocumentationOwned::descriptionOnly("Foo bar baz")));
 }
 
 TEST(Hover, ForwardStructNoCrash) {
   Annotations T(R"cpp(
   struct Foo;
   int bar;
   auto baz = (Fo^o*)&bar;
     )cpp");

clang-tools-extra/clangd/unittests/IndexTests.cpp

 //===-- IndexTests.cpp  -------------------------------*- C++ -*-----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Annotations.h"
+#include "SymbolDocumentationMatchers.h"
 #include "SyncAPI.h"
 #include "TestIndex.h"
 #include "TestTU.h"
 #include "index/FileIndex.h"
 #include "index/Index.h"
 #include "index/MemIndex.h"
 #include "index/Merge.h"
 #include "index/Symbol.h"
TEST(MergeTest, Merge) {
   L.ID = R.ID = SymbolID("hello");
   L.Name = R.Name = "Foo";                           // same in both
   L.CanonicalDeclaration.FileURI = "file:///left.h"; // differs
   R.CanonicalDeclaration.FileURI = "file:///right.h";
   L.References = 1;
   R.References = 2;
   L.Signature = "()";                   // present in left only
   R.CompletionSnippetSuffix = "{$1:0}"; // present in right only
-  R.Documentation = "--doc--";
+  R.Documentation = SymbolDocumentationRef::descriptionOnly("--doc--");
   L.Origin = SymbolOrigin::Preamble;
   R.Origin = SymbolOrigin::Static;
   R.Type = "expectedType";
 
   Symbol M = mergeSymbol(L, R);
   EXPECT_EQ(M.Name, "Foo");
   EXPECT_EQ(StringRef(M.CanonicalDeclaration.FileURI), "file:///left.h");
   EXPECT_EQ(M.References, 3u);
   EXPECT_EQ(M.Signature, "()");
   EXPECT_EQ(M.CompletionSnippetSuffix, "{$1:0}");
-  EXPECT_EQ(M.Documentation, "--doc--");
+  EXPECT_THAT(M.Documentation,
+              matchesDoc(SymbolDocumentationRef::descriptionOnly("--doc--")));
   EXPECT_EQ(M.Type, "expectedType");
   EXPECT_EQ(M.Origin, SymbolOrigin::Preamble | SymbolOrigin::Static |
                           SymbolOrigin::Merge);
 }
 
 TEST(MergeTest, PreferSymbolWithDefn) {
   Symbol L, R;
 
TEST(MergeIndexTest, IndexedFiles) {
   EXPECT_EQ(ContainsFile("unittest:///foobar.cc"), IndexContents::None);
 }
 
 TEST(MergeIndexTest, NonDocumentation) {
   using index::SymbolKind;
   Symbol L, R;
   L.ID = R.ID = SymbolID("x");
   L.Definition.FileURI = "file:/x.h";
-  R.Documentation = "Forward declarations because x.h is too big to include";
+  R.Documentation = SymbolDocumentationRef::descriptionOnly(
+      "Forward declarations because x.h is too big to include");
   for (auto ClassLikeKind :
        {SymbolKind::Class, SymbolKind::Struct, SymbolKind::Union}) {
     L.SymInfo.Kind = ClassLikeKind;
-    EXPECT_EQ(mergeSymbol(L, R).Documentation, "");
+    ASSERT_TRUE(mergeSymbol(L, R).Documentation.empty());
   }
 
   L.SymInfo.Kind = SymbolKind::Function;
-  R.Documentation = "Documentation from non-class symbols should be included";
-  EXPECT_EQ(mergeSymbol(L, R).Documentation, R.Documentation);
+  R.Documentation = SymbolDocumentationRef::descriptionOnly(
+      "Documentation from non-class symbols should be included");
+  EXPECT_THAT(mergeSymbol(L, R).Documentation, matchesDoc(R.Documentation));
 }
 
 MATCHER_P2(IncludeHeaderWithRef, IncludeHeader, References, "") {
   return (arg.IncludeHeader == IncludeHeader) && (arg.References == References);
 }
 
 TEST(MergeTest, MergeIncludesOnDifferentDefinitions) {
   Symbol L, R;

clang-tools-extra/clangd/unittests/SerializationTests.cpp

 //===-- SerializationTests.cpp - Binary and YAML serialization unit tests -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Headers.h"
 #include "RIFF.h"
+#include "SymbolDocumentationMatchers.h"
 #include "index/Serialization.h"
 #include "support/Logger.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
CanonicalDeclaration:
   FileURI:        file:///path/foo.h
   Start:
     Line: 1
     Column: 0
   End:
     Line: 1
     Column: 1
 Flags:    129
-Documentation:    'Foo doc'
+Documentation:
+  Brief:       'Foo brief'
+  Returns:     'Foo returns'
+  Description: 'Foo description'
+  Notes:
+    - 'Foo note 1'
+    - 'Foo note 2'
+  Warnings:
+    - 'Foo warning 1'
+    - 'Foo warning 2'
+  Parameters:
+    - Name: 'param1'
+      Description: 'Foo param 1'
+    - Name: 'param2'
+      Description: 'Foo param 2'
+  CommentText: 'Full text would be here'
 ReturnType:    'int'
 IncludeHeaders:
   - Header:    'include1'
     References:    7
   - Header:    'include2'
     References:    3
 ...
 ---
TEST(SerializationTest, YAMLConversions) {
 
   auto Sym1 = *ParsedYAML->Symbols->find(
       cantFail(SymbolID::fromStr("057557CEBF6E6B2D")));
   auto Sym2 = *ParsedYAML->Symbols->find(
       cantFail(SymbolID::fromStr("057557CEBF6E6B2E")));
 
   EXPECT_THAT(Sym1, qName("clang::Foo1"));
   EXPECT_EQ(Sym1.Signature, "");
-  EXPECT_EQ(Sym1.Documentation, "Foo doc");
+
+  SymbolDocumentationRef ExpectedDocumentation;
+  ExpectedDocumentation.Brief = "Foo brief";
+  ExpectedDocumentation.Returns = "Foo returns";
+  ExpectedDocumentation.Description = "Foo description";
+  ExpectedDocumentation.Notes = {"Foo note 1", "Foo note 2"};
+  ExpectedDocumentation.Warnings = {"Foo warning 1", "Foo warning 2"};
+  ExpectedDocumentation.Parameters = {
+      {"param1", "Foo param 1"},
+      {"param2", "Foo param 2"},
+  };
+  ExpectedDocumentation.CommentText = "Full text would be here";
+  EXPECT_THAT(Sym1.Documentation, matchesDoc(ExpectedDocumentation));
+
   EXPECT_EQ(Sym1.ReturnType, "int");
   EXPECT_EQ(StringRef(Sym1.CanonicalDeclaration.FileURI), "file:///path/foo.h");
   EXPECT_EQ(Sym1.Origin, SymbolOrigin::Static);
   EXPECT_EQ(static_cast<uint8_t>(Sym1.Flags), 129);
   EXPECT_TRUE(Sym1.Flags & Symbol::IndexedForCodeCompletion);
   EXPECT_FALSE(Sym1.Flags & Symbol::Deprecated);
   EXPECT_THAT(Sym1.IncludeHeaders,
               UnorderedElementsAre(IncludeHeaderWithRef("include1", 7u),

clang-tools-extra/clangd/unittests/SymbolCollectorTests.cpp

 using ::testing::UnorderedElementsAre;
 using ::testing::UnorderedElementsAreArray;
 
 // GMock helpers for matching Symbol.
 MATCHER_P(labeled, Label, "") {
   return (arg.Name + arg.Signature).str() == Label;
 }
 MATCHER_P(returnType, D, "") { return arg.ReturnType == D; }
-MATCHER_P(doc, D, "") { return arg.Documentation == D; }
+MATCHER_P(doc, D, "") { return arg.Documentation.CommentText == D; }
 MATCHER_P(snippet, S, "") {
   return (arg.Name + arg.CompletionSnippetSuffix).str() == S;
 }
 MATCHER_P(qName, Name, "") { return (arg.Scope + arg.Name).str() == Name; }
 MATCHER_P(hasName, Name, "") { return arg.Name == Name; }
 MATCHER_P(templateArgs, TemplArgs, "") {
   return arg.TemplateSpecializationArgs == TemplArgs;
 }

clang-tools-extra/clangd/unittests/SymbolDocumentationMatchers.h

+//===-- SymbolDocumentationMatchers.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// GMock matchers for the SymbolDocumentation class
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_MATCHERS_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_MATCHERS_H
+#include "SymbolDocumentation.h"
+#include "gmock/gmock.h"
+
+namespace clang {
+namespace clangd {
+
+template <class S>
+testing::Matcher<SymbolDocumentation<S>>
+matchesDoc(const SymbolDocumentation<S> &Expected) {
+  using namespace ::testing;
+
+  std::vector<Matcher<ParameterDocumentation<S>>> ParamMatchers;
+  for (const auto &P : Expected.Parameters)
+    ParamMatchers.push_back(
+        AllOf(Field("Name", &ParameterDocumentation<S>::Name, P.Name),
+              Field("Description", &ParameterDocumentation<S>::Description,
+                    P.Description)));
+
+  return AllOf(
+      Field("Brief", &SymbolDocumentation<S>::Brief, Expected.Brief),
+      Field("Returns", &SymbolDocumentation<S>::Returns, Expected.Returns),
+      Field("Notes", &SymbolDocumentation<S>::Notes,
+            ElementsAreArray(Expected.Notes)),
+      Field("Warnings", &SymbolDocumentation<S>::Warnings,
+            ElementsAreArray(Expected.Warnings)),
+      Field("Parameters", &SymbolDocumentation<S>::Parameters,
+            ElementsAreArray(ParamMatchers)),
+      Field("Description", &SymbolDocumentation<S>::Description,
+            Expected.Description),
+      Field("CommentText", &SymbolDocumentation<S>::CommentText,
+            Expected.CommentText));
+}
+
+} // namespace clangd
+} // namespace clang
+
+#endif

llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn

sources = [
     "Protocol.cpp",
     "Quality.cpp",
     "QueryDriverDatabase.cpp",
     "RIFF.cpp",
     "Selection.cpp",
     "SemanticHighlighting.cpp",
     "SemanticSelection.cpp",
     "SourceCode.cpp",
+    "SymbolDocumentation.cpp"
     "TUScheduler.cpp",
     "TidyProvider.cpp",
     "URI.cpp",
     "XRefs.cpp",
     "index/Background.cpp",
     "index/BackgroundIndexLoader.cpp",
     "index/BackgroundIndexStorage.cpp",
     "index/BackgroundQueue.cpp",