Index: tools/CMakeLists.txt =================================================================== --- tools/CMakeLists.txt +++ tools/CMakeLists.txt @@ -3,6 +3,7 @@ add_clang_subdirectory(diagtool) add_clang_subdirectory(driver) add_clang_subdirectory(clang-diff) +add_clang_subdirectory(clang-doc) add_clang_subdirectory(clang-format) add_clang_subdirectory(clang-format-vs) add_clang_subdirectory(clang-fuzzer) Index: tools/clang-doc/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-doc/CMakeLists.txt @@ -0,0 +1,21 @@ +set(LLVM_LINK_COMPONENTS + support + ) + +add_clang_library(clangDoc + ClangDoc.cpp + ClangDocReporter.cpp + + LINK_LIBS + clangAnalysis + clangAST + clangASTMatchers + clangBasic + clangFormat + clangFrontend + clangLex + clangTooling + clangToolingCore + ) + +add_subdirectory(tool) Index: tools/clang-doc/ClangDoc.h =================================================================== --- /dev/null +++ tools/clang-doc/ClangDoc.h @@ -0,0 +1,88 @@ +//===-- ClangDoc.cpp - ClangDoc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H + +#include "ClangDocReporter.h" +#include "clang/AST/AST.h" +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Frontend/ASTConsumers.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/Tooling.h" +#include +#include + +namespace clang { +namespace doc { + +// A Context which contains extra options which are used in ClangMoveTool. +struct ClangDocContext { + // Which format to emit representation in. + OutFormat EmitFormat; +}; + +class ClangDocVisitor : public RecursiveASTVisitor { +public: + explicit ClangDocVisitor(ASTContext *Context, ClangDocReporter &Reporter) + : Context(Context), Reporter(Reporter) {} + + virtual bool VisitNamedDecl(const NamedDecl *D); + + void ParseUnattachedComments(); + bool IsNewComment(SourceLocation Loc, SourceManager &Manager) const; + +private: + ASTContext *Context; + ClangDocReporter &Reporter; +}; + +class ClangDocConsumer : public clang::ASTConsumer { +public: + explicit ClangDocConsumer(ASTContext *Context, ClangDocReporter &Reporter) + : Visitor(Context, Reporter), Reporter(Reporter) {} + + virtual void HandleTranslationUnit(clang::ASTContext &Context); + +private: + ClangDocVisitor Visitor; + ClangDocReporter &Reporter; +}; + +class ClangDocAction : public clang::ASTFrontendAction { +public: + ClangDocAction(ClangDocReporter &Reporter) : Reporter(Reporter) {} + + virtual std::unique_ptr CreateASTConsumer(clang::CompilerInstance &Compiler, llvm::StringRef InFile); + virtual void EndSourceFileAction(); + +private: + ClangDocReporter &Reporter; +}; + +class ClangDocActionFactory : public tooling::FrontendActionFactory { +public: + ClangDocActionFactory(ClangDocContext &Context, ClangDocReporter &Reporter) + : Context(Context), Reporter(Reporter) {} + + clang::FrontendAction *create() override { + return new ClangDocAction(Reporter); + } + +private: + ClangDocContext &Context; + ClangDocReporter &Reporter; +}; + +} // namespace doc +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANGDOC_H Index: tools/clang-doc/ClangDoc.cpp =================================================================== --- /dev/null +++ tools/clang-doc/ClangDoc.cpp @@ -0,0 +1,90 @@ +//===-- ClangDoc.cpp - ClangDoc ---------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ClangDoc.h" +#include "clang/AST/AST.h" +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/Comment.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Frontend/ASTConsumers.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/Tooling.h" + +using namespace clang; +using namespace clang::tooling; +using namespace llvm; + +namespace clang { +namespace doc { + +// TODO: limit to functions/objects/namespaces/etc? +bool ClangDocVisitor::VisitNamedDecl(const NamedDecl *D) { + SourceManager &Manager = Context->getSourceManager(); + if (!IsNewComment(D->getLocation(), Manager)) + return true; + + DeclInfo DI; + DI.D = D; + DI.QualifiedName = D->getQualifiedNameAsString(); + RawComment *Comment = Context->getRawCommentForDeclNoCache(D); + + // TODO: Move set attached to the initial comment parsing, not here + if (Comment) { + Comment->setAttached(); + DI.Comment = + Reporter.ParseFullComment(Comment->parse(*Context, nullptr, D)); + } + Reporter.AddDecl(Manager.getFilename(D->getLocation()), DI); + return true; +} + +void ClangDocVisitor::ParseUnattachedComments() { + SourceManager &Manager = Context->getSourceManager(); + for (RawComment *Comment : Context->getRawCommentList().getComments()) { + if (!IsNewComment(Comment->getLocStart(), Manager) || Comment->isAttached()) + continue; + CommentInfo CI = + Reporter.ParseFullComment(Comment->parse(*Context, nullptr, nullptr)); + Reporter.AddComment(Manager.getFilename(Comment->getLocStart()), CI); + } +} + +bool ClangDocVisitor::IsNewComment(SourceLocation Loc, + SourceManager &Manager) const { + if (!Loc.isValid()) + return false; + const std::string &Filename = Manager.getFilename(Loc); + if (!Reporter.HasFile(Filename) || Reporter.HasSeenFile(Filename)) + return false; + if (Manager.isInSystemHeader(Loc) || Manager.isInExternCSystemHeader(Loc)) + return false; + Reporter.AddFileInTU(Filename); + return true; +} + +void ClangDocConsumer::HandleTranslationUnit(ASTContext &Context) { + Visitor.TraverseDecl(Context.getTranslationUnitDecl()); + Visitor.ParseUnattachedComments(); +} + +std::unique_ptr ClangDocAction::CreateASTConsumer(CompilerInstance &Compiler, StringRef InFile) { + return llvm::make_unique(&Compiler.getASTContext(), Reporter); +} + +void ClangDocAction::EndSourceFileAction() { + for (const auto &Filename : Reporter.GetFilesInThisTU()) { + Reporter.AddFileSeen(Filename); + } + Reporter.ClearFilesInThisTU(); +} + +} // namespace doc +} // namespace clang Index: tools/clang-doc/ClangDocReporter.h =================================================================== --- /dev/null +++ tools/clang-doc/ClangDocReporter.h @@ -0,0 +1,114 @@ +//===-- Doc.cpp - ClangDoc --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANG_DOC_REPORTER_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANG_DOC_REPORTER_H + +#include "clang/AST/AST.h" +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/CommentVisitor.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Frontend/ASTConsumers.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include + +using namespace clang::comments; + +namespace clang { +namespace doc { + +enum class OutFormat {YAML, LLVM}; + +struct StringPair { + std::string Key; + std::string Value; +}; + +struct CommentInfo { + std::string Kind; + std::string Text; + std::string Name; + std::string Direction; + std::string ParamName; + std::string CloseName; + bool SelfClosing = false; + bool Explicit = false; + llvm::StringMap Attrs; + llvm::SmallVector Args; + llvm::SmallVector Position; + std::vector Children; +}; + +// TODO: collect declarations of the same object, comment is preferentially: +// 1) docstring on definition, 2) combined docstring from non-def decls, or +// 3) comment on definition, 4) no comment. +struct DeclInfo { + const Decl *D; + std::string QualifiedName; + CommentInfo Comment; +}; + +struct FileRecord { + std::string Filename; + std::vector Decls; + std::vector UnattachedComments; +}; + +class ClangDocReporter : public ConstCommentVisitor { +public: + ClangDocReporter(const std::vector &SourcePathList); + + void AddComment(StringRef Filename, CommentInfo &CI); + void AddDecl(StringRef Filename, DeclInfo &D); + void AddFile(StringRef Filename); + void AddFileInTU(StringRef Filename) { FilesInThisTU.insert(Filename); } + void AddFileSeen(StringRef Filename) { FilesSeen.insert(Filename); } + void ClearFilesInThisTU() { FilesInThisTU.clear(); }; + + void visitTextComment(const TextComment *C); + void visitInlineCommandComment(const InlineCommandComment *C); + void visitHTMLStartTagComment(const HTMLStartTagComment *C); + void visitHTMLEndTagComment(const HTMLEndTagComment *C); + void visitBlockCommandComment(const BlockCommandComment *C); + void visitParamCommandComment(const ParamCommandComment *C); + void visitTParamCommandComment(const TParamCommandComment *C); + void visitVerbatimBlockComment(const VerbatimBlockComment *C); + void visitVerbatimBlockLineComment(const VerbatimBlockLineComment *C); + void visitVerbatimLineComment(const VerbatimLineComment *C); + + CommentInfo ParseFullComment(comments::FullComment *Comment); + + const std::set& GetFilesInThisTU() const { return FilesInThisTU; } + bool HasFile(StringRef Filename) const; + bool HasSeenFile(StringRef Filename) const; + void Serialize(clang::doc::OutFormat Format, llvm::raw_ostream &OS) const; + +private: + void parseComment(CommentInfo *CI, comments::Comment *C); + void serializeYAML(llvm::raw_ostream &OS) const; + void serializeLLVM(llvm::raw_ostream &OS) const; + const char *getCommandName(unsigned CommandID); + bool isWhitespaceOnly(StringRef S); + + CommentInfo *CurrentCI; + llvm::StringMap FileRecords; + std::set FilesInThisTU; + std::set FilesSeen; +}; + +} // namespace doc +} // namespace clang + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_CLANG_DOC_REPORTER_H Index: tools/clang-doc/ClangDocReporter.cpp =================================================================== --- /dev/null +++ tools/clang-doc/ClangDocReporter.cpp @@ -0,0 +1,259 @@ +//===-- Doc.cpp - ClangDoc --------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ClangDocReporter.h" +#include "clang/AST/AST.h" +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/CommentVisitor.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Frontend/ASTConsumers.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; +using namespace clang::tooling; +using namespace llvm; + +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::doc::DeclInfo) +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::doc::CommentInfo) +LLVM_YAML_IS_SEQUENCE_VECTOR(clang::doc::StringPair) + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &IO, clang::doc::StringPair &Pair) { + IO.mapRequired("Key", Pair.Key); + IO.mapRequired("Value", Pair.Value); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, clang::doc::FileRecord &Info) { + IO.mapRequired("Filename", Info.Filename); + IO.mapRequired("Decls", Info.Decls); + IO.mapRequired("UnattachedComments", Info.UnattachedComments); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, clang::doc::DeclInfo &Info) { + IO.mapRequired("Name", Info.QualifiedName); + IO.mapRequired("Comment", Info.Comment); + } +}; + +template <> struct MappingTraits { + + struct NormalizedStringMap { + NormalizedStringMap(IO &) {} + NormalizedStringMap(IO &, const llvm::StringMap &Map) { + for (const auto &Entry : Map) { + clang::doc::StringPair Pair{Entry.getKeyData(), Entry.getValue()}; + VectorMap.push_back(Pair); + } + } + + llvm::StringMap denormalize(IO &) { + llvm::StringMap Map; + for (const auto &Pair : VectorMap) + Map[Pair.Key] = Pair.Value; + return Map; + } + + std::vector VectorMap; + }; + + static void mapping(IO &IO, clang::doc::CommentInfo &Info) { + MappingNormalization> keys(IO, Info.Attrs); + + IO.mapRequired("Kind", Info.Kind); + if (!Info.Text.empty()) + IO.mapOptional("Text", Info.Text); + if (!Info.Name.empty()) + IO.mapOptional("Text", Info.Name); + if (!Info.Direction.empty()) + IO.mapOptional("Direction", Info.Direction); + if (!Info.ParamName.empty()) + IO.mapOptional("ParamName", Info.ParamName); + if (!Info.CloseName.empty()) + IO.mapOptional("CloseName", Info.CloseName); + if (Info.SelfClosing) + IO.mapOptional("SelfClosing", Info.SelfClosing); + if (Info.Explicit) + IO.mapOptional("Explicit", Info.Explicit); + if (Info.Args.size() > 0) + IO.mapOptional("Args", Info.Args); + if (Info.Attrs.size() > 0) + IO.mapOptional("Attrs", keys->VectorMap); + if (Info.Position.size() > 0) + IO.mapOptional("Position", Info.Position); + if (Info.Children.size() > 0) + IO.mapOptional("Children", Info.Children); + } +}; + +} // end namespace yaml +} // end namespace llvm + +namespace clang { +namespace doc { + +ClangDocReporter::ClangDocReporter(const std::vector &SourcePathList) { + for (const std::string &Path : SourcePathList) + AddFile(Path); +} + +void ClangDocReporter::AddComment(StringRef Filename, CommentInfo &CI) { + FileRecords[Filename].UnattachedComments.push_back(CI); +} + +void ClangDocReporter::AddDecl(StringRef Filename, DeclInfo &DI) { + FileRecords[Filename].Decls.push_back(DI); +} + +void ClangDocReporter::AddFile(StringRef Filename) { + FileRecord FI; + FI.Filename = Filename; + FileRecords.insert(std::make_pair(Filename, FI)); +} + +CommentInfo ClangDocReporter::ParseFullComment(comments::FullComment *Comment) { + CommentInfo CI; + parseComment(&CI, Comment); + return CI; +} + +void ClangDocReporter::visitTextComment(const TextComment *C) { + if (!isWhitespaceOnly(C->getText())) + CurrentCI->Text = C->getText(); +} + +void ClangDocReporter::visitInlineCommandComment( + const InlineCommandComment *C) { + CurrentCI->Name = getCommandName(C->getCommandID()); + for (unsigned i = 0, e = C->getNumArgs(); i != e; ++i) + CurrentCI->Args.push_back(C->getArgText(i)); +} + +void ClangDocReporter::visitHTMLStartTagComment(const HTMLStartTagComment *C) { + CurrentCI->Name = C->getTagName(); + CurrentCI->SelfClosing = C->isSelfClosing(); + if (C->getNumAttrs() != 0) { + for (unsigned i = 0, e = C->getNumAttrs(); i != e; ++i) { + const HTMLStartTagComment::Attribute &Attr = C->getAttr(i); + CurrentCI->Attrs.insert(std::make_pair(Attr.Name, Attr.Value)); + } + } +} + +void ClangDocReporter::visitHTMLEndTagComment(const HTMLEndTagComment *C) { + CurrentCI->Name = C->getTagName(); + CurrentCI->SelfClosing = true; +} + +void ClangDocReporter::visitBlockCommandComment(const BlockCommandComment *C) { + CurrentCI->Name = getCommandName(C->getCommandID()); + for (unsigned i = 0, e = C->getNumArgs(); i != e; ++i) + CurrentCI->Args.push_back(C->getArgText(i)); +} + +void ClangDocReporter::visitParamCommandComment(const ParamCommandComment *C) { + CurrentCI->Direction = + ParamCommandComment::getDirectionAsString(C->getDirection()); + CurrentCI->Explicit = C->isDirectionExplicit(); + if (C->hasParamName() && C->isParamIndexValid()) + CurrentCI->ParamName = C->getParamNameAsWritten(); +} + +void ClangDocReporter::visitTParamCommandComment( + const TParamCommandComment *C) { + if (C->hasParamName() && C->isPositionValid()) + CurrentCI->ParamName = C->getParamNameAsWritten(); + + if (C->isPositionValid()) { + for (unsigned i = 0, e = C->getDepth(); i != e; ++i) + CurrentCI->Position.push_back(C->getIndex(i)); + } +} + +void ClangDocReporter::visitVerbatimBlockComment( + const VerbatimBlockComment *C) { + CurrentCI->Name = getCommandName(C->getCommandID()); + CurrentCI->CloseName = C->getCloseName(); +} + +void ClangDocReporter::visitVerbatimBlockLineComment( + const VerbatimBlockLineComment *C) { + if (!isWhitespaceOnly(C->getText())) + CurrentCI->Text = C->getText(); +} + +void ClangDocReporter::visitVerbatimLineComment(const VerbatimLineComment *C) { + if (!isWhitespaceOnly(C->getText())) + CurrentCI->Text = C->getText(); +} + +bool ClangDocReporter::HasFile(StringRef Filename) const { + return FileRecords.find(Filename) != FileRecords.end(); +} + +bool ClangDocReporter::HasSeenFile(StringRef Filename) const { + return FilesSeen.find(Filename) != FilesSeen.end(); +} + +void ClangDocReporter::Serialize(clang::doc::OutFormat Format, + llvm::raw_ostream &OS) const { + Format == clang::doc::OutFormat::LLVM ? serializeLLVM(OS) : serializeYAML(OS); +} + +void ClangDocReporter::parseComment(CommentInfo *CI, comments::Comment *C) { + CurrentCI = CI; + CI->Kind = C->getCommentKindName(); + ConstCommentVisitor::visit(C); + for (comments::Comment *Child : llvm::make_range(C->child_begin(), C->child_end())) { + CommentInfo ChildCI; + parseComment(&ChildCI, Child); + CI->Children.push_back(ChildCI); + } +} + +void ClangDocReporter::serializeYAML(llvm::raw_ostream &OS) const { + yaml::Output Output(OS); + for (const auto &F : FileRecords) { + FileRecord NonConstValue = F.second; + Output << NonConstValue; + } +} + +void ClangDocReporter::serializeLLVM(llvm::raw_ostream &OS) const { + // TODO: Implement. + OS << "Not yet implemented.\n"; +} + +const char *ClangDocReporter::getCommandName(unsigned CommandID) { + ; + const CommandInfo *Info = CommandTraits::getBuiltinCommandInfo(CommandID); + if (Info) + return Info->Name; + // TODO: Add parsing for \file command. + return ""; +} + +bool ClangDocReporter::isWhitespaceOnly(StringRef S) { + return S.find_first_not_of(" \t\n\v\f\r") == std::string::npos || S.empty(); +} + +} // namespace doc +} // namespace clang Index: tools/clang-doc/tool/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-doc/tool/CMakeLists.txt @@ -0,0 +1,18 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) + +add_clang_executable(clang-doc + ClangDocMain.cpp + ) + +target_link_libraries(clang-doc + clangAST + clangASTMatchers + clangBasic + clangFormat + clangFrontend + clangDoc + clangRewrite + clangTooling + clangToolingCore + ) + \ No newline at end of file Index: tools/clang-doc/tool/ClangDocMain.cpp =================================================================== --- /dev/null +++ tools/clang-doc/tool/ClangDocMain.cpp @@ -0,0 +1,69 @@ +//===-- ClangDocMain.cpp - Clangdoc -----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "ClangDoc.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Signals.h" +#include + +using namespace clang; +using namespace llvm; + +namespace { + +cl::OptionCategory ClangDocCategory("clang-doc options"); + +cl::opt + EmitLLVM("emit-llvm", + cl::desc("Output in LLVM bitstream format (default is YAML)."), + cl::init(false), cl::cat(ClangDocCategory)); + +cl::opt + DoxygenOnly("doxygen", + cl::desc("Use only doxygen-style comments to generate docs."), + cl::init(false), cl::cat(ClangDocCategory)); + +} // namespace + +int main(int argc, const char **argv) { + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + tooling::CommonOptionsParser OptionsParser(argc, argv, ClangDocCategory); + + clang::doc::OutFormat EmitFormat; + EmitLLVM ? EmitFormat = clang::doc::OutFormat::LLVM : + EmitFormat = clang::doc::OutFormat::YAML; + + // TODO: Update the source path list to only consider changed files for + // incremental doc updates + doc::ClangDocReporter Reporter(OptionsParser.getSourcePathList()); + doc::ClangDocContext Context{EmitFormat}; + + tooling::ClangTool Tool(OptionsParser.getCompilations(), + OptionsParser.getSourcePathList()); + + if (!DoxygenOnly) + Tool.appendArgumentsAdjuster(tooling::getInsertArgumentAdjuster( + "-fparse-all-comments", tooling::ArgumentInsertPosition::BEGIN)); + + doc::ClangDocActionFactory Factory(Context, Reporter); + + llvm::outs() << "Parsing codebase...\n"; + int Status = Tool.run(&Factory); + if (Status) + return Status; + + llvm::outs() << "Writing docs...\n"; + Reporter.Serialize(EmitFormat, llvm::outs()); + + return 0; +}