Add some overview doxygen comments to lib/Format.

[thakis]

Summary

It always takes me a while to understand how clang-format's pieces fit together -- hopefully this will save me some time the next time I need it.

[djasper]

lib/Format/ContinuationIndenter.h

35

You are the first to introduce the term "pyhsical" line and I don't actually think it is the right term. Also, this is not a "helper class".

Maybe:

If an unwrapped line needs to be split over multiple lines to fit into the column limit, this class can be used to keep track of the indentation for the different levels of parentheses and to get the desired indentation for any given line break.

lib/Format/Format.cpp

761

/// \brief Splits a file into FormatTokens.

Also, this comment adds zero information.

1426

If we wanted to call them logical lines, we would have. Stick to the term unwrapped line.

1429

Same here, although I think you don't even mean logical lines.

lib/Format/FormatToken.h

110–111

What information does this comment add? I am not against adding one, but then describe in a bit more detail what is going on.

lib/Format/TokenAnnotator.h

39

What's the added info?

152

s/like/such as/

Also, I think that the comment is misleading. The LineType is the least important thing that this function does. It annotates all the tokens, for that parses all the expressions in it, assigned TokenTypes to the format tokens, and also annotates all Children of this line.

155–156

For all tokens in \c Line, use the additional information gathered during annotation to calculate formatting hints such as required line breaks, spaces and split penalties.

lib/Format/UnwrappedLineFormatter.h

31

Go through all of your comments and s/logical/unwrapped/

32

And the same with s/physical//. If that removes all the information, rephrase. E.g. here:

Takes a list of unwrapped lines and produces whitespace adjustments to lay them out within the column limit.

lib/Format/UnwrappedLineParser.h

61–62

This might be a good place to introduce the term unwrapped line. The most notable fact about unwrapped lines btw. is not that you would put them on a single line if there was no column limit. This is a nice thought model and we should also write it, but it isn't always true. More importantly, formatting decisions in one unwrapped line do not influence formatting decisions in a different unwrapped line.

lib/Format/ContinuationIndenter.h

 namespace format {
 
 class AnnotatedLine;
 struct FormatToken;
 struct LineState;
 struct ParenState;
 class WhitespaceManager;
 
+/// \brief Helper class for breaking a logical line into multiple physical
+/// lines.  Manages moving state from one physical line to the next.
 class ContinuationIndenter {
 public:
   /// \brief Constructs a \c ContinuationIndenter to format \p Line starting in
   /// column \p FirstIndent.
   ContinuationIndenter(const FormatStyle &Style,
                        const AdditionalKeywords &Keywords,
                        SourceManager &SourceMgr, WhitespaceManager &Whitespaces,
                        encoding::Encoding Encoding,

lib/Format/Format.cpp

std::string configurationAsText(const FormatStyle &Style) {
   // reference here.
   FormatStyle NonConstStyle = expandPresets(Style);
   Output << NonConstStyle;
   return Stream.str();
 }
 
 namespace {
 
+/// \brief Responsible for splitting up a file into \c FormatTokens.
 class FormatTokenLexer {
 public:
   FormatTokenLexer(SourceManager &SourceMgr, FileID ID, FormatStyle &Style,
                    encoding::Encoding Encoding)
       : FormatTok(nullptr), IsFirstToken(true), GreaterStashed(false),
         LessStashed(false), Column(0), TrailingWhitespace(0),
         SourceMgr(SourceMgr), ID(ID), Style(Style),
         IdentTable(getFormattingLangOpts(Style)), Keywords(IdentTable),
case FormatStyle::LK_JavaScript:
     return "JavaScript";
   case FormatStyle::LK_Proto:
     return "Proto";
   default:
     return "Unknown";
   }
 }
 
+/// \brief Responsible for taking a file and producing whitespace adjustments
+/// to reformat the file according to a desired style.
+///
+/// The pipeline used by \c Formatter:
+/// -# A \c FormatTokenLexer splits the file into raw tokens
+/// -# A \c UnwrappedLineParser converts those into logical lines
+/// -# A \c TokenAnnotator computes metadata for the tokens in each line
+/// -# A \c UnwrappedLineFormatter then produces whitespace adjustments
+///    which result in the desired logical lines
 class Formatter : public UnwrappedLineConsumer {
 public:
   Formatter(const FormatStyle &Style, SourceManager &SourceMgr, FileID ID,
             ArrayRef<CharSourceRange> Ranges)
       : Style(Style), ID(ID), SourceMgr(SourceMgr),
         Whitespaces(SourceMgr, Style,
                     inputUsesCRLF(SourceMgr.getBufferData(ID))),
         Ranges(Ranges.begin(), Ranges.end()), UnwrappedLines(1),

lib/Format/FormatToken.h

 // The packing kind of a function's parameters.
 enum ParameterPackingKind { PPK_BinPacked, PPK_OnePerLine, PPK_Inconclusive };
 
 enum FormatDecision { FD_Unformatted, FD_Continue, FD_Break };
 
 class TokenRole;
 class AnnotatedLine;
 
-/// \brief A wrapper around a \c Token storing information about the
-/// whitespace characters preceding it.
+/// \brief A wrapper around a \c Token storing additional information useful
+/// for formatting.
 struct FormatToken {
   FormatToken() {}
 
   /// \brief The \c Token.
   Token Tok;
 
   /// \brief The number of newlines immediately before the \c Token.
   ///

lib/Format/TokenAnnotator.h

enum LineType {
   LT_ObjCDecl, // An @interface, @implementation, or @protocol line.
   LT_ObjCMethodDecl,
   LT_ObjCProperty, // An @property line.
   LT_Other,
   LT_PreprocessorDirective,
   LT_VirtualFunctionDecl
 };
 
+/// \brief Stores additional information for an \c UnwrappedLine.
 class AnnotatedLine {
 public:
   AnnotatedLine(const UnwrappedLine &Line)
       : First(Line.Tokens.front().Tok), Level(Line.Level),
         InPPDirective(Line.InPPDirective),
         MustBeDeclaration(Line.MustBeDeclaration), MightBeFunctionDecl(false),
         IsMultiVariableDeclStmt(false), Affected(false),
         LeadingEmptyLinesAffected(false), ChildrenAffected(false) {
public:
   TokenAnnotator(const FormatStyle &Style, const AdditionalKeywords &Keywords)
       : Style(Style), Keywords(Keywords) {}
 
   /// \brief Adapts the indent levels of comment lines to the indent of the
   /// subsequent line.
   // FIXME: Can/should this be done in the UnwrappedLineParser?
   void setCommentLineLevels(SmallVectorImpl<AnnotatedLine *> &Lines);
 
+  /// \brief Compute metadata of \c AnnotatedLine itself, like \c LineType.
   void annotate(AnnotatedLine &Line);
+
+  /// \brief For all the tokens in \c Line, compute if a line break is
+  /// needed for that token, and similar information.
   void calculateFormattingInformation(AnnotatedLine &Line);
 
 private:
   /// \brief Calculate the penalty for splitting before \c Tok.
   unsigned splitPenalty(const AnnotatedLine &Line, const FormatToken &Tok,
                         bool InFunctionDecl);
 
   bool spaceRequiredBetween(const AnnotatedLine &Line, const FormatToken &Left,

lib/Format/UnwrappedLineFormatter.h

 #include <string>
 
 namespace clang {
 namespace format {
 
 class ContinuationIndenter;
 class WhitespaceManager;
 
+/// \brief Responsible for taking a list of logical lines, and for producing
+/// a set of whitespace adjustments to turn them into physical lines so
+/// that each physical line is below a column limit.
 class UnwrappedLineFormatter {
 public:
   UnwrappedLineFormatter(ContinuationIndenter *Indenter,
                          WhitespaceManager *Whitespaces,
                          const FormatStyle &Style,
                          const AdditionalKeywords &Keywords,
                          bool *IncompleteFormat)
       : Indenter(Indenter), Whitespaces(Whitespaces), Style(Style),

lib/Format/UnwrappedLineParser.h

 public:
   virtual ~UnwrappedLineConsumer() {}
   virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
   virtual void finishRun() = 0;
 };
 
 class FormatTokenSource;
 
+/// \brief Takes a sequence of tokens, and splits it into chunks of tokens
+/// that would be a single line each if there was no column limit.
 class UnwrappedLineParser {
 public:
   UnwrappedLineParser(const FormatStyle &Style,
                       const AdditionalKeywords &Keywords,
                       ArrayRef<FormatToken *> Tokens,
                       UnwrappedLineConsumer &Callback);
 
   void parse();