[pseudo] Use a real language option in the parser.

[hokein]

rebase

clang-tools-extra/pseudo/include/clang-pseudo/Token.h

 //                  (pseudo::TokenStream is similar, but a flat token list).
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef CLANG_PSEUDO_TOKEN_H
 #define CLANG_PSEUDO_TOKEN_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangStandard.h"
 #include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdint>
 #include <limits>
 #include <memory>
 #include <vector>
 
enum class LexFlags : uint8_t {
   ///
   /// Careful, the directive may not start exactly on the StartsPPLine token:
   ///   /*comment*/ #include <foo.h>
   StartsPPLine = 1 << 0,
   /// Marks tokens containing trigraphs, escaped newlines, UCNs etc.
   /// The text() of such tokens will contain the raw trigrah.
   NeedsCleaning = 1 << 1,
 };
+/// A generic lang options suitable for lexing/parsing a langage.
+clang::LangOptions genericLangOpts(
+    clang::Language = clang::Language::CXX,
+    clang::LangStandard::Kind = clang::LangStandard::lang_unspecified);
 
 /// Derives a token stream by decoding escapes, interpreting raw_identifiers and
 /// splitting the greatergreater token.
 ///
 /// Tokens containing UCNs, escaped newlines, trigraphs etc are decoded and
 /// their backing data is owned by the returned stream.
 /// raw_identifier tokens are assigned specific types (identifier, keyword etc).
 ///

clang-tools-extra/pseudo/lib/Token.cpp

 //===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang-pseudo/Token.h"
+#include "clang/Basic/LangOptions.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormatVariadic.h"
 
 namespace clang {
 namespace pseudo {
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) {
for (const auto &T : tokens()) {
     OS << Text;
     LastLine = T.Line;
     LastText = Text;
   }
   if (!FirstToken)
     OS << '\n';
 }
 
+clang::LangOptions genericLangOpts(clang::Language Lang,
+                                   clang::LangStandard::Kind Standard) {
+  clang::LangOptions Opts;
+  std::vector<std::string> UnusedIncludes;
+  LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes,
+                               Standard);
+
+  // Some options are "on by default", but e.g. at the driver level.
+  if (Opts.CPlusPlus)
+    Opts.CXXOperatorNames = true;
+  if (Opts.CPlusPlus20)
+    Opts.Coroutines = true;
+
+  // Some options are off by default, but define keywords we want to tolerate.
+  if (Opts.CPlusPlus)
+    Opts.MicrosoftExt = true;  // kw__try, kw__finally
+  Opts.DeclSpecKeyword = true; // __declspec
+  Opts.WChar = true;
+
+  return Opts;
+}
+
 TokenStream stripComments(const TokenStream &Input) {
   TokenStream Out;
   for (const Token &T : Input.tokens()) {
     if (T.Kind == tok::comment)
       continue;
     Out.push(T);
   }
   Out.finalize();
   return Out;
 }
 
 } // namespace pseudo
 } // namespace clang

clang-tools-extra/pseudo/test/glr.cpp

 // CHECK-NEXT:   │ │ ├─type-name~IDENTIFIER := tok[5]
 // CHECK-NEXT:   │ │ └─type-name~IDENTIFIER := tok[5]
 // CHECK-NEXT:   │ └─simple-type-specifier~IDENTIFIER := tok[5]
 // CHECK-NEXT:   ├─init-declarator-list~ptr-declarator := ptr-operator ptr-declarator
 // CHECK-NEXT:   │ ├─ptr-operator~* := tok[6]
 // CHECK-NEXT:   │ └─ptr-declarator~IDENTIFIER := tok[7]
 // CHECK-NEXT:   └─; := tok[8]
 }
+
+bool operator<();
+// CHECK:      declaration~simple-declaration := decl-specifier-seq init-declarator-list ;
+// CHECK-NEXT: ├─decl-specifier-seq~BOOL
+// CHECK-NEXT: ├─init-declarator-list~noptr-declarator := noptr-declarator parameters-and-qualifiers
+// CHECK-NEXT: │ ├─noptr-declarator~operator-function-id := OPERATOR operator-name
+// CHECK-NEXT: │ │ ├─OPERATOR
+// CHECK-NEXT: │ │ └─operator-name~<
+// CHECK-NEXT: │ └─parameters-and-qualifiers := ( )
+// CHECK-NEXT: │   ├─(
+// CHECK-NEXT: │   └─)
+// CHECK-NEXT: └─;

clang-tools-extra/pseudo/tool/ClangPseudo.cpp

if (std::error_code EC = Text.getError()) {
     ::exit(1);
   }
   return Text.get()->getBuffer().str();
 }
 
 int main(int argc, char *argv[]) {
   llvm::cl::ParseCommandLineOptions(argc, argv, "");
 
-  clang::LangOptions LangOpts; // FIXME: use real options.
-  LangOpts.CPlusPlus = 1;
+  clang::LangOptions LangOpts = clang::pseudo::genericLangOpts();
   std::string SourceText;
   llvm::Optional<clang::pseudo::TokenStream> RawStream;
   llvm::Optional<clang::pseudo::DirectiveTree> DirectiveStructure;
   llvm::Optional<clang::pseudo::TokenStream> ParseableStream;
   if (Source.getNumOccurrences()) {
     SourceText = readOrDie(Source);
     RawStream = clang::pseudo::lex(SourceText, LangOpts);
     DirectiveStructure = clang::pseudo::DirectiveTree::parse(*RawStream);