[clangd] Use tokenize instead of raw lexer in SourceCode/lex

[hokein]

clang-tools-extra/clangd/SourceCode.cpp

728

why not inline the above if here?

[kadircet]

• Address comments

clang-tools-extra/clangd/SourceCode.cpp

 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Driver/Types.h"
 #include "clang/Format/Format.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/Token.h"
 #include "clang/Tooling/Core/Replacement.h"
+#include "clang/Tooling/Syntax/Tokens.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Errc.h"
cleanupAndFormat(StringRef Code, const tooling::Replacements &Replaces,
   auto CleanReplaces = cleanupAroundReplacements(Code, Replaces, Style);
   if (!CleanReplaces)
     return CleanReplaces;
   return formatReplacements(Code, std::move(*CleanReplaces), Style);
 }
 
 static void
 lex(llvm::StringRef Code, const LangOptions &LangOpts,
-    llvm::function_ref<void(const clang::Token &, const SourceManager &SM)>
+    llvm::function_ref<void(const syntax::Token &, const SourceManager &SM)>
         Action) {
   // FIXME: InMemoryFileAdapter crashes unless the buffer is null terminated!
   std::string NullTerminatedCode = Code.str();
   SourceManagerForFile FileSM("dummy.cpp", NullTerminatedCode);
   auto &SM = FileSM.get();
-  auto FID = SM.getMainFileID();
-  // Create a raw lexer (with no associated preprocessor object).
-  Lexer Lex(FID, SM.getBuffer(FID), SM, LangOpts);
-  Token Tok;
-
-  while (!Lex.LexFromRawLexer(Tok))
-    Action(Tok, SM);
-  // LexFromRawLexer returns true after it lexes last token, so we still have
-  // one more token to report.
+  for (const auto &Tok : syntax::tokenize(SM.getMainFileID(), SM, LangOpts))
     Action(Tok, SM);
 }
 
 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
                                              const format::FormatStyle &Style) {
   llvm::StringMap<unsigned> Identifiers;
   auto LangOpt = format::getFormattingLangOpts(Style);
-  lex(Content, LangOpt, [&](const clang::Token &Tok, const SourceManager &) {
-    if (Tok.getKind() == tok::raw_identifier)
-      ++Identifiers[Tok.getRawIdentifier()];
+  lex(Content, LangOpt, [&](const syntax::Token &Tok, const SourceManager &SM) {
+    if (Tok.kind() == tok::identifier)
+      ++Identifiers[Tok.text(SM)];
+    // FIXME: Should this function really return keywords too ?
+    else if (auto *Keyword = tok::getKeywordSpelling(Tok.kind()))
+      ++Identifiers[Keyword];
   });
   return Identifiers;
 }
 
 std::vector<Range> collectIdentifierRanges(llvm::StringRef Identifier,
                                            llvm::StringRef Content,
                                            const LangOptions &LangOpts) {
   std::vector<Range> Ranges;
-  lex(Content, LangOpts, [&](const clang::Token &Tok, const SourceManager &SM) {
-    if (Tok.getKind() != tok::raw_identifier)
-      return;
-    if (Tok.getRawIdentifier() != Identifier)
-      return;
-    auto Range = getTokenRange(SM, LangOpts, Tok.getLocation());
-    if (!Range)
+  lex(Content, LangOpts,
+      [&](const syntax::Token &Tok, const SourceManager &SM) {
+        if (Tok.kind() != tok::identifier || Tok.text(SM) != Identifier)
           return;
+        if (auto Range = getTokenRange(SM, LangOpts, Tok.location()))
           Ranges.push_back(*Range);
       });
   return Ranges;
 }
 
 namespace {
 struct NamespaceEvent {
   enum {
     BeginNamespace, // namespace <ns> {.     Payload is resolved <ns>.
     EndNamespace,   // } // namespace <ns>.  Payload is resolved *outer*
enum {
     Using,              // just saw 'using'
     UsingNamespace,     // just saw 'using namespace'
     UsingNamespaceName, // just saw 'using namespace' NSName
   } State = Default;
   std::string NSName;
 
   NamespaceEvent Event;
   lex(Code, format::getFormattingLangOpts(Style),
-      [&](const clang::Token &Tok,const SourceManager &SM) {
-    Event.Pos = sourceLocToPosition(SM, Tok.getLocation());
-    switch (Tok.getKind()) {
-    case tok::raw_identifier:
-      // In raw mode, this could be a keyword or a name.
+      [&](const syntax::Token &Tok, const SourceManager &SM) {
+        Event.Pos = sourceLocToPosition(SM, Tok.location());
+        switch (Tok.kind()) {
+        case tok::kw_using:
+          State = State == Default ? Using : Default;
+          break;
+        case tok::kw_namespace:
+          switch (State) {
+          case Using:
+            State = UsingNamespace;
+            break;
+          case Default:
+            State = Namespace;
+            break;
+          default:
+            State = Default;
+            break;
+          }
+          break;
+        case tok::identifier:
+          if (State == Using || State == Default) {
+            State = Default;
+            break;
+          }
           switch (State) {
           case UsingNamespace:
+            NSName.clear();
+            LLVM_FALLTHROUGH;
           case UsingNamespaceName:
-        NSName.append(std::string(Tok.getRawIdentifier()));
+            NSName.append(Tok.text(SM).str());
             State = UsingNamespaceName;
             break;
           case Namespace:
+            NSName.clear();
+            LLVM_FALLTHROUGH;
           case NamespaceName:
-        NSName.append(std::string(Tok.getRawIdentifier()));
+            NSName.append(Tok.text(SM).str());
             State = NamespaceName;
             break;
           case Using:
-        State =
-            (Tok.getRawIdentifier() == "namespace") ? UsingNamespace : Default;
-        break;
           case Default:
-        NSName.clear();
-        if (Tok.getRawIdentifier() == "namespace")
-          State = Namespace;
-        else if (Tok.getRawIdentifier() == "using")
-          State = Using;
-        break;
+            llvm_unreachable("Using and Default handled above.");
           }
           break;
         case tok::coloncolon:
-      // This can come at the beginning or in the middle of a namespace name.
+          // This can come at the beginning or in the middle of a namespace
+          // name.
           switch (State) {
           case UsingNamespace:
+            NSName.clear();
+            LLVM_FALLTHROUGH;
           case UsingNamespaceName:
             NSName.append("::");
             State = UsingNamespaceName;
             break;
           case NamespaceName:
             NSName.append("::");
             State = NamespaceName;
             break;
           case Namespace: // Not legal here.
           case Using:
           case Default:
             State = Default;
             break;
           }
           break;
         case tok::l_brace:
           // Record which { started a namespace, so we know when } ends one.
           if (State == NamespaceName) {
             // Parsed: namespace <name> {
             BraceStack.push_back(true);
             Enclosing.push_back(NSName);
             Event.Trigger = NamespaceEvent::BeginNamespace;
             Event.Payload = llvm::join(Enclosing, "::");
             Callback(Event);
           } else {
             // This case includes anonymous namespaces (State = Namespace).
             // For our purposes, they're not namespaces and we ignore them.
             BraceStack.push_back(false);
           }
           State = Default;
           break;
         case tok::r_brace:
-      // If braces are unmatched, we're going to be confused, but don't crash.
+          // If braces are unmatched, we're going to be confused, but don't
+          // crash.
           if (!BraceStack.empty()) {
             if (BraceStack.back()) {
               // Parsed: } // namespace
               Enclosing.pop_back();
               Event.Trigger = NamespaceEvent::EndNamespace;
               Event.Payload = llvm::join(Enclosing, "::");
               Callback(Event);
             }
             BraceStack.pop_back();
           }
           break;
         case tok::semi:
           if (State == UsingNamespaceName) {
             // Parsed: using namespace <name> ;
             Event.Trigger = NamespaceEvent::UsingDirective;
             Event.Payload = std::move(NSName);
             Callback(Event);
           }
           State = Default;
           break;
         default:
           State = Default;
           break;
         }
       });
 }
 
 // Returns the prefix namespaces of NS: {"" ... NS}.
 llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) {
   llvm::SmallVector<llvm::StringRef, 8> Results;
   Results.push_back(NS.take_front(0));
   NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
   for (llvm::StringRef &R : Results)