[clangd] Treat UserDefinedLiteral as a leaf in SelectionTree, sidestepping tokenization issues

[sammccall]

Summary

Fixes https://github.com/clangd/clangd/issues/203

[merge_guards_bot]

Build result: pass - 60179 tests passed, 0 failed and 732 were skipped.
Log files: console-log.txt, CMakeCache.txt

[kadircet]

LGTM

clang-tools-extra/clangd/Selection.cpp

 //===--- Selection.cpp ----------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Selection.h"
 #include "Logger.h"
 #include "SourceCode.h"
 #include "clang/AST/ASTTypeTraits.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/Lexer.h"
public:
   // Stmt is the same, but this form allows the data recursion optimization.
   bool dataTraverseStmtPre(Stmt *X) {
     if (!X || isImplicit(X))
       return false;
     auto N = DynTypedNode::create(*X);
     if (canSafelySkipNode(N))
       return false;
     push(std::move(N));
+    if (shouldSkipChildren(X)) {
+      pop();
+      return false;
+    }
     return true;
   }
   bool dataTraverseStmtPost(Stmt *X) {
     pop();
     return true;
   }
   // QualifiedTypeLoc is handled strangely in RecursiveASTVisitor: the derived
   // TraverseTypeLoc is not called for the inner UnqualTypeLoc.
bool canSafelySkipNode(const DynTypedNode &N) {
     if (B.second < SelEnd && E.second >= SelBeginTokenStart)
       return false;
     // Otherwise, allow skipping over the node.
     dlog("{1}skip: {0}", printNodeToString(N, PrintPolicy), indent());
     dlog("{1}skipped range = {0}", S.printToString(SM), indent(1));
     return true;
   }
 
+  // There are certain nodes we want to treat as leaves in the SelectionTree,
+  // although they do have children.
+  bool shouldSkipChildren(const Stmt *X) const {
+    // UserDefinedLiteral (e.g. 12_i) has two children (12 and _i).
+    // Unfortunately TokenBuffer sees 12_i as one token and can't split it.
+    // So we treat UserDefinedLiteral as a leaf node, owning the token.
+    return llvm::isa<UserDefinedLiteral>(X);
+  }
+
   // Pushes a node onto the ancestor stack. Pairs with pop().
   // Performs early hit detection for some nodes (on the earlySourceRange).
   void push(DynTypedNode Node) {
     SourceRange Early = earlySourceRange(Node);
     dlog("{1}push: {0}", printNodeToString(Node, PrintPolicy), indent());
     Nodes.emplace_back();
     Nodes.back().ASTNode = std::move(Node);
     Nodes.back().Parent = Stack.top();

clang-tools-extra/clangd/unittests/SelectionTests.cpp

Case Cases[] = {
             };
             Str makeStr(const char*);
             void loop() {
               for (const char* C : [[mak^eStr("foo"^)]])
                 ;
             }
           )cpp",
           "CallExpr"},
+
+      // User-defined literals are tricky: is 12_i one token or two?
+      // For now we treat it as one, and the UserDefinedLiteral as a leaf.
+      {
+          R"cpp(
+            struct Foo{};
+            Foo operator""_ud(unsigned long long);
+            Foo x = [[^12_ud]];
+          )cpp",
+          "UserDefinedLiteral"},
   };
   for (const Case &C : Cases) {
     Annotations Test(C.Code);
 
     TestTU TU;
     TU.Code = Test.code();
 
     // FIXME: Auto-completion in a template requires disabling delayed template