[clang] Expose helper function to turn PP keywords spelling into PPKeywordKind

[riccibruno]

clang/include/clang/Basic/TokenKinds.h

99

A string is expensive here and unneeded. Why not a StringRef?

clang/lib/Basic/IdentifierTable.cpp

274

Better to use IdentifierInfo::getName() to avoid recomputing the length.

[kbobyrev]

clang/include/clang/Basic/TokenKinds.h

99

Yeah, this piece is unfortunate :( llvm::StringRef doesn't really work with the current code. It relies on checking if using \0 at the end of given string which is not always the case for StringRef (and StringRef itself would not be a suitable data structure to access Name[2] given Name.size() == 2 because of the bounds check.

This is a piece I didn't figure out and hence did not send the patch to the review yet.

[kbobyrev]

Update the patch. Still WIP, stash progress.

[kbobyrev]

Not needed yet, waiting for myself to start adding some PP-based FoldingRanges to continue.

clang-tools-extra/clangd/SemanticSelection.cpp

 //===--- SemanticSelection.cpp -----------------------------------*- C++-*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 #include "SemanticSelection.h"
 #include "FindSymbols.h"
 #include "ParsedAST.h"
 #include "Protocol.h"
 #include "Selection.h"
 #include "SourceCode.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Error.h"
+#include <stack>
 
 namespace clang {
 namespace clangd {
 namespace {
 
 // Adds Range \p R to the Result if it is distinct from the last added Range.
 // Assumes that only consecutive ranges can coincide.
 void addIfDistinct(const Range &R, std::vector<Range> &Result) {
void collectFoldingRanges(DocumentSymbol Symbol,
   Range.startCharacter = Symbol.range.start.character;
   Range.endLine = Symbol.range.end.line;
   Range.endCharacter = Symbol.range.end.character;
   Result.push_back(Range);
   for (const auto &Child : Symbol.children)
     collectFoldingRanges(Child, Result);
 }
 
+void collectPreprocessorFoldingRanges(SourceManager &SM,
+                                      std::vector<FoldingRange> &Result) {
+  const syntax::TokenBuffer Buffer(SM);
+  const auto Tokens = Buffer.spelledTokens(SM.getMainFileID());
+  std::stack<clang::SourceLocation> FoldingRangeBeginnings;
+  for (size_t I = 1; I < Tokens.size(); ++I) {
+    const auto TokenText = Tokens[I].text(SM).str();
+    if (tok::getPPKeywordFromSpelling(TokenText) == tok::PPKeywordKind::pp_if &&
+        +Tokens[I - 1].kind() == tok::TokenKind::hash) {
+      FoldingRangeBeginnings.push(Tokens[I - 1].location());
+    }
+    if (tok::getPPKeywordFromSpelling(TokenText) ==
+            tok::PPKeywordKind::pp_endif &&
+        +Tokens[I - 1].kind() == tok::TokenKind::hash) {
+      auto StartLoc = FoldingRangeBeginnings.top();
+      FoldingRangeBeginnings.pop();
+      auto EndLoc = Tokens[I].endLocation();
+      FoldingRange NextRange;
+      NextRange.startLine = SM.getSpellingLineNumber(StartLoc);
+      NextRange.startCharacter = SM.getSpellingColumnNumber(StartLoc);
+      NextRange.endLine = SM.getSpellingLineNumber(EndLoc);
+      NextRange.endCharacter = SM.getSpellingColumnNumber(EndLoc);
+      Result.push_back(NextRange);
+    }
+  }
+}
+
 } // namespace
 
 llvm::Expected<SelectionRange> getSemanticRanges(ParsedAST &AST, Position Pos) {
   std::vector<Range> Ranges;
   const auto &SM = AST.getSourceManager();
   const auto &LangOpts = AST.getLangOpts();
 
   auto FID = SM.getMainFileID();
llvm::Expected<std::vector<FoldingRange>> getFoldingRanges(ParsedAST &AST) {
   // nodes themselves instead of their contents which is less useful). Replace
   // this with a more general RecursiveASTVisitor implementation instead.
   auto DocumentSymbols = getDocumentSymbols(AST);
   if (!DocumentSymbols)
     return DocumentSymbols.takeError();
   std::vector<FoldingRange> Result;
   for (const auto &Symbol : *DocumentSymbols)
     collectFoldingRanges(Symbol, Result);
+  collectPreprocessorFoldingRanges(AST.getSourceManager(), Result);
   return Result;
 }
 
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp

for (const char *Test : Tests) {
     auto T = Annotations(Test);
     auto AST = TestTU::withCode(T.code()).build();
     EXPECT_THAT(gatherFoldingRanges(llvm::cantFail(getFoldingRanges(AST))),
                 UnorderedElementsAreArray(T.ranges()))
         << Test;
   }
 }
 
+TEST(FoldingRanges, Preprocessor) {
+  const char *Tests[] = {
+      R"cpp(
+        [[#if true
+        void foo();
+        #endif]]
+      )cpp",
+  };
+  for (const char *Test : Tests) {
+    auto T = Annotations(Test);
+    auto AST = TestTU::withCode(T.code()).build();
+    EXPECT_THAT(gatherFoldingRanges(llvm::cantFail(getFoldingRanges(AST))),
+                UnorderedElementsAreArray(T.ranges()))
+        << Test;
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang

clang/include/clang/Basic/TokenKinds.h

 }
 
 /// Return true if this is any of tok::annot_* kinds.
 bool isAnnotation(TokenKind K);
 
 /// Return true if this is an annotation token representing a pragma.
 bool isPragmaAnnotation(TokenKind K);
 
+// NOTE: \p Spelling has to be null-terminated C string.
+PPKeywordKind getPPKeywordFromSpelling(llvm::StringRef Spelling);
+
 } // end namespace tok
 } // end namespace clang
 
 namespace llvm {
 template <> struct DenseMapInfo<clang::tok::PPKeywordKind> {
   static inline clang::tok::PPKeywordKind getEmptyKey() {
     return clang::tok::PPKeywordKind::pp_not_keyword;
   }

clang/lib/Basic/IdentifierTable.cpp

bool IdentifierInfo::isCPlusPlusKeyword(const LangOptions &LangOpts) const {
   LangOptions LangOptsNoCPP = LangOpts;
   LangOptsNoCPP.CPlusPlus = false;
   LangOptsNoCPP.CPlusPlus11 = false;
   LangOptsNoCPP.CPlusPlus20 = false;
   return !isKeyword(LangOptsNoCPP);
 }
 
 tok::PPKeywordKind IdentifierInfo::getPPKeywordID() const {
-  // We use a perfect hash function here involving the length of the keyword,
-  // the first and third character.  For preprocessor ID's there are no
-  // collisions (if there were, the switch below would complain about duplicate
-  // case values).  Note that this depends on 'if' being null terminated.
-
-#define HASH(LEN, FIRST, THIRD) \
-  (LEN << 5) + (((FIRST-'a') + (THIRD-'a')) & 31)
-#define CASE(LEN, FIRST, THIRD, NAME) \
-  case HASH(LEN, FIRST, THIRD): \
-    return memcmp(Name, #NAME, LEN) ? tok::pp_not_keyword : tok::pp_ ## NAME
-
-  unsigned Len = getLength();
-  if (Len < 2) return tok::pp_not_keyword;
-  const char *Name = getNameStart();
-  switch (HASH(Len, Name[0], Name[2])) {
-  default: return tok::pp_not_keyword;
-  CASE( 2, 'i', '\0', if);
-  CASE( 4, 'e', 'i', elif);
-  CASE( 4, 'e', 's', else);
-  CASE( 4, 'l', 'n', line);
-  CASE( 4, 's', 'c', sccs);
-  CASE( 5, 'e', 'd', endif);
-  CASE( 5, 'e', 'r', error);
-  CASE( 5, 'i', 'e', ident);
-  CASE( 5, 'i', 'd', ifdef);
-  CASE( 5, 'u', 'd', undef);
-
-  CASE( 6, 'a', 's', assert);
-  CASE( 6, 'd', 'f', define);
-  CASE( 6, 'i', 'n', ifndef);
-  CASE( 6, 'i', 'p', import);
-  CASE( 6, 'p', 'a', pragma);
-
-  CASE( 7, 'd', 'f', defined);
-  CASE( 7, 'i', 'c', include);
-  CASE( 7, 'w', 'r', warning);
-
-  CASE( 8, 'u', 'a', unassert);
-  CASE(12, 'i', 'c', include_next);
-
-  CASE(14, '_', 'p', __public_macro);
-
-  CASE(15, '_', 'p', __private_macro);
-
-  CASE(16, '_', 'i', __include_macros);
-#undef CASE
-#undef HASH
-  }
+  return tok::getPPKeywordFromSpelling(getNameStart());
 }
 
 //===----------------------------------------------------------------------===//
 // Stats Implementation
 //===----------------------------------------------------------------------===//
 
 /// PrintStats - Print statistics about how well the identifier table is doing
 /// at hashing identifiers.

clang/lib/Basic/TokenKinds.cpp

bool tok::isPragmaAnnotation(TokenKind Kind) {
   switch (Kind) {
 #define PRAGMA_ANNOTATION(X) case annot_ ## X: return true;
 #include "clang/Basic/TokenKinds.def"
   default:
     break;
   }
   return false;
 }
+
+tok::PPKeywordKind tok::getPPKeywordFromSpelling(llvm::StringRef Spelling) {
+  // We use a perfect hash function here involving the length of the keyword,
+  // the first and third character.  For preprocessor ID's there are no
+  // collisions (if there were, the switch below would complain about duplicate
+  // case values).  Note that this depends on 'if' being null terminated.
+  unsigned Len = Spelling.size();
+  assert((*(Spelling.begin() + Len) == '\0') &&
+         "Spelling has to be null-terminated string.");
+
+#define HASH(LEN, FIRST, THIRD)                                                \
+  (LEN << 5) + (((FIRST - 'a') + (THIRD - 'a')) & 31)
+#define CASE(LEN, FIRST, THIRD, NAME)                                          \
+  case HASH(LEN, FIRST, THIRD):                                                \
+    return memcmp(Spelling.begin(), #NAME, LEN) ? tok::pp_not_keyword          \
+                                                : tok::pp_##NAME
+
+  if (Len < 2)
+    return tok::pp_not_keyword;
+  switch (HASH(Len, Spelling[0], *(Spelling.begin() + 2))) {
+  default:
+    return tok::pp_not_keyword;
+    CASE(2, 'i', '\0', if);
+    CASE(4, 'e', 'i', elif);
+    CASE(4, 'e', 's', else);
+    CASE(4, 'l', 'n', line);
+    CASE(4, 's', 'c', sccs);
+    CASE(5, 'e', 'd', endif);
+    CASE(5, 'e', 'r', error);
+    CASE(5, 'i', 'e', ident);
+    CASE(5, 'i', 'd', ifdef);
+    CASE(5, 'u', 'd', undef);
+
+    CASE(6, 'a', 's', assert);
+    CASE(6, 'd', 'f', define);
+    CASE(6, 'i', 'n', ifndef);
+    CASE(6, 'i', 'p', import);
+    CASE(6, 'p', 'a', pragma);
+
+    CASE(7, 'd', 'f', defined);
+    CASE(7, 'i', 'c', include);
+    CASE(7, 'w', 'r', warning);
+
+    CASE(8, 'u', 'a', unassert);
+    CASE(12, 'i', 'c', include_next);
+
+    CASE(14, '_', 'p', __public_macro);
+
+    CASE(15, '_', 'p', __private_macro);
+
+    CASE(16, '_', 'i', __include_macros);
+#undef CASE
+#undef HASH
+  }
+}