[clang] Expose helper function to turn PP keywords spelling into PPKeywordKind

[riccibruno]

clang/include/clang/Basic/TokenKinds.h

99

A string is expensive here and unneeded. Why not a StringRef?

clang/lib/Basic/IdentifierTable.cpp

274

Better to use IdentifierInfo::getName() to avoid recomputing the length.

[kbobyrev]

clang/include/clang/Basic/TokenKinds.h

99

Yeah, this piece is unfortunate :( llvm::StringRef doesn't really work with the current code. It relies on checking if using \0 at the end of given string which is not always the case for StringRef (and StringRef itself would not be a suitable data structure to access Name[2] given Name.size() == 2 because of the bounds check.

This is a piece I didn't figure out and hence did not send the patch to the review yet.

[kbobyrev]

Update the patch. Still WIP, stash progress.

[kbobyrev]

Not needed yet, waiting for myself to start adding some PP-based FoldingRanges to continue.

clang-tools-extra/clangd/SemanticSelection.cpp

void collectFoldingRanges(DocumentSymbol Symbol,
   Range.startCharacter = Symbol.range.start.character;
   Range.endLine = Symbol.range.end.line;
   Range.endCharacter = Symbol.range.end.character;
   Result.push_back(Range);
   for (const auto &Child : Symbol.children)
     collectFoldingRanges(Child, Result);
 }
 
+void collectPPFoldingRanges(std::vector<FoldingRange> &Result) {}
+
 } // namespace
 
 llvm::Expected<SelectionRange> getSemanticRanges(ParsedAST &AST, Position Pos) {
   std::vector<Range> Ranges;
   const auto &SM = AST.getSourceManager();
   const auto &LangOpts = AST.getLangOpts();
 
   auto FID = SM.getMainFileID();

clang/include/clang/Basic/TokenKinds.h

 }
 
 /// Return true if this is any of tok::annot_* kinds.
 bool isAnnotation(TokenKind K);
 
 /// Return true if this is an annotation token representing a pragma.
 bool isPragmaAnnotation(TokenKind K);
 
+PPKeywordKind getPPKeywordFromSpelling(const std::string &Name);
+
 } // end namespace tok
 } // end namespace clang
 
 namespace llvm {
 template <> struct DenseMapInfo<clang::tok::PPKeywordKind> {
   static inline clang::tok::PPKeywordKind getEmptyKey() {
     return clang::tok::PPKeywordKind::pp_not_keyword;
   }

clang/lib/Basic/IdentifierTable.cpp

bool IdentifierInfo::isCPlusPlusKeyword(const LangOptions &LangOpts) const {
   LangOptions LangOptsNoCPP = LangOpts;
   LangOptsNoCPP.CPlusPlus = false;
   LangOptsNoCPP.CPlusPlus11 = false;
   LangOptsNoCPP.CPlusPlus20 = false;
   return !isKeyword(LangOptsNoCPP);
 }
 
 tok::PPKeywordKind IdentifierInfo::getPPKeywordID() const {
-  // We use a perfect hash function here involving the length of the keyword,
-  // the first and third character.  For preprocessor ID's there are no
-  // collisions (if there were, the switch below would complain about duplicate
-  // case values).  Note that this depends on 'if' being null terminated.
-
-#define HASH(LEN, FIRST, THIRD) \
-  (LEN << 5) + (((FIRST-'a') + (THIRD-'a')) & 31)
-#define CASE(LEN, FIRST, THIRD, NAME) \
-  case HASH(LEN, FIRST, THIRD): \
-    return memcmp(Name, #NAME, LEN) ? tok::pp_not_keyword : tok::pp_ ## NAME
-
-  unsigned Len = getLength();
-  if (Len < 2) return tok::pp_not_keyword;
-  const char *Name = getNameStart();
-  switch (HASH(Len, Name[0], Name[2])) {
-  default: return tok::pp_not_keyword;
-  CASE( 2, 'i', '\0', if);
-  CASE( 4, 'e', 'i', elif);
-  CASE( 4, 'e', 's', else);
-  CASE( 4, 'l', 'n', line);
-  CASE( 4, 's', 'c', sccs);
-  CASE( 5, 'e', 'd', endif);
-  CASE( 5, 'e', 'r', error);
-  CASE( 5, 'i', 'e', ident);
-  CASE( 5, 'i', 'd', ifdef);
-  CASE( 5, 'u', 'd', undef);
-
-  CASE( 6, 'a', 's', assert);
-  CASE( 6, 'd', 'f', define);
-  CASE( 6, 'i', 'n', ifndef);
-  CASE( 6, 'i', 'p', import);
-  CASE( 6, 'p', 'a', pragma);
-
-  CASE( 7, 'd', 'f', defined);
-  CASE( 7, 'i', 'c', include);
-  CASE( 7, 'w', 'r', warning);
-
-  CASE( 8, 'u', 'a', unassert);
-  CASE(12, 'i', 'c', include_next);
-
-  CASE(14, '_', 'p', __public_macro);
-
-  CASE(15, '_', 'p', __private_macro);
-
-  CASE(16, '_', 'i', __include_macros);
-#undef CASE
-#undef HASH
-  }
+  return tok::getPPKeywordFromSpelling(getNameStart());
 }
 
 //===----------------------------------------------------------------------===//
 // Stats Implementation
 //===----------------------------------------------------------------------===//
 
 /// PrintStats - Print statistics about how well the identifier table is doing
 /// at hashing identifiers.

clang/lib/Basic/TokenKinds.cpp

bool tok::isPragmaAnnotation(TokenKind Kind) {
   switch (Kind) {
 #define PRAGMA_ANNOTATION(X) case annot_ ## X: return true;
 #include "clang/Basic/TokenKinds.def"
   default:
     break;
   }
   return false;
 }
+
+tok::PPKeywordKind tok::getPPKeywordFromSpelling(const std::string &Name) {
+  // We use a perfect hash function here involving the length of the keyword,
+  // the first and third character.  For preprocessor ID's there are no
+  // collisions (if there were, the switch below would complain about duplicate
+  // case values).  Note that this depends on 'if' being null terminated.
+  unsigned Len = Name.size();
+
+#define HASH(LEN, FIRST, THIRD)                                                \
+  (LEN << 5) + (((FIRST - 'a') + (THIRD - 'a')) & 31)
+#define CASE(LEN, FIRST, THIRD, NAME)                                          \
+  case HASH(LEN, FIRST, THIRD):                                                \
+    return memcmp(Name.c_str(), #NAME, LEN) ? tok::pp_not_keyword              \
+                                            : tok::pp_##NAME
+
+  if (Len < 2)
+    return tok::pp_not_keyword;
+  switch (HASH(Len, Name[0], Name[2])) {
+  default:
+    return tok::pp_not_keyword;
+    CASE(2, 'i', '\0', if);
+    CASE(4, 'e', 'i', elif);
+    CASE(4, 'e', 's', else);
+    CASE(4, 'l', 'n', line);
+    CASE(4, 's', 'c', sccs);
+    CASE(5, 'e', 'd', endif);
+    CASE(5, 'e', 'r', error);
+    CASE(5, 'i', 'e', ident);
+    CASE(5, 'i', 'd', ifdef);
+    CASE(5, 'u', 'd', undef);
+
+    CASE(6, 'a', 's', assert);
+    CASE(6, 'd', 'f', define);
+    CASE(6, 'i', 'n', ifndef);
+    CASE(6, 'i', 'p', import);
+    CASE(6, 'p', 'a', pragma);
+
+    CASE(7, 'd', 'f', defined);
+    CASE(7, 'i', 'c', include);
+    CASE(7, 'w', 'r', warning);
+
+    CASE(8, 'u', 'a', unassert);
+    CASE(12, 'i', 'c', include_next);
+
+    CASE(14, '_', 'p', __public_macro);
+
+    CASE(15, '_', 'p', __private_macro);
+
+    CASE(16, '_', 'i', __include_macros);
+#undef CASE
+#undef HASH
+  }
+}