[Concepts] lex keywords: concept and requires

[hubert.reinterpretcast]

Summary

This patch enables lexing of concept and requires as keywords.
Further changes which add messages for future keyword compat are to
follow.

[• fraggamuffin]

This looks fine to me. Just one question: since we are setting -std=c++14,
is there no define for -DCXX14 instead of having to set -DCXX11? If this is
clang policy, then LGTM, otherwise, why are we not adding a define for
C++14?

[hubert.reinterpretcast]

In D10233#183453, @fraggamuffin wrote:

This looks fine to me. Just one question: since we are setting -std=c++14,
is there no define for -DCXX14 instead of having to set -DCXX11? If this is
clang policy, then LGTM, otherwise, why are we not adding a define for
C++14?

In the context of this test file, CXX14 would be redundant because there are no new keywords introduced through C++14. The corresponding implementation of keywords in the compiler does not call out C++14 apart from C++11. This change is consistent with that.

[rsmith]

LGTM

include/clang/Basic/TokenKinds.def

 //===--- TokenKinds.def - C Family Token Kind Database ----------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the TokenKind database.  This includes normal tokens like
 // tok::ampamp (corresponding to the && token) as well as keywords for various
 // languages.  Users of this file must optionally #define the TOK, KEYWORD,
-// CXX11_KEYWORD, ALIAS, or PPKEYWORD macros to make use of this file.
+// CXX11_KEYWORD, CONCEPTS_KEYWORD, ALIAS, or PPKEYWORD macros to make use of
+// this file.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef TOK
 #define TOK(X)
 #endif
 #ifndef PUNCTUATOR
 #define PUNCTUATOR(X,Y) TOK(X)
 #endif
 #ifndef KEYWORD
 #define KEYWORD(X,Y) TOK(kw_ ## X)
 #endif
 #ifndef CXX11_KEYWORD
 #define CXX11_KEYWORD(X,Y) KEYWORD(X,KEYCXX11|(Y))
 #endif
+#ifndef CONCEPTS_KEYWORD
+#define CONCEPTS_KEYWORD(X) KEYWORD(X,KEYCONCEPTS)
+#endif
 #ifndef TYPE_TRAIT
 #define TYPE_TRAIT(N,I,K) KEYWORD(I,K)
 #endif
 #ifndef TYPE_TRAIT_1
 #define TYPE_TRAIT_1(I,E,K) TYPE_TRAIT(1,I,K)
 #endif
 #ifndef TYPE_TRAIT_2
 #define TYPE_TRAIT_2(I,E,K) TYPE_TRAIT(2,I,K)

 //              is a keyword in the implementation namespace that should
 //              always be treated as a keyword
 //   KEYC99   - This is a keyword introduced to C in C99
 //   KEYC11   - This is a keyword introduced to C in C11
 //   KEYCXX   - This is a C++ keyword, or a C++-specific keyword in the
 //              implementation namespace
 //   KEYNOCXX - This is a keyword in every non-C++ dialect.
 //   KEYCXX11 - This is a C++ keyword introduced to C++ in C++11
+//   KEYCONCEPTS - This is a keyword if the C++ extensions for concepts
+//                 are enabled.
 //   KEYGNU   - This is a keyword if GNU extensions are enabled
 //   KEYMS    - This is a keyword if Microsoft extensions are enabled
 //   KEYNOMS18 - This is a keyword that must never be enabled under
 //               MSVC <= v18.
 //   KEYOPENCL  - This is a keyword in OpenCL
 //   KEYNOOPENCL  - This is a keyword that is not supported in OpenCL
 //   KEYALTIVEC - This is a keyword in AltiVec
 //   KEYBORLAND - This is a keyword if Borland extensions are enabled

 CXX11_KEYWORD(char32_t              , KEYNOMS18)
 CXX11_KEYWORD(constexpr             , 0)
 CXX11_KEYWORD(decltype              , 0)
 CXX11_KEYWORD(noexcept              , 0)
 CXX11_KEYWORD(nullptr               , 0)
 CXX11_KEYWORD(static_assert         , 0)
 CXX11_KEYWORD(thread_local          , 0)
 
+// C++ concepts TS keywords
+CONCEPTS_KEYWORD(concept)
+CONCEPTS_KEYWORD(requires)
+
 // GNU Extensions (in impl-reserved namespace)
 KEYWORD(_Decimal32                  , KEYALL)
 KEYWORD(_Decimal64                  , KEYALL)
 KEYWORD(_Decimal128                 , KEYALL)
 KEYWORD(__null                      , KEYCXX)
 KEYWORD(__alignof                   , KEYALL)
 KEYWORD(__attribute                 , KEYALL)
 KEYWORD(__builtin_choose_expr       , KEYALL)

 #undef OBJC1_AT_KEYWORD
 #undef CXX_KEYWORD_OPERATOR
 #undef PPKEYWORD
 #undef ALIAS
 #undef TYPE_TRAIT_N
 #undef TYPE_TRAIT_2
 #undef TYPE_TRAIT_1
 #undef TYPE_TRAIT
+#undef CONCEPTS_KEYWORD
 #undef CXX11_KEYWORD
 #undef KEYWORD
 #undef PUNCTUATOR
 #undef TOK

lib/Basic/IdentifierTable.cpp

enum {
     KEYBORLAND = 0x100,
     KEYOPENCL = 0x200,
     KEYC11 = 0x400,
     KEYARC = 0x800,
     KEYNOMS18 = 0x01000,
     KEYNOOPENCL = 0x02000,
     WCHARSUPPORT = 0x04000,
     HALFSUPPORT = 0x08000,
-    KEYALL = (0xffff & ~KEYNOMS18 &
+    KEYCONCEPTS = 0x10000,
+    KEYALL = (0x1ffff & ~KEYNOMS18 &
               ~KEYNOOPENCL) // KEYNOMS18 and KEYNOOPENCL are used to exclude.
   };
 
   /// \brief How a keyword is treated in the selected standard.
   enum KeywordStatus {
     KS_Disabled,    // Disabled
     KS_Extension,   // Is an extension
     KS_Enabled,     // Enabled
static KeywordStatus getKeywordStatus(const LangOptions &LangOpts,
   if (LangOpts.WChar && (Flags & WCHARSUPPORT)) return KS_Enabled;
   if (LangOpts.AltiVec && (Flags & KEYALTIVEC)) return KS_Enabled;
   if (LangOpts.OpenCL && (Flags & KEYOPENCL)) return KS_Enabled;
   if (!LangOpts.CPlusPlus && (Flags & KEYNOCXX)) return KS_Enabled;
   if (LangOpts.C11 && (Flags & KEYC11)) return KS_Enabled;
   // We treat bridge casts as objective-C keywords so we can warn on them
   // in non-arc mode.
   if (LangOpts.ObjC2 && (Flags & KEYARC)) return KS_Enabled;
+  if (LangOpts.ConceptsTS && (Flags & KEYCONCEPTS)) return KS_Enabled;
   if (LangOpts.CPlusPlus && (Flags & KEYCXX11)) return KS_Future;
   return KS_Disabled;
 }
 
 /// AddKeyword - This method is used to associate a token ID with specific
 /// identifiers because they are language keywords.  This causes the lexer to
 /// automatically map matching identifiers to specialized token codes.
 static void AddKeyword(StringRef Keyword,

test/Lexer/keywords_test.cpp

 // RUN: %clang_cc1 -std=c++03 -fsyntax-only %s
 // RUN: %clang_cc1 -std=c++11 -DCXX11 -fsyntax-only %s
+// RUN: %clang_cc1 -std=c++14 -fconcepts-ts -DCXX11 -DCONCEPTS -fsyntax-only %s
 
 #define IS_KEYWORD(NAME) _Static_assert(!__is_identifier(NAME), #NAME)
 #define NOT_KEYWORD(NAME) _Static_assert(__is_identifier(NAME), #NAME)
 #define IS_TYPE(NAME) void is_##NAME##_type() { int f(NAME); }
 
+#ifdef CONCEPTS
+#define CONCEPTS_KEYWORD(NAME)  IS_KEYWORD(NAME)
+#else
+#define CONCEPTS_KEYWORD(NAME)  NOT_KEYWORD(NAME)
+#endif
+
 #ifdef CXX11
 #define CXX11_KEYWORD(NAME)  IS_KEYWORD(NAME)
 #define CXX11_TYPE(NAME)     IS_TYPE(NAME)
 #else
 #define CXX11_KEYWORD(NAME)  NOT_KEYWORD(NAME)
 #define CXX11_TYPE(NAME)
 #endif
 
 // C++11 keywords
 CXX11_KEYWORD(nullptr);
 CXX11_KEYWORD(decltype);
 CXX11_KEYWORD(alignof);
 CXX11_KEYWORD(alignas);
 CXX11_KEYWORD(char16_t);
 CXX11_TYPE(char16_t);
 CXX11_KEYWORD(char32_t);
 CXX11_TYPE(char32_t);
 CXX11_KEYWORD(constexpr);
 CXX11_KEYWORD(noexcept);
 CXX11_KEYWORD(static_assert);
 CXX11_KEYWORD(thread_local);
 
+// Concepts TS keywords
+CONCEPTS_KEYWORD(concept);
+CONCEPTS_KEYWORD(requires);
+
 // Clang extension
 IS_KEYWORD(__char16_t);
 IS_TYPE(__char16_t);
 IS_KEYWORD(__char32_t);
 IS_TYPE(__char32_t);