Index: include/clang/Basic/IdentifierTable.h =================================================================== --- include/clang/Basic/IdentifierTable.h +++ include/clang/Basic/IdentifierTable.h @@ -467,10 +467,13 @@ IdentifierInfoLookup* ExternalLookup; public: + /// \brief Create the identifier table. + explicit IdentifierTable(IdentifierInfoLookup *ExternalLookup = nullptr); + /// \brief Create the identifier table, populating it with info about the /// language keywords for the language specified by \p LangOpts. - IdentifierTable(const LangOptions &LangOpts, - IdentifierInfoLookup* externalLookup = nullptr); + explicit IdentifierTable(const LangOptions &LangOpts, + IdentifierInfoLookup *ExternalLookup = nullptr); /// \brief Set the external identifier lookup mechanism. void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) { @@ -558,6 +561,8 @@ /// hashing is doing. void PrintStats() const; + /// \brief Populate the identifier table with info about the language keywords + /// for the language specified by \p LangOpts. void AddKeywords(const LangOptions &LangOpts); }; Index: lib/Basic/IdentifierTable.cpp =================================================================== --- lib/Basic/IdentifierTable.cpp +++ lib/Basic/IdentifierTable.cpp @@ -79,16 +79,16 @@ return new EmptyLookupIterator(); } +IdentifierTable::IdentifierTable(IdentifierInfoLookup *ExternalLookup) + : HashTable(8192), // Start with space for 8K identifiers. + ExternalLookup(ExternalLookup) {} + IdentifierTable::IdentifierTable(const LangOptions &LangOpts, - IdentifierInfoLookup* externalLookup) - : HashTable(8192), // Start with space for 8K identifiers. - ExternalLookup(externalLookup) { + IdentifierInfoLookup *ExternalLookup) + : IdentifierTable(ExternalLookup) { // Populate the identifier table with info about keywords for the current // language. AddKeywords(LangOpts); - - // Add the '_experimental_modules_import' contextual keyword. - get("import").setModulesImport(true); } //===----------------------------------------------------------------------===// @@ -237,6 +237,9 @@ if (LangOpts.DeclSpecKeyword) AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this); + + // Add the '_experimental_modules_import' contextual keyword. + get("import").setModulesImport(true); } /// \brief Checks if the specified token kind represents a keyword in the Index: lib/Lex/Preprocessor.cpp =================================================================== --- lib/Lex/Preprocessor.cpp +++ lib/Lex/Preprocessor.cpp @@ -85,12 +85,14 @@ IdentifierInfoLookup *IILookup, bool OwnsHeaders, TranslationUnitKind TUKind) : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), - FileMgr(Headers.getFileMgr()), SourceMgr(SM), - PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), - HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), - ExternalSource(nullptr), Identifiers(opts, IILookup), - PragmaHandlers(new PragmaNamespace(StringRef())), TUKind(TUKind), - SkipMainFilePreamble(0, true), + FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache), + ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), + TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), + // As the language options may have not been loaded yet (when + // deserializing an ASTUnit), adding keywords to the identifier table is + // deferred to Preprocessor::Initialize(). + Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), + TUKind(TUKind), SkipMainFilePreamble(0, true), CurSubmoduleState(&NullSubmoduleState) { OwnsHeaderSearch = OwnsHeaders; @@ -190,6 +192,9 @@ // Initialize information about built-ins. BuiltinInfo.InitializeTarget(Target, AuxTarget); HeaderInfo.setTarget(Target); + + // Populate the identifier table with info about keywords for the current language. + Identifiers.AddKeywords(LangOpts); } void Preprocessor::InitializeForModelFile() { Index: unittests/libclang/LibclangTest.cpp =================================================================== --- unittests/libclang/LibclangTest.cpp +++ unittests/libclang/LibclangTest.cpp @@ -698,3 +698,67 @@ clang_disposeSourceRangeList(Ranges); } } + +class LibclangSerializationTest : public LibclangParseTest { +public: + bool SaveAndLoadTU(const std::string &Filename) { + unsigned options = clang_defaultSaveOptions(ClangTU); + if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) != + CXSaveError_None) { + DEBUG(llvm::dbgs() << "Saving failed\n"); + return false; + } + + clang_disposeTranslationUnit(ClangTU); + + ClangTU = clang_createTranslationUnit(Index, Filename.c_str()); + + if (!ClangTU) { + DEBUG(llvm::dbgs() << "Loading failed\n"); + return false; + } + + return true; + } +}; + +TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) { + // Ensure that "class" is recognized as a keyword token after serializing + // and reloading the AST, as it is not a keyword for the default LangOptions. + std::string HeaderName = "test.h"; + WriteFile(HeaderName, "enum class Something {};"); + + const char *Argv[] = {"-xc++-header", "-std=c++11"}; + + ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv, + sizeof(Argv) / sizeof(Argv[0]), nullptr, + 0, TUFlags); + + auto CheckTokenKinds = [=]() { + CXSourceRange Range = + clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU)); + + CXToken *Tokens; + unsigned int NumTokens; + clang_tokenize(ClangTU, Range, &Tokens, &NumTokens); + + ASSERT_EQ(6u, NumTokens); + EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0])); + EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1])); + EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5])); + + clang_disposeTokens(ClangTU, Tokens, NumTokens); + }; + + CheckTokenKinds(); + + std::string ASTName = "test.ast"; + WriteFile(ASTName, ""); + + ASSERT_TRUE(SaveAndLoadTU(ASTName)); + + CheckTokenKinds(); +}