Index: include/clang/Basic/IdentifierTable.h =================================================================== --- include/clang/Basic/IdentifierTable.h +++ include/clang/Basic/IdentifierTable.h @@ -477,9 +477,11 @@ public: /// \brief Create the identifier table, populating it with info about the - /// language keywords for the language specified by \p LangOpts. + /// language keywords for the language specified by \p LangOpts if + /// \p DeferKeywordAddition is not set. IdentifierTable(const LangOptions &LangOpts, - IdentifierInfoLookup* externalLookup = nullptr); + IdentifierInfoLookup *externalLookup = nullptr, + bool DeferKeywordAddition = false); /// \brief Set the external identifier lookup mechanism. void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) { Index: include/clang/Lex/Preprocessor.h =================================================================== --- include/clang/Lex/Preprocessor.h +++ include/clang/Lex/Preprocessor.h @@ -691,7 +691,8 @@ HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup = nullptr, bool OwnsHeaderSearch = false, - TranslationUnitKind TUKind = TU_Complete); + TranslationUnitKind TUKind = TU_Complete, + bool DeferKeywordAddition = false); ~Preprocessor(); Index: lib/Basic/IdentifierTable.cpp =================================================================== --- lib/Basic/IdentifierTable.cpp +++ lib/Basic/IdentifierTable.cpp @@ -73,17 +73,15 @@ } IdentifierTable::IdentifierTable(const LangOptions &LangOpts, - IdentifierInfoLookup* externalLookup) - : HashTable(8192), // Start with space for 8K identifiers. - ExternalLookup(externalLookup) { + IdentifierInfoLookup *externalLookup, + bool DeferKeywordAddition) + : HashTable(8192), // Start with space for 8K identifiers. + ExternalLookup(externalLookup) { // Populate the identifier table with info about keywords for the current // language. - AddKeywords(LangOpts); - - - // Add the '_experimental_modules_import' contextual keyword. - get("import").setModulesImport(true); + if (!DeferKeywordAddition) + AddKeywords(LangOpts); } //===----------------------------------------------------------------------===// @@ -227,6 +225,9 @@ if (LangOpts.DeclSpecKeyword) AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this); + + // Add the '_experimental_modules_import' contextual keyword. + get("import").setModulesImport(true); } /// \brief Checks if the specified token kind represents a keyword in the Index: lib/Frontend/ASTUnit.cpp =================================================================== --- lib/Frontend/ASTUnit.cpp +++ lib/Frontend/ASTUnit.cpp @@ -536,6 +536,10 @@ // Initialize the preprocessor. PP.Initialize(*Target); + // Populate the identifier table with info about keywords for the current + // language. + PP.getIdentifierTable().AddKeywords(LangOpt); + if (!Context) return; @@ -718,11 +722,13 @@ HeaderSearch &HeaderInfo = *AST->HeaderInfo; unsigned Counter; + // As the language options have not been loaded yet, adding keywords to the + // identifier table is deferred and will be initiated by ASTInfoCollector. AST->PP = std::make_shared( AST->PPOpts, AST->getDiagnostics(), *AST->LangOpts, AST->getSourceManager(), *AST->PCMCache, HeaderInfo, AST->ModuleLoader, - /*IILookup=*/nullptr, - /*OwnsHeaderSearch=*/false); + /*IILookup=*/nullptr, /*OwnsHeaderSearch=*/false, TU_Complete, + /*DeferKeywordAddition=*/true); Preprocessor &PP = *AST->PP; if (ToLoad >= LoadASTOnly) Index: lib/Lex/Preprocessor.cpp =================================================================== --- lib/Lex/Preprocessor.cpp +++ lib/Lex/Preprocessor.cpp @@ -73,12 +73,14 @@ SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup, bool OwnsHeaders, - TranslationUnitKind TUKind) + TranslationUnitKind TUKind, + bool DeferKeywordAddition) : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr), AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), - ExternalSource(nullptr), Identifiers(opts, IILookup), + ExternalSource(nullptr), + Identifiers(opts, IILookup, DeferKeywordAddition), PragmaHandlers(new PragmaNamespace(StringRef())), IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr), CodeCompletionFile(nullptr), CodeCompletionOffset(0), Index: unittests/libclang/LibclangTest.cpp =================================================================== --- unittests/libclang/LibclangTest.cpp +++ unittests/libclang/LibclangTest.cpp @@ -572,3 +572,67 @@ EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); DisplayDiagnostics(); } + +class LibclangSerializationTest : public LibclangParseTest { +public: + bool SaveAndLoadTU(const std::string &Filename) { + unsigned options = clang_defaultSaveOptions(ClangTU); + if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) != + CXSaveError_None) { + DEBUG(llvm::dbgs() << "Saving failed\n"); + return false; + } + + clang_disposeTranslationUnit(ClangTU); + + ClangTU = clang_createTranslationUnit(Index, Filename.c_str()); + + if (!ClangTU) { + DEBUG(llvm::dbgs() << "Loading failed\n"); + return false; + } + + return true; + } +}; + +TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) { + // Ensure that "class" is recognized as a keyword token after serializing + // and reloading the AST, as it is not a keyword for the default LangOptions. + std::string HeaderName = "test.h"; + WriteFile(HeaderName, "enum class Something {};"); + + const char *Argv[] = {"-xc++-header", "-std=c++11"}; + + ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv, + sizeof(Argv) / sizeof(Argv[0]), nullptr, + 0, TUFlags); + + auto CheckTokenKinds = [=]() { + CXSourceRange Range = + clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU)); + + CXToken *Tokens; + unsigned int NumTokens; + clang_tokenize(ClangTU, Range, &Tokens, &NumTokens); + + ASSERT_EQ(6u, NumTokens); + EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0])); + EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1])); + EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4])); + EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5])); + + clang_disposeTokens(ClangTU, Tokens, NumTokens); + }; + + CheckTokenKinds(); + + std::string ASTName = "test.ast"; + WriteFile(ASTName, ""); + + ASSERT_TRUE(SaveAndLoadTU(ASTName)); + + CheckTokenKinds(); +}