jklaehn updated this revision to Diff 141033.
jklaehn set the repository for this revision to rC Clang.
jklaehn added a comment.
ping? (rebased)
Repository:
rC Clang
https://reviews.llvm.org/D35181
Files:
include/clang/Basic/IdentifierTable.h
lib/Basic/IdentifierTable.cpp
lib/Lex/Preprocessor.cpp
unittests/libclang/LibclangTest.cpp
Index: unittests/libclang/LibclangTest.cpp
===================================================================
--- unittests/libclang/LibclangTest.cpp
+++ unittests/libclang/LibclangTest.cpp
@@ -683,3 +683,67 @@
clang_disposeSourceRangeList(Ranges);
}
}
+
+class LibclangSerializationTest : public LibclangParseTest {
+public:
+ bool SaveAndLoadTU(const std::string &Filename) {
+ unsigned options = clang_defaultSaveOptions(ClangTU);
+ if (clang_saveTranslationUnit(ClangTU, Filename.c_str(), options) !=
+ CXSaveError_None) {
+ DEBUG(llvm::dbgs() << "Saving failed\n");
+ return false;
+ }
+
+ clang_disposeTranslationUnit(ClangTU);
+
+ ClangTU = clang_createTranslationUnit(Index, Filename.c_str());
+
+ if (!ClangTU) {
+ DEBUG(llvm::dbgs() << "Loading failed\n");
+ return false;
+ }
+
+ return true;
+ }
+};
+
+TEST_F(LibclangSerializationTest, TokenKindsAreCorrectAfterLoading) {
+ // Ensure that "class" is recognized as a keyword token after serializing
+ // and reloading the AST, as it is not a keyword for the default LangOptions.
+ std::string HeaderName = "test.h";
+ WriteFile(HeaderName, "enum class Something {};");
+
+ const char *Argv[] = {"-xc++-header", "-std=c++11"};
+
+ ClangTU = clang_parseTranslationUnit(Index, HeaderName.c_str(), Argv,
+ sizeof(Argv) / sizeof(Argv[0]), nullptr,
+ 0, TUFlags);
+
+ auto CheckTokenKinds = [=]() {
+ CXSourceRange Range =
+ clang_getCursorExtent(clang_getTranslationUnitCursor(ClangTU));
+
+ CXToken *Tokens;
+ unsigned int NumTokens;
+ clang_tokenize(ClangTU, Range, &Tokens, &NumTokens);
+
+ ASSERT_EQ(6u, NumTokens);
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[0]));
+ EXPECT_EQ(CXToken_Keyword, clang_getTokenKind(Tokens[1]));
+ EXPECT_EQ(CXToken_Identifier, clang_getTokenKind(Tokens[2]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[3]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[4]));
+ EXPECT_EQ(CXToken_Punctuation, clang_getTokenKind(Tokens[5]));
+
+ clang_disposeTokens(ClangTU, Tokens, NumTokens);
+ };
+
+ CheckTokenKinds();
+
+ std::string ASTName = "test.ast";
+ WriteFile(ASTName, "");
+
+ ASSERT_TRUE(SaveAndLoadTU(ASTName));
+
+ CheckTokenKinds();
+}
Index: lib/Lex/Preprocessor.cpp
===================================================================
--- lib/Lex/Preprocessor.cpp
+++ lib/Lex/Preprocessor.cpp
@@ -85,12 +85,14 @@
IdentifierInfoLookup *IILookup, bool OwnsHeaders,
TranslationUnitKind TUKind)
: PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts),
- FileMgr(Headers.getFileMgr()), SourceMgr(SM),
- PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)),
- HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
- ExternalSource(nullptr), Identifiers(opts, IILookup),
- PragmaHandlers(new PragmaNamespace(StringRef())), TUKind(TUKind),
- SkipMainFilePreamble(0, true),
+ FileMgr(Headers.getFileMgr()), SourceMgr(SM), PCMCache(PCMCache),
+ ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
+ TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
+ // As the language options may have not been loaded yet (when
+ // deserializing an ASTUnit), adding keywords to the identifier table is
+ // deferred to Preprocessor::Initialize().
+ Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
+ TUKind(TUKind), SkipMainFilePreamble(0, true),
CurSubmoduleState(&NullSubmoduleState) {
OwnsHeaderSearch = OwnsHeaders;
@@ -190,6 +192,9 @@
// Initialize information about built-ins.
BuiltinInfo.InitializeTarget(Target, AuxTarget);
HeaderInfo.setTarget(Target);
+
+ // Populate the identifier table with info about keywords for the current language.
+ Identifiers.AddKeywords(LangOpts);
}
void Preprocessor::InitializeForModelFile() {
Index: lib/Basic/IdentifierTable.cpp
===================================================================
--- lib/Basic/IdentifierTable.cpp
+++ lib/Basic/IdentifierTable.cpp
@@ -79,16 +79,16 @@
return new EmptyLookupIterator();
}
+IdentifierTable::IdentifierTable(IdentifierInfoLookup *externalLookup)
+ : HashTable(8192), // Start with space for 8K identifiers.
+ ExternalLookup(externalLookup) {}
+
IdentifierTable::IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup)
- : HashTable(8192), // Start with space for 8K identifiers.
- ExternalLookup(externalLookup) {
+ IdentifierInfoLookup *externalLookup)
+ : IdentifierTable(externalLookup) {
// Populate the identifier table with info about keywords for the current
// language.
AddKeywords(LangOpts);
-
- // Add the '_experimental_modules_import' contextual keyword.
- get("import").setModulesImport(true);
}
//===----------------------------------------------------------------------===//
@@ -237,6 +237,9 @@
if (LangOpts.DeclSpecKeyword)
AddKeyword("__declspec", tok::kw___declspec, KEYALL, LangOpts, *this);
+
+ // Add the '_experimental_modules_import' contextual keyword.
+ get("import").setModulesImport(true);
}
/// \brief Checks if the specified token kind represents a keyword in the
Index: include/clang/Basic/IdentifierTable.h
===================================================================
--- include/clang/Basic/IdentifierTable.h
+++ include/clang/Basic/IdentifierTable.h
@@ -467,10 +467,13 @@
IdentifierInfoLookup* ExternalLookup;
public:
+ /// \brief Create the identifier table.
+ IdentifierTable(IdentifierInfoLookup *externalLookup = nullptr);
+
/// \brief Create the identifier table, populating it with info about the
/// language keywords for the language specified by \p LangOpts.
IdentifierTable(const LangOptions &LangOpts,
- IdentifierInfoLookup* externalLookup = nullptr);
+ IdentifierInfoLookup *externalLookup = nullptr);
/// \brief Set the external identifier lookup mechanism.
void setExternalIdentifierLookup(IdentifierInfoLookup *IILookup) {
@@ -558,6 +561,8 @@
/// hashing is doing.
void PrintStats() const;
+ /// \brief Populate the identifier table with info about the language keywords
+ /// for the language specified by \p LangOpts.
void AddKeywords(const LangOptions &LangOpts);
};
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
