Hahnfeld created this revision.
Hahnfeld added reviewers: hokein, aaron.ballman.
Herald added subscribers: kbarton, nemanjai.
Herald added a project: All.
Hahnfeld requested review of this revision.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
This new method repeatedly calls `Lex()` until end of file is reached and
returns a `std::vector` of `Token`s. Use it in Clang's unit tests to avoid
quite some code duplication.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D158413
Files:
clang/include/clang/Lex/Preprocessor.h
clang/lib/Lex/Preprocessor.cpp
clang/unittests/Analysis/MacroExpansionContextTest.cpp
clang/unittests/Basic/SourceManagerTest.cpp
clang/unittests/Lex/LexerTest.cpp
clang/unittests/Lex/ModuleDeclStateTest.cpp
clang/unittests/Lex/PPCallbacksTest.cpp
clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
clang/unittests/Lex/PPDependencyDirectivesTest.cpp
clang/unittests/Lex/PPMemoryAllocationsTest.cpp
Index: clang/unittests/Lex/PPMemoryAllocationsTest.cpp
===================================================================
--- clang/unittests/Lex/PPMemoryAllocationsTest.cpp
+++ clang/unittests/Lex/PPMemoryAllocationsTest.cpp
@@ -75,12 +75,7 @@
PP.Initialize(*Target);
PP.EnterMainSourceFile();
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
size_t NumAllocated = PP.getPreprocessorAllocator().getBytesAllocated();
float BytesPerDefine = float(NumAllocated) / float(NumMacros);
Index: clang/unittests/Lex/PPDependencyDirectivesTest.cpp
===================================================================
--- clang/unittests/Lex/PPDependencyDirectivesTest.cpp
+++ clang/unittests/Lex/PPDependencyDirectivesTest.cpp
@@ -133,12 +133,7 @@
SmallVector<StringRef> IncludedFiles;
PP.addPPCallbacks(std::make_unique<IncludeCollector>(PP, IncludedFiles));
PP.EnterMainSourceFile();
- while (true) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
SmallVector<StringRef> ExpectedIncludes{
"main.c", "./head1.h", "./head2.h", "./head2.h", "./head3.h", "./head3.h",
Index: clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
===================================================================
--- clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
+++ clang/unittests/Lex/PPConditionalDirectiveRecordTest.cpp
@@ -86,14 +86,7 @@
PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(PPRec));
PP.EnterMainSourceFile();
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP.LexAll();
// Make sure we got the tokens that we expected.
ASSERT_EQ(10U, toks.size());
Index: clang/unittests/Lex/PPCallbacksTest.cpp
===================================================================
--- clang/unittests/Lex/PPCallbacksTest.cpp
+++ clang/unittests/Lex/PPCallbacksTest.cpp
@@ -229,13 +229,7 @@
// Lex source text.
PP.EnterMainSourceFile();
-
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
// Callbacks have been executed at this point -- return filename range.
return Callbacks;
@@ -259,13 +253,7 @@
// Lex source text.
PP.EnterMainSourceFile();
-
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
return Callbacks->Results;
}
@@ -290,12 +278,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
return Callbacks->Marks;
}
@@ -334,12 +317,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
PragmaOpenCLExtensionCallbacks::CallbackParameters RetVal = {
Callbacks->Name,
@@ -477,12 +455,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
ASSERT_EQ(1u, Callbacks->NumCalls);
ASSERT_EQ(0u, DiagConsumer->getNumErrors());
Index: clang/unittests/Lex/ModuleDeclStateTest.cpp
===================================================================
--- clang/unittests/Lex/ModuleDeclStateTest.cpp
+++ clang/unittests/Lex/ModuleDeclStateTest.cpp
@@ -90,12 +90,7 @@
PP.addPPCallbacks(std::move(C));
PP.EnterMainSourceFile();
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
}
FileSystemOptions FileMgrOpts;
Index: clang/unittests/Lex/LexerTest.cpp
===================================================================
--- clang/unittests/Lex/LexerTest.cpp
+++ clang/unittests/Lex/LexerTest.cpp
@@ -71,14 +71,7 @@
TrivialModuleLoader ModLoader;
PP = CreatePP(Source, ModLoader);
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP->Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP->LexAll();
return toks;
}
@@ -626,12 +619,7 @@
TEST_F(LexerTest, CreatedFIDCountForPredefinedBuffer) {
TrivialModuleLoader ModLoader;
auto PP = CreatePP("", ModLoader);
- while (1) {
- Token tok;
- PP->Lex(tok);
- if (tok.is(tok::eof))
- break;
- }
+ (void)PP->LexAll();
EXPECT_EQ(SourceMgr.getNumCreatedFIDsForFileID(PP->getPredefinesFileID()),
1U);
}
Index: clang/unittests/Basic/SourceManagerTest.cpp
===================================================================
--- clang/unittests/Basic/SourceManagerTest.cpp
+++ clang/unittests/Basic/SourceManagerTest.cpp
@@ -137,14 +137,7 @@
PP.Initialize(*Target);
PP.EnterMainSourceFile();
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP.LexAll();
// Make sure we got the tokens that we expected.
ASSERT_EQ(3U, toks.size());
@@ -194,14 +187,7 @@
PP.EnterMainSourceFile();
llvm::SmallString<8> Scratch;
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP.LexAll();
// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size()) << "a >> b c";
@@ -451,14 +437,7 @@
PP.Initialize(*Target);
PP.EnterMainSourceFile();
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP.LexAll();
// Make sure we got the tokens that we expected.
ASSERT_EQ(4U, toks.size());
@@ -573,14 +552,7 @@
PP.EnterMainSourceFile();
- std::vector<Token> toks;
- while (1) {
- Token tok;
- PP.Lex(tok);
- if (tok.is(tok::eof))
- break;
- toks.push_back(tok);
- }
+ std::vector<Token> toks = PP.LexAll();
// Make sure we got the tokens that we expected.
ASSERT_EQ(0U, toks.size());
Index: clang/unittests/Analysis/MacroExpansionContextTest.cpp
===================================================================
--- clang/unittests/Analysis/MacroExpansionContextTest.cpp
+++ clang/unittests/Analysis/MacroExpansionContextTest.cpp
@@ -73,12 +73,7 @@
// Lex source text.
PP.EnterMainSourceFile();
- while (true) {
- Token Tok;
- PP.Lex(Tok);
- if (Tok.is(tok::eof))
- break;
- }
+ (void)PP.LexAll();
// Callbacks have been executed at this point.
return Ctx;
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -995,6 +995,18 @@
}
}
+std::vector<Token> Preprocessor::LexAll() {
+ std::vector<Token> toks;
+ while (1) {
+ Token tok;
+ Lex(tok);
+ if (tok.is(tok::eof))
+ break;
+ toks.push_back(tok);
+ }
+ return toks;
+}
+
/// Lex a header-name token (including one formed from header-name-tokens if
/// \p AllowConcatenation is \c true).
///
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -1718,6 +1718,9 @@
/// Lex the next token for this preprocessor.
void Lex(Token &Result);
+ /// Lex all tokens for this preprocessor until end of file.
+ std::vector<Token> LexAll();
+
/// Lex a token, forming a header-name token if possible.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
