ilya-biryukov updated this revision to Diff 198662.
ilya-biryukov added a comment.
Herald added subscribers: jsji, kbarton, nemanjai.
- Propagate whether the token came from a token stream through CachingLex.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D59885/new/
https://reviews.llvm.org/D59885
Files:
clang/include/clang/Lex/Preprocessor.h
clang/include/clang/Lex/TokenLexer.h
clang/lib/Lex/PPCaching.cpp
clang/lib/Lex/Preprocessor.cpp
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -636,7 +636,8 @@
break;
case CLK_CachingLexer:
bool IsNewToken;
- CachingLex(Tok, IsNewToken);
+ bool Report;
+ CachingLex(Tok, IsNewToken, Report);
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Tok);
@@ -877,6 +878,11 @@
}
void Preprocessor::Lex(Token &Result) {
+ bool Report;
+ Lex(Result, Report);
+}
+
+void Preprocessor::Lex(Token &Result, bool &Report) {
++LexLevel;
// We loop here until a lex function returns a token; this avoids recursion.
@@ -885,17 +891,21 @@
do {
switch (CurLexerKind) {
case CLK_Lexer:
+ Report = true;
ReturnedToken = CurLexer->Lex(Result);
break;
case CLK_TokenLexer:
+ Report = CurTokenLexer->isMacroExpansion();
ReturnedToken = CurTokenLexer->Lex(Result);
break;
case CLK_CachingLexer:
- CachingLex(Result, IsNewToken);
+ CachingLex(Result, IsNewToken, Report);
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
- ReturnedToken = LexAfterModuleImport(Result);
+ LexAfterModuleImport(Result);
+ Report = true;
+ ReturnedToken = true;
break;
}
} while (!ReturnedToken);
@@ -952,6 +962,8 @@
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && Report)
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/lib/Lex/PPCaching.cpp
===================================================================
--- clang/lib/Lex/PPCaching.cpp
+++ clang/lib/Lex/PPCaching.cpp
@@ -45,7 +45,7 @@
recomputeCurLexerKind();
}
-void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
+void Preprocessor::CachingLex(Token &Result, bool &IsNewToken, bool &Report) {
if (!InCachingLexMode())
return;
@@ -56,11 +56,12 @@
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
IsNewToken = false;
+ Report = false;
return;
}
ExitCachingLexMode();
- Lex(Result);
+ Lex(Result, Report);
if (isBacktrackEnabled()) {
// Cache the lexed token.
Index: clang/include/clang/Lex/TokenLexer.h
===================================================================
--- clang/include/clang/Lex/TokenLexer.h
+++ clang/include/clang/Lex/TokenLexer.h
@@ -147,6 +147,10 @@
/// preprocessor directive.
bool isParsingPreprocessorDirective() const;
+ /// Returns true iff the TokenLexer is expanding a macro and not replaying a
+ /// stream of tokens.
+ bool isMacroExpansion() const { return Macro != nullptr; }
+
private:
void destroy();
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -997,6 +999,12 @@
}
/// \}
+ /// Register a function that would be called on each token in the final
+ /// expanded token stream.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
@@ -1341,6 +1349,12 @@
/// Lex the next token for this preprocessor.
void Lex(Token &Result);
+private:
+ /// Lex the next token and tell whether it should be reported to OnToken
+ /// callback.
+ void Lex(Token &Result, bool &Report);
+
+public:
/// Lex a token, forming a header-name token if possible.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
@@ -2135,7 +2149,7 @@
//===--------------------------------------------------------------------===//
// Caching stuff.
- void CachingLex(Token &Result, bool &IsNewToken);
+ void CachingLex(Token &Result, bool &IsNewToken, bool &Report);
bool InCachingLexMode() const {
// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
