ilya-biryukov updated this revision to Diff 199485.
ilya-biryukov added a comment.
- Use a flag inside clang::Token instead
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D59885/new/
https://reviews.llvm.org/D59885
Files:
clang/include/clang/Lex/Preprocessor.h
clang/include/clang/Lex/Token.h
clang/include/clang/Lex/TokenLexer.h
clang/lib/Lex/PPCaching.cpp
clang/lib/Lex/Preprocessor.cpp
Index: clang/lib/Lex/Preprocessor.cpp
===================================================================
--- clang/lib/Lex/Preprocessor.cpp
+++ clang/lib/Lex/Preprocessor.cpp
@@ -635,8 +635,7 @@
CurTokenLexer->Lex(Tok);
break;
case CLK_CachingLexer:
- bool IsNewToken;
- CachingLex(Tok, IsNewToken);
+ CachingLex(Tok);
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Tok);
@@ -880,22 +879,27 @@
++LexLevel;
// We loop here until a lex function returns a token; this avoids recursion.
+ // FIXME: IsNewToken should be set inside lexers.
bool ReturnedToken;
- bool IsNewToken = true;
do {
switch (CurLexerKind) {
case CLK_Lexer:
ReturnedToken = CurLexer->Lex(Result);
+ Result.setFlag(Token::IsNewToken);
break;
- case CLK_TokenLexer:
+ case CLK_TokenLexer: {
+ bool IsNewToken = CurTokenLexer->isMacroExpansion();
ReturnedToken = CurTokenLexer->Lex(Result);
+ Result.setFlagValue(Token::IsNewToken, IsNewToken);
break;
+ }
case CLK_CachingLexer:
- CachingLex(Result, IsNewToken);
+ CachingLex(Result);
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
ReturnedToken = LexAfterModuleImport(Result);
+ Result.setFlag(Token::IsNewToken);
break;
}
} while (!ReturnedToken);
@@ -911,7 +915,8 @@
// Update ImportSeqState to track our position within a C++20 import-seq
// if this token is being produced as a result of phase 4 of translation.
- if (getLangOpts().CPlusPlusModules && LexLevel == 1 && IsNewToken) {
+ if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
+ Result.getFlag(Token::IsNewToken)) {
switch (Result.getKind()) {
case tok::l_paren: case tok::l_square: case tok::l_brace:
ImportSeqState.handleOpenBracket();
@@ -952,6 +957,8 @@
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && Result.getFlag(Token::IsNewToken))
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
Index: clang/lib/Lex/PPCaching.cpp
===================================================================
--- clang/lib/Lex/PPCaching.cpp
+++ clang/lib/Lex/PPCaching.cpp
@@ -45,7 +45,7 @@
recomputeCurLexerKind();
}
-void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
+void Preprocessor::CachingLex(Token &Result) {
if (!InCachingLexMode())
return;
@@ -55,7 +55,8 @@
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
- IsNewToken = false;
+ // FIXME: do this flag when writing to CachedTokens.
+ Result.clearFlag(Token::IsNewToken);
return;
}
Index: clang/include/clang/Lex/TokenLexer.h
===================================================================
--- clang/include/clang/Lex/TokenLexer.h
+++ clang/include/clang/Lex/TokenLexer.h
@@ -147,6 +147,10 @@
/// preprocessor directive.
bool isParsingPreprocessorDirective() const;
+ /// Returns true iff the TokenLexer is expanding a macro and not replaying a
+ /// stream of tokens.
+ bool isMacroExpansion() const { return Macro != nullptr; }
+
private:
void destroy();
Index: clang/include/clang/Lex/Token.h
===================================================================
--- clang/include/clang/Lex/Token.h
+++ clang/include/clang/Lex/Token.h
@@ -70,20 +70,22 @@
public:
// Various flags set per token:
enum TokenFlags {
- StartOfLine = 0x01, // At start of line or only after whitespace
- // (considering the line after macro expansion).
- LeadingSpace = 0x02, // Whitespace exists before this token (considering
- // whitespace after macro expansion).
- DisableExpand = 0x04, // This identifier may never be macro expanded.
- NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
+ StartOfLine = 0x01, // At start of line or only after whitespace
+ // (considering the line after macro expansion).
+ LeadingSpace = 0x02, // Whitespace exists before this token (considering
+ // whitespace after macro expansion).
+ DisableExpand = 0x04, // This identifier may never be macro expanded.
+ NeedsCleaning = 0x08, // Contained an escaped newline or trigraph.
LeadingEmptyMacro = 0x10, // Empty macro exists before this token.
- HasUDSuffix = 0x20, // This string or character literal has a ud-suffix.
- HasUCN = 0x40, // This identifier contains a UCN.
- IgnoredComma = 0x80, // This comma is not a macro argument separator (MS).
+ HasUDSuffix = 0x20, // This string or character literal has a ud-suffix.
+ HasUCN = 0x40, // This identifier contains a UCN.
+ IgnoredComma = 0x80, // This comma is not a macro argument separator (MS).
StringifiedInMacro = 0x100, // This string or character literal is formed by
// macro stringizing or charizing operator.
CommaAfterElided = 0x200, // The comma following this token was elided (MS).
IsEditorPlaceholder = 0x400, // This identifier is a placeholder.
+ IsNewToken = 0x800, // This tokens has not been seen before, i.e. not from a
+ // cached token stream.
};
tok::TokenKind getKind() const { return Kind; }
Index: clang/include/clang/Lex/Preprocessor.h
===================================================================
--- clang/include/clang/Lex/Preprocessor.h
+++ clang/include/clang/Lex/Preprocessor.h
@@ -33,6 +33,7 @@
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/FunctionExtras.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/PointerUnion.h"
@@ -48,8 +49,8 @@
#include <cassert>
#include <cstddef>
#include <cstdint>
-#include <memory>
#include <map>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -124,6 +125,7 @@
friend class VAOptDefinitionContext;
friend class VariadicMacroScopeGuard;
+ llvm::unique_function<void(const clang::Token &)> OnToken;
std::shared_ptr<PreprocessorOptions> PPOpts;
DiagnosticsEngine *Diags;
LangOptions &LangOpts;
@@ -997,6 +999,12 @@
}
/// \}
+ /// Register a function that would be called on each token in the final
+ /// expanded token stream.
+ void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
+ OnToken = std::move(F);
+ }
+
bool isMacroDefined(StringRef Id) {
return isMacroDefined(&Identifiers.get(Id));
}
@@ -1341,6 +1349,7 @@
/// Lex the next token for this preprocessor.
void Lex(Token &Result);
+public:
/// Lex a token, forming a header-name token if possible.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
@@ -2135,7 +2144,7 @@
//===--------------------------------------------------------------------===//
// Caching stuff.
- void CachingLex(Token &Result, bool &IsNewToken);
+ void CachingLex(Token &Result);
bool InCachingLexMode() const {
// If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
