hokein created this revision.
hokein added reviewers: sammccall, ilya-biryukov.
Herald added a project: All.
hokein requested review of this revision.
Herald added a project: clang.
This enables us to use a different underlying token implementation for the
syntax Leaf node -- in clang pseudoparser, we want to produce a
syntax-tree with its own pseudo::Token rather than syntax::Token.
Now Leaf node points to a BaseToken, which can be a different subclass
depending on token implementation.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D128411
Files:
clang/include/clang/Tooling/Syntax/BaseToken.h
clang/include/clang/Tooling/Syntax/Tokens.h
clang/include/clang/Tooling/Syntax/Tree.h
clang/lib/Tooling/Syntax/BuildTree.cpp
clang/lib/Tooling/Syntax/ComputeReplacements.cpp
clang/lib/Tooling/Syntax/Synthesis.cpp
clang/lib/Tooling/Syntax/Tokens.cpp
clang/lib/Tooling/Syntax/Tree.cpp
clang/tools/clang-check/ClangCheck.cpp
clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
clang/unittests/Tooling/Syntax/SynthesisTest.cpp
clang/unittests/Tooling/Syntax/TokensTest.cpp
clang/unittests/Tooling/Syntax/TreeTest.cpp
clang/unittests/Tooling/Syntax/TreeTestBase.cpp
Index: clang/unittests/Tooling/Syntax/TreeTestBase.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/TreeTestBase.cpp
+++ clang/unittests/Tooling/Syntax/TreeTestBase.cpp
@@ -38,10 +38,10 @@
ArrayRef<syntax::Token> tokens(syntax::Node *N) {
assert(N->isOriginal() && "tokens of modified nodes are not well-defined");
if (auto *L = dyn_cast<syntax::Leaf>(N))
- return llvm::makeArrayRef(L->getToken(), 1);
+ return llvm::makeArrayRef(L->getTokenAs<syntax::Token>(), 1);
auto *T = cast<syntax::Tree>(N);
- return llvm::makeArrayRef(T->findFirstLeaf()->getToken(),
- T->findLastLeaf()->getToken() + 1);
+ return llvm::makeArrayRef(T->findFirstLeaf()->getTokenAs<syntax::Token>(),
+ T->findLastLeaf()->getTokenAs<syntax::Token>() + 1);
}
} // namespace
Index: clang/unittests/Tooling/Syntax/TreeTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/TreeTest.cpp
+++ clang/unittests/Tooling/Syntax/TreeTest.cpp
@@ -180,7 +180,7 @@
private:
std::string dumpQuotedTokensOrNull(const Node *N) {
return N ? "'" +
- StringRef(N->dumpTokens(Arena->getSourceManager()))
+ StringRef(N->dumpTokens(&Arena->getSourceManager()))
.trim()
.str() +
"'"
Index: clang/unittests/Tooling/Syntax/TokensTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -220,7 +220,7 @@
}
// An equality test for search.
auto TextMatches = [this](llvm::StringRef Q, const syntax::Token &T) {
- return Q == T.text(*SourceMgr);
+ return Q == T.text(&*SourceMgr);
};
// Find a match.
auto Found =
@@ -906,7 +906,7 @@
SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
Code.points()[Index]);
const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);
- return Tok ? Tok->text(*SourceMgr) : "";
+ return Tok ? Tok->text(&*SourceMgr) : "";
};
EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));
Index: clang/unittests/Tooling/Syntax/SynthesisTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/SynthesisTest.cpp
+++ clang/unittests/Tooling/Syntax/SynthesisTest.cpp
@@ -27,7 +27,8 @@
return ::testing::AssertionFailure()
<< "Root was not built successfully.";
- auto Actual = StringRef(Root->dump(Arena->getSourceManager())).trim().str();
+ auto Actual =
+ StringRef(Root->dump(&Arena->getSourceManager())).trim().str();
auto Expected = Dump.trim().str();
// EXPECT_EQ shows the diff between the two strings if they are different.
EXPECT_EQ(Expected, Actual);
@@ -175,7 +176,7 @@
auto *Copy = deepCopyExpandingMacros(*Arena, StatementContinue);
EXPECT_TRUE(
- treeDumpEqual(Copy, StatementContinue->dump(Arena->getSourceManager())));
+ treeDumpEqual(Copy, StatementContinue->dump(&Arena->getSourceManager())));
// FIXME: Test that copy is independent of original, once the Mutations API is
// more developed.
}
Index: clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
===================================================================
--- clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
+++ clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
@@ -26,7 +26,7 @@
auto ErrorOK = errorOK(Code);
if (!ErrorOK)
return ErrorOK;
- auto Actual = StringRef(Root->dump(Arena->getSourceManager())).trim().str();
+ auto Actual = StringRef(Root->dump(&Arena->getSourceManager())).trim().str();
// EXPECT_EQ shows the diff between the two strings if they are different.
EXPECT_EQ(Tree.trim().str(), Actual);
if (Actual != Tree.trim().str()) {
@@ -59,7 +59,7 @@
auto *AnnotatedNode = nodeByRange(AnnotatedRanges[i], Root);
assert(AnnotatedNode);
auto AnnotatedNodeDump =
- StringRef(AnnotatedNode->dump(Arena->getSourceManager()))
+ StringRef(AnnotatedNode->dump(&Arena->getSourceManager()))
.trim()
.str();
// EXPECT_EQ shows the diff between the two strings if they are different.
Index: clang/tools/clang-check/ClangCheck.cpp
===================================================================
--- clang/tools/clang-check/ClangCheck.cpp
+++ clang/tools/clang-check/ClangCheck.cpp
@@ -159,7 +159,7 @@
llvm::outs() << TB.dumpForTests();
clang::syntax::Arena A(AST.getSourceManager(), AST.getLangOpts(), TB);
llvm::outs() << clang::syntax::buildSyntaxTree(A, AST)->dump(
- AST.getSourceManager());
+ &AST.getSourceManager());
}
private:
Index: clang/lib/Tooling/Syntax/Tree.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Tree.cpp
+++ clang/lib/Tooling/Syntax/Tree.cpp
@@ -49,7 +49,7 @@
return {FID, It.first->second};
}
-syntax::Leaf::Leaf(const syntax::Token *Tok) : Node(NodeKind::Leaf), Tok(Tok) {
+syntax::Leaf::Leaf(const syntax::BaseToken *Tok) : Node(NodeKind::Leaf), Tok(Tok) {
assert(Tok != nullptr);
}
@@ -191,7 +191,7 @@
namespace {
static void dumpLeaf(raw_ostream &OS, const syntax::Leaf *L,
- const SourceManager &SM) {
+ const void* SM) {
assert(L);
const auto *Token = L->getToken();
assert(Token);
@@ -203,7 +203,7 @@
}
static void dumpNode(raw_ostream &OS, const syntax::Node *N,
- const SourceManager &SM, llvm::BitVector IndentMask) {
+ const void* SM, llvm::BitVector IndentMask) {
auto DumpExtraInfo = [&OS](const syntax::Node *N) {
if (N->getRole() != syntax::NodeRole::Unknown)
OS << " " << N->getRole();
@@ -248,14 +248,14 @@
}
} // namespace
-std::string syntax::Node::dump(const SourceManager &SM) const {
+std::string syntax::Node::dump(const void* SM) const {
std::string Str;
llvm::raw_string_ostream OS(Str);
dumpNode(OS, this, SM, /*IndentMask=*/{});
return std::move(OS.str());
}
-std::string syntax::Node::dumpTokens(const SourceManager &SM) const {
+std::string syntax::Node::dumpTokens(const void* SM) const {
std::string Storage;
llvm::raw_string_ostream OS(Storage);
traverse(this, [&](const syntax::Node *N) {
Index: clang/lib/Tooling/Syntax/Tokens.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Tokens.cpp
+++ clang/lib/Tooling/Syntax/Tokens.cpp
@@ -100,7 +100,7 @@
syntax::Token::Token(SourceLocation Location, unsigned Length,
tok::TokenKind Kind)
- : Location(Location), Length(Length), Kind(Kind) {
+ : BaseToken(Kind), Location(Location), Length(Length) {
assert(Location.isValid());
}
@@ -115,6 +115,10 @@
assert(!Invalid);
return llvm::StringRef(Start, length());
}
+llvm::StringRef syntax::Token::text(const void* Ctx) const {
+ const auto* SM = reinterpret_cast<const SourceManager*>(Ctx);
+ return text(*SM);
+}
FileRange syntax::Token::range(const SourceManager &SM) const {
assert(location().isFileID() && "must be a spelled token");
@@ -860,7 +864,7 @@
}
std::string syntax::Token::dumpForTests(const SourceManager &SM) const {
- return std::string(llvm::formatv("Token(`{0}`, {1}, length = {2})", text(SM),
+ return std::string(llvm::formatv("Token(`{0}`, {1}, length = {2})", text(&SM),
tok::getTokenName(kind()), length()));
}
@@ -868,7 +872,7 @@
auto PrintToken = [this](const syntax::Token &T) -> std::string {
if (T.kind() == tok::eof)
return "<eof>";
- return std::string(T.text(*SourceMgr));
+ return std::string(T.text(SourceMgr));
};
auto DumpTokens = [this, &PrintToken](llvm::raw_ostream &OS,
@@ -877,7 +881,7 @@
OS << "<empty>";
return;
}
- OS << Tokens[0].text(*SourceMgr);
+ OS << Tokens[0].text(SourceMgr);
for (unsigned I = 1; I < Tokens.size(); ++I) {
if (Tokens[I].kind() == tok::eof)
continue;
Index: clang/lib/Tooling/Syntax/Synthesis.cpp
===================================================================
--- clang/lib/Tooling/Syntax/Synthesis.cpp
+++ clang/lib/Tooling/Syntax/Synthesis.cpp
@@ -213,7 +213,7 @@
// `L->getToken()` gives us the expanded token, thus we implicitly expand
// any macros here.
return createLeaf(A, L->getToken()->kind(),
- L->getToken()->text(A.getSourceManager()));
+ L->getToken()->text(&A.getSourceManager()));
const auto *T = cast<syntax::Tree>(N);
std::vector<std::pair<syntax::Node *, syntax::NodeRole>> Children;
Index: clang/lib/Tooling/Syntax/ComputeReplacements.cpp
===================================================================
--- clang/lib/Tooling/Syntax/ComputeReplacements.cpp
+++ clang/lib/Tooling/Syntax/ComputeReplacements.cpp
@@ -48,7 +48,7 @@
if (SpanBegin)
Callback(llvm::makeArrayRef(SpanBegin, SpanEnd), SpanIsOriginal);
// Start recording a new span.
- SpanBegin = L->getToken();
+ SpanBegin = L->getTokenAs<syntax::Token>();
SpanEnd = SpanBegin + 1;
SpanIsOriginal = L->isOriginal();
}
Index: clang/lib/Tooling/Syntax/BuildTree.cpp
===================================================================
--- clang/lib/Tooling/Syntax/BuildTree.cpp
+++ clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -671,7 +671,7 @@
R += std::string(
formatv("- '{0}' covers '{1}'+{2} tokens\n", It->second->getKind(),
It->first->text(A.getSourceManager()), CoveredTokens));
- R += It->second->dump(A.getSourceManager());
+ R += It->second->dump(&A.getSourceManager());
}
return R;
}
@@ -896,7 +896,7 @@
// token kind, we run `NumericLiteralParser` again to distinguish them.
auto TokLoc = S->getBeginLoc();
auto TokSpelling =
- Builder.findToken(TokLoc)->text(Context.getSourceManager());
+ Builder.findToken(TokLoc)->text(&Context.getSourceManager());
auto Literal =
NumericLiteralParser(TokSpelling, TokLoc, Context.getSourceManager(),
Context.getLangOpts(), Context.getTargetInfo(),
Index: clang/include/clang/Tooling/Syntax/Tree.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Tree.h
+++ clang/include/clang/Tooling/Syntax/Tree.h
@@ -122,9 +122,9 @@
Node *getPreviousSibling() { return PreviousSibling; }
/// Dumps the structure of a subtree. For debugging and testing purposes.
- std::string dump(const SourceManager &SM) const;
+ std::string dump(const void *SM) const;
/// Dumps the tokens forming this subtree.
- std::string dumpTokens(const SourceManager &SM) const;
+ std::string dumpTokens(const void *SM) const;
/// Asserts invariants on this node of the tree and its immediate children.
/// Will not recurse into the subtree. No-op if NDEBUG is set.
@@ -153,16 +153,19 @@
unsigned CanModify : 1;
};
-/// A leaf node points to a single token inside the expanded token stream.
+/// A leaf node points to a single token.
class Leaf final : public Node {
public:
- Leaf(const Token *T);
+ Leaf(const BaseToken *T);
static bool classof(const Node *N);
- const Token *getToken() const { return Tok; }
+ const BaseToken *getToken() const { return Tok; }
+ template <typename T> const T *getTokenAs() const {
+ return static_cast<const T *>(Tok);
+ }
private:
- const Token *Tok;
+ const BaseToken *Tok;
};
/// A node that has children and represents a syntactic language construct.
Index: clang/include/clang/Tooling/Syntax/Tokens.h
===================================================================
--- clang/include/clang/Tooling/Syntax/Tokens.h
+++ clang/include/clang/Tooling/Syntax/Tokens.h
@@ -33,6 +33,7 @@
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TokenKinds.h"
#include "clang/Lex/Token.h"
+#include "clang/Tooling/Syntax/BaseToken.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Optional.h"
@@ -102,13 +103,12 @@
/// A token coming directly from a file or from a macro invocation. Has just
/// enough information to locate the token in the source code.
/// Can represent both expanded and spelled tokens.
-class Token {
+class Token : public BaseToken {
public:
Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind);
/// EXPECTS: clang::Token is not an annotation token.
explicit Token(const clang::Token &T);
- tok::TokenKind kind() const { return Kind; }
/// Location of the first character of a token.
SourceLocation location() const { return Location; }
/// Location right after the last character of a token.
@@ -122,7 +122,10 @@
/// in\
/// t
/// both have the same kind tok::kw_int, but results of text() are different.
- llvm::StringRef text(const SourceManager &SM) const;
+ llvm::StringRef text(const SourceManager& SM) const;
+ /// A wrapper of the above text().
+ /// EXPECTS: The opauqe parameter must be a `const clang::SourceManager*`.
+ llvm::StringRef text(const void* SM) const override;
/// Gets a range of this token.
/// EXPECTS: token comes from a file, not from a macro expansion.
Index: clang/include/clang/Tooling/Syntax/BaseToken.h
===================================================================
--- /dev/null
+++ clang/include/clang/Tooling/Syntax/BaseToken.h
@@ -0,0 +1,44 @@
+//===- BaseToken.h - base token class for syntax tree ------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines common interfaces for the Token class. The syntax-tree Leaf node
+// points to a single token, this allows using different underlying token
+// implementations (syntax::Token, pseudo::Token) in the syntax-tree.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H
+#define LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H
+
+#include "clang/Basic/TokenKinds.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace syntax {
+
+/// Base class that represents a token for the C++ source code.
+///
+/// Depending on the subclass implementation, it can be a spelled token from
+/// the original source file or an expanded token.
+class BaseToken {
+public:
+ BaseToken(tok::TokenKind K) : Kind(K) {}
+
+ tok::TokenKind kind() const { return Kind; }
+
+ /// Returns the text of the token.
+ virtual llvm::StringRef text(const void *Ctx = nullptr) const = 0;
+
+private:
+ tok::TokenKind Kind;
+};
+
+} // namespace syntax
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLING_SYNTAX_BASE_TOKEN_H
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
