Author: Haojian Wu Date: 2022-03-07T20:24:37+01:00 New Revision: 2d01ac18df93f2f8dd925888aabb7681224fb670
URL: https://github.com/llvm/llvm-project/commit/2d01ac18df93f2f8dd925888aabb7681224fb670 DIFF: https://github.com/llvm/llvm-project/commit/2d01ac18df93f2f8dd925888aabb7681224fb670.diff LOG: [pseudo] Strip comments for TokenStream. Add a utility function to strip comments from a "raw" tokenstream. The derived stream will be fed to the GLR parser (for early testing). Differential Revision: https://reviews.llvm.org/D121092 Added: Modified: clang/include/clang/Tooling/Syntax/Pseudo/Token.h clang/lib/Tooling/Syntax/Pseudo/Token.cpp clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp Removed: ################################################################################ diff --git a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h index 7a73a85eae94d..a54146c2f6f32 100644 --- a/clang/include/clang/Tooling/Syntax/Pseudo/Token.h +++ b/clang/include/clang/Tooling/Syntax/Pseudo/Token.h @@ -195,6 +195,9 @@ enum class LexFlags : uint8_t { /// (And having cooked token kinds in PP-disabled sections is useful for us). TokenStream cook(const TokenStream &, const clang::LangOptions &); +/// Drops comment tokens. +TokenStream stripComments(const TokenStream &); + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp index 070bda4c50031..e7a572a0c5d90 100644 --- a/clang/lib/Tooling/Syntax/Pseudo/Token.cpp +++ b/clang/lib/Tooling/Syntax/Pseudo/Token.cpp @@ -93,6 +93,17 @@ void TokenStream::print(llvm::raw_ostream &OS) const { OS << '\n'; } +TokenStream stripComments(const TokenStream &Input) { + TokenStream Out; + for (const Token &T : Input.tokens()) { + if (T.Kind == tok::comment) + continue; + Out.push(T); + } + Out.finalize(); + return Out; +} + } // namespace pseudo } // namespace syntax } // namespace clang diff --git a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp index f790e65245741..c4314570f2e9e 100644 --- a/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp +++ b/clang/unittests/Tooling/Syntax/Pseudo/TokenTest.cpp @@ -172,6 +172,23 @@ no_indent \ })); } +TEST(TokenTest, DropComments) { + LangOptions Opts; + std::string Code = R"cpp( + // comment + int /*abc*/; +)cpp"; + TokenStream Raw = cook(lex(Code, Opts), Opts); + TokenStream Stripped = stripComments(Raw); + EXPECT_THAT(Raw.tokens(), + ElementsAreArray( + {token("// comment", tok::comment), token("int", tok::kw_int), + token("/*abc*/", tok::comment), token(";", tok::semi)})); + + EXPECT_THAT(Stripped.tokens(), ElementsAreArray({token("int", tok::kw_int), + token(";", tok::semi)})); +} + } // namespace } // namespace pseudo } // namespace syntax _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
