Author: Zequan Wu Date: 2022-03-18T10:15:48-07:00 New Revision: 217f267efe3082438e698e2f08566b9df8c530fa
URL: https://github.com/llvm/llvm-project/commit/217f267efe3082438e698e2f08566b9df8c530fa DIFF: https://github.com/llvm/llvm-project/commit/217f267efe3082438e698e2f08566b9df8c530fa.diff LOG: Revert "[pseudo] Split greatergreater token." This reverts commit f66d3758bda99e9f57bfdad168212feda18792ae. It breaks windows bot. Added: Modified: clang-tools-extra/pseudo/include/clang-pseudo/Token.h clang-tools-extra/pseudo/lib/Lex.cpp clang-tools-extra/pseudo/lib/cxx.bnf clang-tools-extra/pseudo/unittests/TokenTest.cpp Removed: ################################################################################ diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h index 4563477b2c4fe..24b6729151e61 100644 --- a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h @@ -180,8 +180,7 @@ enum class LexFlags : uint8_t { NeedsCleaning = 1 << 1, }; -/// Derives a token stream by decoding escapes, interpreting raw_identifiers and -/// splitting the greatergreater token. +/// Derives a token stream by decoding escapes and interpreting raw_identifiers. /// /// Tokens containing UCNs, escaped newlines, trigraphs etc are decoded and /// their backing data is owned by the returned stream. diff --git a/clang-tools-extra/pseudo/lib/Lex.cpp b/clang-tools-extra/pseudo/lib/Lex.cpp index e99bf3a63e5e1..f5a239533c532 100644 --- a/clang-tools-extra/pseudo/lib/Lex.cpp +++ b/clang-tools-extra/pseudo/lib/Lex.cpp @@ -98,21 +98,9 @@ TokenStream cook(const TokenStream &Code, const LangOptions &LangOpts) { Tok.Length = Text.size(); Tok.Flags &= ~static_cast<decltype(Tok.Flags)>(LexFlags::NeedsCleaning); } - - if (Tok.Kind == tok::raw_identifier) { - // Cook raw_identifiers into identifier, keyword, etc. + // Cook raw_identifiers into identifier, keyword, etc. + if (Tok.Kind == tok::raw_identifier) Tok.Kind = Identifiers.get(Tok.text()).getTokenID(); - } else if (Tok.Kind == tok::greatergreater) { - // Split the greatergreater token. - // FIXME: split lessless token to support Cuda triple angle brackets <<<. - assert(Tok.text() == ">>"); - Tok.Kind = tok::greater; - Tok.Length = 1; - Result.push(Tok); - // Line is wrong if the first greater is followed by an escaped newline! - Tok.Data = Tok.text().data() + 1; - } - Result.push(std::move(Tok)); } diff --git a/clang-tools-extra/pseudo/lib/cxx.bnf b/clang-tools-extra/pseudo/lib/cxx.bnf index cf664b8e13e55..48bf4621eefe5 100644 --- a/clang-tools-extra/pseudo/lib/cxx.bnf +++ b/clang-tools-extra/pseudo/lib/cxx.bnf @@ -13,9 +13,6 @@ # - the file merely describes the core C++ grammar. Preprocessor directives and # lexical conversions are omitted as we reuse clang's lexer and run a fake # preprocessor; -# - grammar rules with the >> token are adjusted, the greatergreater token is -# split into two > tokens, to make the GLR parser aware of nested templates -# and right shift operator; # # Guidelines: # - nonterminals are lower_case; terminals (aka tokens) correspond to @@ -99,7 +96,7 @@ fold-operator := % fold-operator := ^ fold-operator := | fold-operator := << -fold-operator := greatergreater +fold-operator := >> fold-operator := += fold-operator := -= fold-operator := *= @@ -205,7 +202,7 @@ additive-expression := additive-expression - multiplicative-expression # expr.shift shift-expression := additive-expression shift-expression := shift-expression << additive-expression -shift-expression := shift-expression greatergreater additive-expression +shift-expression := shift-expression >> additive-expression # expr.spaceship compare-expression := shift-expression compare-expression := compare-expression <=> shift-expression @@ -618,7 +615,7 @@ operator-name := <=> operator-name := ^^ operator-name := || operator-name := << -operator-name := greatergreater +operator-name := >> operator-name := <<= operator-name := >>= operator-name := ++ @@ -740,8 +737,3 @@ contextual-zero := NUMERIC_CONSTANT module-keyword := IDENTIFIER import-keyword := IDENTIFIER export-keyword := IDENTIFIER - -#! greatergreater token -- clang lexer always lexes it as a single token, we -#! split it into two tokens to make the GLR parser aware of the nested-template -#! case. -greatergreater := > > diff --git a/clang-tools-extra/pseudo/unittests/TokenTest.cpp b/clang-tools-extra/pseudo/unittests/TokenTest.cpp index b17f8c93c1763..1357d23501193 100644 --- a/clang-tools-extra/pseudo/unittests/TokenTest.cpp +++ b/clang-tools-extra/pseudo/unittests/TokenTest.cpp @@ -171,25 +171,6 @@ no_indent \ })); } -TEST(TokenTest, SplitGreaterGreater) { - LangOptions Opts; - std::string Code = R"cpp( ->> // split -// >> with an escaped newline in the middle, split ->\ -> ->>= // not split -)cpp"; - TokenStream Split = stripComments(cook(lex(Code, Opts), Opts)); - EXPECT_THAT(Split.tokens(), ElementsAreArray({ - token(">", tok::greater), - token(">", tok::greater), - token(">", tok::greater), - token(">", tok::greater), - token(">>=", tok::greatergreaterequal), - })); -} - TEST(TokenTest, DropComments) { LangOptions Opts; std::string Code = R"cpp( _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
