Author: Manuel Klimek Date: 2023-01-31T14:32:31Z New Revision: c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69
URL: https://github.com/llvm/llvm-project/commit/c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69 DIFF: https://github.com/llvm/llvm-project/commit/c3bc61d72f8da9a2b45e610ee3c2ccfc5f884f69.diff LOG: [clang-format][NFC] Pull FormatTokenSource into its own header. Prepare getting FormatTokenSource under unit testing. Added: clang/lib/Format/FormatTokenSource.h Modified: clang/lib/Format/FormatToken.h clang/lib/Format/UnwrappedLineParser.cpp Removed: ################################################################################ diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h index 9d055efd80075..f3349a4dc2dfa 100644 --- a/clang/lib/Format/FormatToken.h +++ b/clang/lib/Format/FormatToken.h @@ -1803,6 +1803,25 @@ struct AdditionalKeywords { std::unordered_set<IdentifierInfo *> VerilogExtraKeywords; }; +inline bool isLineComment(const FormatToken &FormatTok) { + return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*"); +} + +// Checks if \p FormatTok is a line comment that continues the line comment +// \p Previous. The original column of \p MinColumnToken is used to determine +// whether \p FormatTok is indented enough to the right to continue \p Previous. +inline bool continuesLineComment(const FormatToken &FormatTok, + const FormatToken *Previous, + const FormatToken *MinColumnToken) { + if (!Previous || !MinColumnToken) + return false; + unsigned MinContinueColumn = + MinColumnToken->OriginalColumn + (isLineComment(*MinColumnToken) ? 0 : 1); + return isLineComment(FormatTok) && FormatTok.NewlinesBefore == 1 && + isLineComment(*Previous) && + FormatTok.OriginalColumn >= MinContinueColumn; +} + } // namespace format } // namespace clang diff --git a/clang/lib/Format/FormatTokenSource.h b/clang/lib/Format/FormatTokenSource.h new file mode 100644 index 0000000000000..11c735d368647 --- /dev/null +++ b/clang/lib/Format/FormatTokenSource.h @@ -0,0 +1,132 @@ + +//===--- FormatTokenSource.h - Format C++ code ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file defines the \c TokenSource interface, which provides a token +/// stream as well as the ability to manipulate the token stream. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H +#define LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H + +#include "FormatToken.h" +#include "UnwrappedLineParser.h" + +#define DEBUG_TYPE "format-token-source" + +namespace clang { +namespace format { + +class FormatTokenSource { +public: + virtual ~FormatTokenSource() {} + + // Returns the next token in the token stream. + virtual FormatToken *getNextToken() = 0; + + // Returns the token preceding the token returned by the last call to + // getNextToken() in the token stream, or nullptr if no such token exists. + virtual FormatToken *getPreviousToken() = 0; + + // Returns the token that would be returned by the next call to + // getNextToken(). + virtual FormatToken *peekNextToken(bool SkipComment = false) = 0; + + // Returns whether we are at the end of the file. + // This can be diff erent from whether getNextToken() returned an eof token + // when the FormatTokenSource is a view on a part of the token stream. + virtual bool isEOF() = 0; + + // Gets the current position in the token stream, to be used by setPosition(). + virtual unsigned getPosition() = 0; + + // Resets the token stream to the state it was in when getPosition() returned + // Position, and return the token at that position in the stream. + virtual FormatToken *setPosition(unsigned Position) = 0; +}; + +class ScopedMacroState : public FormatTokenSource { +public: + ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource, + FormatToken *&ResetToken) + : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken), + PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource), + Token(nullptr), PreviousToken(nullptr) { + FakeEOF.Tok.startToken(); + FakeEOF.Tok.setKind(tok::eof); + TokenSource = this; + Line.Level = 0; + Line.InPPDirective = true; + // InMacroBody gets set after the `#define x` part. + } + + ~ScopedMacroState() override { + TokenSource = PreviousTokenSource; + ResetToken = Token; + Line.InPPDirective = false; + Line.InMacroBody = false; + Line.Level = PreviousLineLevel; + } + + FormatToken *getNextToken() override { + // The \c UnwrappedLineParser guards against this by never calling + // \c getNextToken() after it has encountered the first eof token. + assert(!eof()); + PreviousToken = Token; + Token = PreviousTokenSource->getNextToken(); + if (eof()) + return &FakeEOF; + return Token; + } + + FormatToken *getPreviousToken() override { + return PreviousTokenSource->getPreviousToken(); + } + + FormatToken *peekNextToken(bool SkipComment) override { + if (eof()) + return &FakeEOF; + return PreviousTokenSource->peekNextToken(SkipComment); + } + + bool isEOF() override { return PreviousTokenSource->isEOF(); } + + unsigned getPosition() override { return PreviousTokenSource->getPosition(); } + + FormatToken *setPosition(unsigned Position) override { + PreviousToken = nullptr; + Token = PreviousTokenSource->setPosition(Position); + return Token; + } + +private: + bool eof() { + return Token && Token->HasUnescapedNewline && + !continuesLineComment(*Token, PreviousToken, + /*MinColumnToken=*/PreviousToken); + } + + FormatToken FakeEOF; + UnwrappedLine &Line; + FormatTokenSource *&TokenSource; + FormatToken *&ResetToken; + unsigned PreviousLineLevel; + FormatTokenSource *PreviousTokenSource; + + FormatToken *Token; + FormatToken *PreviousToken; +}; + +} // namespace format +} // namespace clang + +#undef DEBUG_TYPE + +#endif \ No newline at end of file diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp index 3e58b6f90559b..9d90e5390bb81 100644 --- a/clang/lib/Format/UnwrappedLineParser.cpp +++ b/clang/lib/Format/UnwrappedLineParser.cpp @@ -14,6 +14,7 @@ #include "UnwrappedLineParser.h" #include "FormatToken.h" +#include "FormatTokenSource.h" #include "TokenAnnotator.h" #include "clang/Basic/TokenKinds.h" #include "llvm/ADT/STLExtras.h" @@ -28,34 +29,6 @@ namespace clang { namespace format { -class FormatTokenSource { -public: - virtual ~FormatTokenSource() {} - - // Returns the next token in the token stream. - virtual FormatToken *getNextToken() = 0; - - // Returns the token preceding the token returned by the last call to - // getNextToken() in the token stream, or nullptr if no such token exists. - virtual FormatToken *getPreviousToken() = 0; - - // Returns the token that would be returned by the next call to - // getNextToken(). - virtual FormatToken *peekNextToken(bool SkipComment = false) = 0; - - // Returns whether we are at the end of the file. - // This can be diff erent from whether getNextToken() returned an eof token - // when the FormatTokenSource is a view on a part of the token stream. - virtual bool isEOF() = 0; - - // Gets the current position in the token stream, to be used by setPosition(). - virtual unsigned getPosition() = 0; - - // Resets the token stream to the state it was in when getPosition() returned - // Position, and return the token at that position in the stream. - virtual FormatToken *setPosition(unsigned Position) = 0; -}; - namespace { void printLine(llvm::raw_ostream &OS, const UnwrappedLine &Line, @@ -112,97 +85,6 @@ class ScopedDeclarationState { llvm::BitVector &Stack; }; -static bool isLineComment(const FormatToken &FormatTok) { - return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*"); -} - -// Checks if \p FormatTok is a line comment that continues the line comment -// \p Previous. The original column of \p MinColumnToken is used to determine -// whether \p FormatTok is indented enough to the right to continue \p Previous. -static bool continuesLineComment(const FormatToken &FormatTok, - const FormatToken *Previous, - const FormatToken *MinColumnToken) { - if (!Previous || !MinColumnToken) - return false; - unsigned MinContinueColumn = - MinColumnToken->OriginalColumn + (isLineComment(*MinColumnToken) ? 0 : 1); - return isLineComment(FormatTok) && FormatTok.NewlinesBefore == 1 && - isLineComment(*Previous) && - FormatTok.OriginalColumn >= MinContinueColumn; -} - -class ScopedMacroState : public FormatTokenSource { -public: - ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource, - FormatToken *&ResetToken) - : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken), - PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource), - Token(nullptr), PreviousToken(nullptr) { - FakeEOF.Tok.startToken(); - FakeEOF.Tok.setKind(tok::eof); - TokenSource = this; - Line.Level = 0; - Line.InPPDirective = true; - // InMacroBody gets set after the `#define x` part. - } - - ~ScopedMacroState() override { - TokenSource = PreviousTokenSource; - ResetToken = Token; - Line.InPPDirective = false; - Line.InMacroBody = false; - Line.Level = PreviousLineLevel; - } - - FormatToken *getNextToken() override { - // The \c UnwrappedLineParser guards against this by never calling - // \c getNextToken() after it has encountered the first eof token. - assert(!eof()); - PreviousToken = Token; - Token = PreviousTokenSource->getNextToken(); - if (eof()) - return &FakeEOF; - return Token; - } - - FormatToken *getPreviousToken() override { - return PreviousTokenSource->getPreviousToken(); - } - - FormatToken *peekNextToken(bool SkipComment) override { - if (eof()) - return &FakeEOF; - return PreviousTokenSource->peekNextToken(SkipComment); - } - - bool isEOF() override { return PreviousTokenSource->isEOF(); } - - unsigned getPosition() override { return PreviousTokenSource->getPosition(); } - - FormatToken *setPosition(unsigned Position) override { - PreviousToken = nullptr; - Token = PreviousTokenSource->setPosition(Position); - return Token; - } - -private: - bool eof() { - return Token && Token->HasUnescapedNewline && - !continuesLineComment(*Token, PreviousToken, - /*MinColumnToken=*/PreviousToken); - } - - FormatToken FakeEOF; - UnwrappedLine &Line; - FormatTokenSource *&TokenSource; - FormatToken *&ResetToken; - unsigned PreviousLineLevel; - FormatTokenSource *PreviousTokenSource; - - FormatToken *Token; - FormatToken *PreviousToken; -}; - } // end anonymous namespace class ScopedLineState { _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
