| //===--- FormatTokenSource.h - Format C++ code ------------------*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| /// | |
| /// \file | |
| /// This file defines the \c FormatTokenSource interface, which provides a token | |
| /// stream as well as the ability to manipulate the token stream. | |
| /// | |
| //===----------------------------------------------------------------------===// | |
| #ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H | |
| #define LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H | |
| #include "UnwrappedLineParser.h" | |
| #define DEBUG_TYPE "format-token-source" | |
| namespace clang { | |
| namespace format { | |
| // Navigate a token stream. | |
| // | |
| // Enables traversal of a token stream, resetting the position in a token | |
| // stream, as well as inserting new tokens. | |
| class FormatTokenSource { | |
| public: | |
| virtual ~FormatTokenSource() {} | |
| // Returns the next token in the token stream. | |
| virtual FormatToken *getNextToken() = 0; | |
| // Returns the token preceding the token returned by the last call to | |
| // getNextToken() in the token stream, or nullptr if no such token exists. | |
| // | |
| // Must not be called directly at the position directly after insertTokens() | |
| // is called. | |
| virtual FormatToken *getPreviousToken() = 0; | |
| // Returns the token that would be returned by the next call to | |
| // getNextToken(). | |
| virtual FormatToken *peekNextToken(bool SkipComment = false) = 0; | |
| // Returns whether we are at the end of the file. | |
| // This can be different from whether getNextToken() returned an eof token | |
| // when the FormatTokenSource is a view on a part of the token stream. | |
| virtual bool isEOF() = 0; | |
| // Gets the current position in the token stream, to be used by setPosition(). | |
| // | |
| // Note that the value of the position is not meaningful, and specifically | |
| // should not be used to get relative token positions. | |
| virtual unsigned getPosition() = 0; | |
| // Resets the token stream to the state it was in when getPosition() returned | |
| // Position, and return the token at that position in the stream. | |
| virtual FormatToken *setPosition(unsigned Position) = 0; | |
| // Insert the given tokens before the current position. | |
| // Returns the first token in \c Tokens. | |
| // The next returned token will be the second token in \c Tokens. | |
| // Requires the last token in Tokens to be EOF; once the EOF token is reached, | |
| // the next token will be the last token returned by getNextToken(); | |
| // | |
| // For example, given the token sequence 'a1 a2': | |
| // getNextToken() -> a1 | |
| // insertTokens('b1 b2') -> b1 | |
| // getNextToken() -> b2 | |
| // getNextToken() -> a1 | |
| // getNextToken() -> a2 | |
| virtual FormatToken *insertTokens(ArrayRef<FormatToken *> Tokens) = 0; | |
| [[nodiscard]] FormatToken *getNextNonComment() { | |
| FormatToken *Tok; | |
| do { | |
| Tok = getNextToken(); | |
| assert(Tok); | |
| } while (Tok->is(tok::comment)); | |
| return Tok; | |
| } | |
| }; | |
| class IndexedTokenSource : public FormatTokenSource { | |
| public: | |
| IndexedTokenSource(ArrayRef<FormatToken *> Tokens) | |
| : Tokens(Tokens), Position(-1) {} | |
| FormatToken *getNextToken() override { | |
| if (Position >= 0 && isEOF()) { | |
| LLVM_DEBUG({ | |
| llvm::dbgs() << "Next "; | |
| dbgToken(Position); | |
| }); | |
| return Tokens[Position]; | |
| } | |
| Position = successor(Position); | |
| LLVM_DEBUG({ | |
| llvm::dbgs() << "Next "; | |
| dbgToken(Position); | |
| }); | |
| return Tokens[Position]; | |
| } | |
| FormatToken *getPreviousToken() override { | |
| assert(Position <= 0 || Tokens[Position - 1]->isNot(tok::eof)); | |
| return Position > 0 ? Tokens[Position - 1] : nullptr; | |
| } | |
| FormatToken *peekNextToken(bool SkipComment = false) override { | |
| if (isEOF()) | |
| return Tokens[Position]; | |
| int Next = successor(Position); | |
| if (SkipComment) | |
| while (Tokens[Next]->is(tok::comment)) | |
| Next = successor(Next); | |
| LLVM_DEBUG({ | |
| llvm::dbgs() << "Peeking "; | |
| dbgToken(Next); | |
| }); | |
| return Tokens[Next]; | |
| } | |
| bool isEOF() override { | |
| return Position == -1 ? false : Tokens[Position]->is(tok::eof); | |
| } | |
| unsigned getPosition() override { | |
| LLVM_DEBUG(llvm::dbgs() << "Getting Position: " << Position << "\n"); | |
| assert(Position >= 0); | |
| return Position; | |
| } | |
| FormatToken *setPosition(unsigned P) override { | |
| LLVM_DEBUG(llvm::dbgs() << "Setting Position: " << P << "\n"); | |
| Position = P; | |
| return Tokens[Position]; | |
| } | |
| FormatToken *insertTokens(ArrayRef<FormatToken *> New) override { | |
| assert(Position != -1); | |
| assert((*New.rbegin())->Tok.is(tok::eof)); | |
| int Next = Tokens.size(); | |
| Tokens.append(New.begin(), New.end()); | |
| LLVM_DEBUG({ | |
| llvm::dbgs() << "Inserting:\n"; | |
| for (int I = Next, E = Tokens.size(); I != E; ++I) | |
| dbgToken(I, " "); | |
| llvm::dbgs() << " Jump from: " << (Tokens.size() - 1) << " -> " | |
| << Position << "\n"; | |
| }); | |
| Jumps[Tokens.size() - 1] = Position; | |
| Position = Next; | |
| LLVM_DEBUG({ | |
| llvm::dbgs() << "At inserted token "; | |
| dbgToken(Position); | |
| }); | |
| return Tokens[Position]; | |
| } | |
| void reset() { Position = -1; } | |
| private: | |
| int successor(int Current) const { | |
| int Next = Current + 1; | |
| auto it = Jumps.find(Next); | |
| if (it != Jumps.end()) { | |
| Next = it->second; | |
| assert(!Jumps.contains(Next)); | |
| } | |
| return Next; | |
| } | |
| void dbgToken(int Position, StringRef Indent = "") { | |
| FormatToken *Tok = Tokens[Position]; | |
| llvm::dbgs() << Indent << "[" << Position | |
| << "] Token: " << Tok->Tok.getName() << " / " << Tok->TokenText | |
| << ", Macro: " << !!Tok->MacroCtx << "\n"; | |
| } | |
| SmallVector<FormatToken *> Tokens; | |
| int Position; | |
| // Maps from position a to position b, so that when we reach a, the token | |
| // stream continues at position b instead. | |
| llvm::DenseMap<int, int> Jumps; | |
| }; | |
| class ScopedMacroState : public FormatTokenSource { | |
| public: | |
| ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource, | |
| FormatToken *&ResetToken) | |
| : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken), | |
| PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource), | |
| Token(nullptr), PreviousToken(nullptr) { | |
| FakeEOF.Tok = Token::createEof(); | |
| TokenSource = this; | |
| Line.Level = 0; | |
| Line.InPPDirective = true; | |
| // InMacroBody gets set after the `#define x` part. | |
| } | |
| ~ScopedMacroState() override { | |
| TokenSource = PreviousTokenSource; | |
| ResetToken = Token; | |
| Line.InPPDirective = false; | |
| Line.InMacroBody = false; | |
| Line.Level = PreviousLineLevel; | |
| } | |
| FormatToken *getNextToken() override { | |
| // The \c UnwrappedLineParser guards against this by never calling | |
| // \c getNextToken() after it has encountered the first eof token. | |
| assert(!eof()); | |
| PreviousToken = Token; | |
| Token = PreviousTokenSource->getNextToken(); | |
| if (eof()) | |
| return &FakeEOF; | |
| return Token; | |
| } | |
| FormatToken *getPreviousToken() override { | |
| return PreviousTokenSource->getPreviousToken(); | |
| } | |
| FormatToken *peekNextToken(bool SkipComment) override { | |
| if (eof()) | |
| return &FakeEOF; | |
| return PreviousTokenSource->peekNextToken(SkipComment); | |
| } | |
| bool isEOF() override { return PreviousTokenSource->isEOF(); } | |
| unsigned getPosition() override { return PreviousTokenSource->getPosition(); } | |
| FormatToken *setPosition(unsigned Position) override { | |
| PreviousToken = nullptr; | |
| Token = PreviousTokenSource->setPosition(Position); | |
| return Token; | |
| } | |
| FormatToken *insertTokens(ArrayRef<FormatToken *> Tokens) override { | |
| llvm_unreachable("Cannot insert tokens while parsing a macro."); | |
| return nullptr; | |
| } | |
| private: | |
| bool eof() { | |
| return Token && Token->HasUnescapedNewline && | |
| !continuesLineComment(*Token, PreviousToken, | |
| /*MinColumnToken=*/PreviousToken); | |
| } | |
| FormatToken FakeEOF; | |
| UnwrappedLine &Line; | |
| FormatTokenSource *&TokenSource; | |
| FormatToken *&ResetToken; | |
| unsigned PreviousLineLevel; | |
| FormatTokenSource *PreviousTokenSource; | |
| FormatToken *Token; | |
| FormatToken *PreviousToken; | |
| }; | |
| } // namespace format | |
| } // namespace clang | |
| #undef DEBUG_TYPE | |
| #endif |