1 //===- unittest/Format/FormatTokenSourceTest.cpp --------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "../../lib/Format/FormatTokenSource.h" 10 #include "TestLexer.h" 11 #include "clang/Basic/TokenKinds.h" 12 #include "gtest/gtest.h" 13 14 namespace clang { 15 namespace format { 16 namespace { 17 18 class IndexedTokenSourceTest : public ::testing::Test { 19 protected: 20 TokenList lex(llvm::StringRef Code, 21 const FormatStyle &Style = getLLVMStyle()) { 22 return TestLexer(Allocator, Buffers, Style).lex(Code); 23 } 24 llvm::SpecificBumpPtrAllocator<FormatToken> Allocator; 25 std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers; 26 }; 27 28 #define EXPECT_TOKEN_KIND(FormatTok, Kind) \ 29 do { \ 30 FormatToken *Tok = FormatTok; \ 31 EXPECT_EQ(Tok->Tok.getKind(), Kind) << *Tok; \ 32 } while (false); 33 #define EXPECT_TOKEN_ID(FormatTok, Name) \ 34 do { \ 35 FormatToken *Tok = FormatTok; \ 36 EXPECT_EQ(Tok->Tok.getKind(), tok::identifier) << *Tok; \ 37 EXPECT_EQ(Tok->TokenText, Name) << *Tok; \ 38 } while (false); 39 40 TEST_F(IndexedTokenSourceTest, EmptyInput) { 41 IndexedTokenSource Source(lex("")); 42 EXPECT_FALSE(Source.isEOF()); 43 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 44 EXPECT_TRUE(Source.isEOF()); 45 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 46 EXPECT_TRUE(Source.isEOF()); 47 EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/false), tok::eof); 48 EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/true), tok::eof); 49 EXPECT_EQ(Source.getPreviousToken(), nullptr); 50 EXPECT_TRUE(Source.isEOF()); 51 } 52 53 TEST_F(IndexedTokenSourceTest, NavigateTokenStream) { 54 IndexedTokenSource Source(lex("int a;")); 55 EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::kw_int); 56 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::kw_int); 57 EXPECT_EQ(Source.getPreviousToken(), nullptr); 58 EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::identifier); 59 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::identifier); 60 EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::kw_int); 61 EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::semi); 62 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::semi); 63 EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::identifier); 64 EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::eof); 65 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 66 EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi); 67 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 68 EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi); 69 } 70 71 TEST_F(IndexedTokenSourceTest, ResetPosition) { 72 IndexedTokenSource Source(lex("int a;")); 73 Source.getNextToken(); 74 unsigned Position = Source.getPosition(); 75 Source.getNextToken(); 76 Source.getNextToken(); 77 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 78 EXPECT_TOKEN_KIND(Source.setPosition(Position), tok::kw_int); 79 } 80 81 TEST_F(IndexedTokenSourceTest, InsertTokens) { 82 IndexedTokenSource Source(lex("A1 A2")); 83 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 84 EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1"); 85 EXPECT_TOKEN_ID(Source.getNextToken(), "B2"); 86 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 87 EXPECT_TOKEN_ID(Source.getNextToken(), "A2"); 88 } 89 90 TEST_F(IndexedTokenSourceTest, InsertTokensAtEOF) { 91 IndexedTokenSource Source(lex("A1")); 92 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 93 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 94 EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1"); 95 EXPECT_TOKEN_ID(Source.getNextToken(), "B2"); 96 EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof); 97 } 98 99 TEST_F(IndexedTokenSourceTest, InsertTokensRecursive) { 100 IndexedTokenSource Source(lex("A1")); 101 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 102 // A1 103 EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1"); 104 // B1 A1 105 EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1"); 106 // C1 B1 A1 107 EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1"); 108 // D1 C1 B1 A1 109 EXPECT_TOKEN_ID(Source.getNextToken(), "C1"); 110 EXPECT_TOKEN_ID(Source.getNextToken(), "B1"); 111 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 112 } 113 114 TEST_F(IndexedTokenSourceTest, InsertTokensRecursiveAtEndOfSequence) { 115 IndexedTokenSource Source(lex("A1")); 116 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 117 EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1"); 118 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 119 EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1"); 120 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 121 EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1"); 122 EXPECT_TOKEN_ID(Source.getNextToken(), "A1"); 123 } 124 125 } // namespace 126 } // namespace format 127 } // namespace clang 128