| OLD | NEW |
| 1 // Copyright 2016 the V8 project authors. All rights reserved. | 1 // Copyright 2016 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the | 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the |
| 6 // Scanner are in cctest/test-parsing.cc, rather than here. | 6 // Scanner are in cctest/test-parsing.cc, rather than here. |
| 7 | 7 |
| 8 #include "src/handles-inl.h" | 8 #include "src/handles-inl.h" |
| 9 #include "src/parsing/scanner-character-streams.h" | 9 #include "src/parsing/scanner-character-streams.h" |
| 10 #include "src/parsing/scanner.h" | 10 #include "src/parsing/scanner.h" |
| 11 #include "src/unicode-cache.h" | 11 #include "src/unicode-cache.h" |
| 12 #include "test/cctest/cctest.h" | 12 #include "test/cctest/cctest.h" |
| 13 | 13 |
| 14 using namespace v8::internal; | 14 using namespace v8::internal; |
| 15 | 15 |
| 16 namespace { | 16 namespace { |
| 17 | 17 |
| 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; | 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; |
| 19 | 19 |
| 20 std::unique_ptr<Scanner> make_scanner(const char* src) { | 20 std::unique_ptr<Scanner> make_scanner(const char* src, UnicodeCache* cache) { |
| 21 std::unique_ptr<Scanner> scanner(new Scanner(new UnicodeCache())); | 21 std::unique_ptr<Scanner> scanner(new Scanner(cache)); |
| 22 scanner->Initialize(ScannerStream::ForTesting(src).release()); | 22 scanner->Initialize(ScannerStream::ForTesting(src).release()); |
| 23 return scanner; | 23 return scanner; |
| 24 } | 24 } |
| 25 | 25 |
| 26 } // anonymous namespace | 26 } // anonymous namespace |
| 27 | 27 |
| 28 // DCHECK_TOK checks token equality, but by checking for equality of the token | 28 // DCHECK_TOK checks token equality, but by checking for equality of the token |
| 29 // names. That should have the same result, but has much nicer error messaages. | 29 // names. That should have the same result, but has much nicer error messaages. |
| 30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) | 30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) |
| 31 | 31 |
| 32 TEST(Bookmarks) { | 32 TEST(Bookmarks) { |
| 33 UnicodeCache unicode_cache; |
| 34 |
| 33 // Scan through the given source and record the tokens for use as reference | 35 // Scan through the given source and record the tokens for use as reference |
| 34 // below. | 36 // below. |
| 35 std::vector<Token::Value> tokens; | 37 std::vector<Token::Value> tokens; |
| 36 { | 38 { |
| 37 auto scanner = make_scanner(src_simple); | 39 auto scanner = make_scanner(src_simple, &unicode_cache); |
| 38 do { | 40 do { |
| 39 tokens.push_back(scanner->Next()); | 41 tokens.push_back(scanner->Next()); |
| 40 } while (scanner->current_token() != Token::EOS); | 42 } while (scanner->current_token() != Token::EOS); |
| 41 } | 43 } |
| 42 | 44 |
| 43 // For each position: | 45 // For each position: |
| 44 // - Scan through file, | 46 // - Scan through file, |
| 45 // - set a bookmark once the position is reached, | 47 // - set a bookmark once the position is reached, |
| 46 // - scan a bit more, | 48 // - scan a bit more, |
| 47 // - reset to the bookmark, and | 49 // - reset to the bookmark, and |
| 48 // - scan until the end. | 50 // - scan until the end. |
| 49 // At each step, compare to the reference token sequence generated above. | 51 // At each step, compare to the reference token sequence generated above. |
| 50 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { | 52 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { |
| 51 auto scanner = make_scanner(src_simple); | 53 auto scanner = make_scanner(src_simple, &unicode_cache); |
| 52 Scanner::BookmarkScope bookmark(scanner.get()); | 54 Scanner::BookmarkScope bookmark(scanner.get()); |
| 53 | 55 |
| 54 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { | 56 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { |
| 55 if (i == bookmark_pos) { | 57 if (i == bookmark_pos) { |
| 56 bookmark.Set(); | 58 bookmark.Set(); |
| 57 } | 59 } |
| 58 DCHECK_TOK(tokens[i], scanner->Next()); | 60 DCHECK_TOK(tokens[i], scanner->Next()); |
| 59 } | 61 } |
| 60 | 62 |
| 61 bookmark.Apply(); | 63 bookmark.Apply(); |
| 62 for (size_t i = bookmark_pos; i < tokens.size(); i++) { | 64 for (size_t i = bookmark_pos; i < tokens.size(); i++) { |
| 63 DCHECK_TOK(tokens[i], scanner->Next()); | 65 DCHECK_TOK(tokens[i], scanner->Next()); |
| 64 } | 66 } |
| 65 } | 67 } |
| 66 } | 68 } |
| 67 | 69 |
| 68 TEST(AllThePushbacks) { | 70 TEST(AllThePushbacks) { |
| 69 const struct { | 71 const struct { |
| 70 const char* src; | 72 const char* src; |
| 71 const Token::Value tokens[5]; // Large enough for any of the test cases. | 73 const Token::Value tokens[5]; // Large enough for any of the test cases. |
| 72 } test_cases[] = { | 74 } test_cases[] = { |
| 73 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 75 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
| 74 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, | 76 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, |
| 75 {"<!-x", | 77 {"<!-x", |
| 76 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 78 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
| 77 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, | 79 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, |
| 78 }; | 80 }; |
| 79 | 81 |
| 82 UnicodeCache unicode_cache; |
| 80 for (const auto& test_case : test_cases) { | 83 for (const auto& test_case : test_cases) { |
| 81 auto scanner = make_scanner(test_case.src); | 84 auto scanner = make_scanner(test_case.src, &unicode_cache); |
| 82 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { | 85 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { |
| 83 DCHECK_TOK(test_case.tokens[i], scanner->Next()); | 86 DCHECK_TOK(test_case.tokens[i], scanner->Next()); |
| 84 } | 87 } |
| 85 DCHECK_TOK(Token::EOS, scanner->Next()); | 88 DCHECK_TOK(Token::EOS, scanner->Next()); |
| 86 } | 89 } |
| 87 } | 90 } |
| OLD | NEW |