Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright 2016 the V8 project authors. All rights reserved. | 1 // Copyright 2016 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the | 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the |
| 6 // Scanner are in cctest/test-parsing.cc, rather than here. | 6 // Scanner are in cctest/test-parsing.cc, rather than here. |
| 7 | 7 |
| 8 #include "src/handles-inl.h" | 8 #include "src/handles-inl.h" |
| 9 #include "src/parsing/scanner-character-streams.h" | 9 #include "src/parsing/scanner-character-streams.h" |
| 10 #include "src/parsing/scanner.h" | 10 #include "src/parsing/scanner.h" |
| 11 #include "src/unicode-cache.h" | 11 #include "src/unicode-cache.h" |
| 12 #include "test/cctest/cctest.h" | 12 #include "test/cctest/cctest.h" |
| 13 | 13 |
| 14 using namespace v8::internal; | 14 using namespace v8::internal; |
| 15 | 15 |
| 16 namespace { | 16 namespace { |
| 17 | 17 |
| 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; | 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; |
| 19 | 19 |
| 20 std::unique_ptr<Scanner> make_scanner(const char* src, UnicodeCache* cache) { | 20 struct ScannerTestHelper { |
|
vogelheim
2016/11/23 18:08:42
This (& below) is an awful lot of boilerplate, jus
| |
| 21 std::unique_ptr<Scanner> scanner(new Scanner(cache)); | 21 ScannerTestHelper() = default; |
| 22 scanner->Initialize(ScannerStream::ForTesting(src).release()); | 22 ScannerTestHelper(ScannerTestHelper&& other) |
| 23 return scanner; | 23 : unicode_cache(std::move(other.unicode_cache)), |
| 24 stream(std::move(other.stream)), | |
| 25 scanner(std::move(other.scanner)) {} | |
| 26 | |
| 27 std::unique_ptr<UnicodeCache> unicode_cache; | |
| 28 std::unique_ptr<Utf16CharacterStream> stream; | |
| 29 std::unique_ptr<Scanner> scanner; | |
| 30 | |
| 31 Scanner* operator->() const { return scanner.get(); } | |
| 32 Scanner* get() const { return scanner.get(); } | |
| 33 }; | |
| 34 | |
| 35 ScannerTestHelper make_scanner(const char* src) { | |
| 36 ScannerTestHelper helper; | |
| 37 helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache); | |
| 38 helper.stream = ScannerStream::ForTesting(src); | |
| 39 helper.scanner = | |
| 40 std::unique_ptr<Scanner>(new Scanner(helper.unicode_cache.get())); | |
| 41 helper.scanner->Initialize(helper.stream.get()); | |
| 42 return helper; | |
| 24 } | 43 } |
| 25 | 44 |
| 26 } // anonymous namespace | 45 } // anonymous namespace |
| 27 | 46 |
| 28 // DCHECK_TOK checks token equality, but by checking for equality of the token | 47 // DCHECK_TOK checks token equality, but by checking for equality of the token |
| 29 // names. That should have the same result, but has much nicer error messaages. | 48 // names. That should have the same result, but has much nicer error messaages. |
| 30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) | 49 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) |
| 31 | 50 |
| 32 TEST(Bookmarks) { | 51 TEST(Bookmarks) { |
| 33 UnicodeCache unicode_cache; | |
| 34 | |
| 35 // Scan through the given source and record the tokens for use as reference | 52 // Scan through the given source and record the tokens for use as reference |
| 36 // below. | 53 // below. |
| 37 std::vector<Token::Value> tokens; | 54 std::vector<Token::Value> tokens; |
| 38 { | 55 { |
| 39 auto scanner = make_scanner(src_simple, &unicode_cache); | 56 auto scanner = make_scanner(src_simple); |
| 40 do { | 57 do { |
| 41 tokens.push_back(scanner->Next()); | 58 tokens.push_back(scanner->Next()); |
| 42 } while (scanner->current_token() != Token::EOS); | 59 } while (scanner->current_token() != Token::EOS); |
| 43 } | 60 } |
| 44 | 61 |
| 45 // For each position: | 62 // For each position: |
| 46 // - Scan through file, | 63 // - Scan through file, |
| 47 // - set a bookmark once the position is reached, | 64 // - set a bookmark once the position is reached, |
| 48 // - scan a bit more, | 65 // - scan a bit more, |
| 49 // - reset to the bookmark, and | 66 // - reset to the bookmark, and |
| 50 // - scan until the end. | 67 // - scan until the end. |
| 51 // At each step, compare to the reference token sequence generated above. | 68 // At each step, compare to the reference token sequence generated above. |
| 52 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { | 69 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { |
| 53 auto scanner = make_scanner(src_simple, &unicode_cache); | 70 auto scanner = make_scanner(src_simple); |
| 54 Scanner::BookmarkScope bookmark(scanner.get()); | 71 Scanner::BookmarkScope bookmark(scanner.get()); |
| 55 | 72 |
| 56 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { | 73 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { |
| 57 if (i == bookmark_pos) { | 74 if (i == bookmark_pos) { |
| 58 bookmark.Set(); | 75 bookmark.Set(); |
| 59 } | 76 } |
| 60 DCHECK_TOK(tokens[i], scanner->Next()); | 77 DCHECK_TOK(tokens[i], scanner->Next()); |
| 61 } | 78 } |
| 62 | 79 |
| 63 bookmark.Apply(); | 80 bookmark.Apply(); |
| 64 for (size_t i = bookmark_pos; i < tokens.size(); i++) { | 81 for (size_t i = bookmark_pos; i < tokens.size(); i++) { |
| 65 DCHECK_TOK(tokens[i], scanner->Next()); | 82 DCHECK_TOK(tokens[i], scanner->Next()); |
| 66 } | 83 } |
| 67 } | 84 } |
| 68 } | 85 } |
| 69 | 86 |
| 70 TEST(AllThePushbacks) { | 87 TEST(AllThePushbacks) { |
| 71 const struct { | 88 const struct { |
| 72 const char* src; | 89 const char* src; |
| 73 const Token::Value tokens[5]; // Large enough for any of the test cases. | 90 const Token::Value tokens[5]; // Large enough for any of the test cases. |
| 74 } test_cases[] = { | 91 } test_cases[] = { |
| 75 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 92 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
| 76 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, | 93 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, |
| 77 {"<!-x", | 94 {"<!-x", |
| 78 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 95 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
| 79 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, | 96 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, |
| 80 }; | 97 }; |
| 81 | 98 |
| 82 UnicodeCache unicode_cache; | |
| 83 for (const auto& test_case : test_cases) { | 99 for (const auto& test_case : test_cases) { |
| 84 auto scanner = make_scanner(test_case.src, &unicode_cache); | 100 auto scanner = make_scanner(test_case.src); |
| 85 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { | 101 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { |
| 86 DCHECK_TOK(test_case.tokens[i], scanner->Next()); | 102 DCHECK_TOK(test_case.tokens[i], scanner->Next()); |
| 87 } | 103 } |
| 88 DCHECK_TOK(Token::EOS, scanner->Next()); | 104 DCHECK_TOK(Token::EOS, scanner->Next()); |
| 89 } | 105 } |
| 90 } | 106 } |
| OLD | NEW |