OLD | NEW |
1 // Copyright 2016 the V8 project authors. All rights reserved. | 1 // Copyright 2016 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Tests v8::internal::Scanner. Note that presently most unit tests for the | 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the |
6 // Scanner are in cctest/test-parsing.cc, rather than here. | 6 // Scanner are in cctest/test-parsing.cc, rather than here. |
7 | 7 |
8 #include "src/handles-inl.h" | 8 #include "src/handles-inl.h" |
9 #include "src/parsing/scanner-character-streams.h" | 9 #include "src/parsing/scanner-character-streams.h" |
10 #include "src/parsing/scanner.h" | 10 #include "src/parsing/scanner.h" |
11 #include "src/unicode-cache.h" | 11 #include "src/unicode-cache.h" |
12 #include "test/cctest/cctest.h" | 12 #include "test/cctest/cctest.h" |
13 | 13 |
14 using namespace v8::internal; | 14 using namespace v8::internal; |
15 | 15 |
16 namespace { | 16 namespace { |
17 | 17 |
18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; | 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; |
19 | 19 |
20 std::unique_ptr<Scanner> make_scanner(const char* src) { | 20 std::unique_ptr<Scanner> make_scanner(const char* src, UnicodeCache* cache) { |
21 std::unique_ptr<Scanner> scanner(new Scanner(new UnicodeCache())); | 21 std::unique_ptr<Scanner> scanner(new Scanner(cache)); |
22 scanner->Initialize(ScannerStream::ForTesting(src).release()); | 22 scanner->Initialize(ScannerStream::ForTesting(src).release()); |
23 return scanner; | 23 return scanner; |
24 } | 24 } |
25 | 25 |
26 } // anonymous namespace | 26 } // anonymous namespace |
27 | 27 |
28 // DCHECK_TOK checks token equality, but by checking for equality of the token | 28 // DCHECK_TOK checks token equality, but by checking for equality of the token |
29 // names. That should have the same result, but has much nicer error messaages. | 29 // names. That should have the same result, but has much nicer error messaages. |
30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) | 30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) |
31 | 31 |
32 TEST(Bookmarks) { | 32 TEST(Bookmarks) { |
| 33 UnicodeCache unicode_cache; |
| 34 |
33 // Scan through the given source and record the tokens for use as reference | 35 // Scan through the given source and record the tokens for use as reference |
34 // below. | 36 // below. |
35 std::vector<Token::Value> tokens; | 37 std::vector<Token::Value> tokens; |
36 { | 38 { |
37 auto scanner = make_scanner(src_simple); | 39 auto scanner = make_scanner(src_simple, &unicode_cache); |
38 do { | 40 do { |
39 tokens.push_back(scanner->Next()); | 41 tokens.push_back(scanner->Next()); |
40 } while (scanner->current_token() != Token::EOS); | 42 } while (scanner->current_token() != Token::EOS); |
41 } | 43 } |
42 | 44 |
43 // For each position: | 45 // For each position: |
44 // - Scan through file, | 46 // - Scan through file, |
45 // - set a bookmark once the position is reached, | 47 // - set a bookmark once the position is reached, |
46 // - scan a bit more, | 48 // - scan a bit more, |
47 // - reset to the bookmark, and | 49 // - reset to the bookmark, and |
48 // - scan until the end. | 50 // - scan until the end. |
49 // At each step, compare to the reference token sequence generated above. | 51 // At each step, compare to the reference token sequence generated above. |
50 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { | 52 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { |
51 auto scanner = make_scanner(src_simple); | 53 auto scanner = make_scanner(src_simple, &unicode_cache); |
52 Scanner::BookmarkScope bookmark(scanner.get()); | 54 Scanner::BookmarkScope bookmark(scanner.get()); |
53 | 55 |
54 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { | 56 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { |
55 if (i == bookmark_pos) { | 57 if (i == bookmark_pos) { |
56 bookmark.Set(); | 58 bookmark.Set(); |
57 } | 59 } |
58 DCHECK_TOK(tokens[i], scanner->Next()); | 60 DCHECK_TOK(tokens[i], scanner->Next()); |
59 } | 61 } |
60 | 62 |
61 bookmark.Apply(); | 63 bookmark.Apply(); |
62 for (size_t i = bookmark_pos; i < tokens.size(); i++) { | 64 for (size_t i = bookmark_pos; i < tokens.size(); i++) { |
63 DCHECK_TOK(tokens[i], scanner->Next()); | 65 DCHECK_TOK(tokens[i], scanner->Next()); |
64 } | 66 } |
65 } | 67 } |
66 } | 68 } |
67 | 69 |
68 TEST(AllThePushbacks) { | 70 TEST(AllThePushbacks) { |
69 const struct { | 71 const struct { |
70 const char* src; | 72 const char* src; |
71 const Token::Value tokens[5]; // Large enough for any of the test cases. | 73 const Token::Value tokens[5]; // Large enough for any of the test cases. |
72 } test_cases[] = { | 74 } test_cases[] = { |
73 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 75 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
74 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, | 76 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, |
75 {"<!-x", | 77 {"<!-x", |
76 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 78 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
77 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, | 79 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, |
78 }; | 80 }; |
79 | 81 |
| 82 UnicodeCache unicode_cache; |
80 for (const auto& test_case : test_cases) { | 83 for (const auto& test_case : test_cases) { |
81 auto scanner = make_scanner(test_case.src); | 84 auto scanner = make_scanner(test_case.src, &unicode_cache); |
82 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { | 85 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { |
83 DCHECK_TOK(test_case.tokens[i], scanner->Next()); | 86 DCHECK_TOK(test_case.tokens[i], scanner->Next()); |
84 } | 87 } |
85 DCHECK_TOK(Token::EOS, scanner->Next()); | 88 DCHECK_TOK(Token::EOS, scanner->Next()); |
86 } | 89 } |
87 } | 90 } |
OLD | NEW |