OLD | NEW |
---|---|
1 // Copyright 2016 the V8 project authors. All rights reserved. | 1 // Copyright 2016 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Tests v8::internal::Scanner. Note that presently most unit tests for the | 5 // Tests v8::internal::Scanner. Note that presently most unit tests for the |
6 // Scanner are in cctest/test-parsing.cc, rather than here. | 6 // Scanner are in cctest/test-parsing.cc, rather than here. |
7 | 7 |
8 #include "src/handles-inl.h" | 8 #include "src/handles-inl.h" |
9 #include "src/parsing/scanner-character-streams.h" | 9 #include "src/parsing/scanner-character-streams.h" |
10 #include "src/parsing/scanner.h" | 10 #include "src/parsing/scanner.h" |
11 #include "src/unicode-cache.h" | 11 #include "src/unicode-cache.h" |
12 #include "test/cctest/cctest.h" | 12 #include "test/cctest/cctest.h" |
13 | 13 |
14 using namespace v8::internal; | 14 using namespace v8::internal; |
15 | 15 |
16 namespace { | 16 namespace { |
17 | 17 |
18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; | 18 const char src_simple[] = "function foo() { var x = 2 * a() + b; }"; |
19 | 19 |
20 std::unique_ptr<Scanner> make_scanner(const char* src, UnicodeCache* cache) { | 20 struct ScannerTestHelper { |
vogelheim
2016/11/23 18:08:42
This (& below) is an awful lot of boilerplate, jus
| |
21 std::unique_ptr<Scanner> scanner(new Scanner(cache)); | 21 ScannerTestHelper() = default; |
22 scanner->Initialize(ScannerStream::ForTesting(src).release()); | 22 ScannerTestHelper(ScannerTestHelper&& other) |
23 return scanner; | 23 : unicode_cache(std::move(other.unicode_cache)), |
24 stream(std::move(other.stream)), | |
25 scanner(std::move(other.scanner)) {} | |
26 | |
27 std::unique_ptr<UnicodeCache> unicode_cache; | |
28 std::unique_ptr<Utf16CharacterStream> stream; | |
29 std::unique_ptr<Scanner> scanner; | |
30 | |
31 Scanner* operator->() const { return scanner.get(); } | |
32 Scanner* get() const { return scanner.get(); } | |
33 }; | |
34 | |
35 ScannerTestHelper make_scanner(const char* src) { | |
36 ScannerTestHelper helper; | |
37 helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache); | |
38 helper.stream = ScannerStream::ForTesting(src); | |
39 helper.scanner = | |
40 std::unique_ptr<Scanner>(new Scanner(helper.unicode_cache.get())); | |
41 helper.scanner->Initialize(helper.stream.get()); | |
42 return helper; | |
24 } | 43 } |
25 | 44 |
26 } // anonymous namespace | 45 } // anonymous namespace |
27 | 46 |
28 // DCHECK_TOK checks token equality, but by checking for equality of the token | 47 // DCHECK_TOK checks token equality, but by checking for equality of the token |
29 // names. That should have the same result, but has much nicer error messaages. | 48 // names. That should have the same result, but has much nicer error messaages. |
30 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) | 49 #define DCHECK_TOK(a, b) DCHECK_EQ(Token::Name(a), Token::Name(b)) |
31 | 50 |
32 TEST(Bookmarks) { | 51 TEST(Bookmarks) { |
33 UnicodeCache unicode_cache; | |
34 | |
35 // Scan through the given source and record the tokens for use as reference | 52 // Scan through the given source and record the tokens for use as reference |
36 // below. | 53 // below. |
37 std::vector<Token::Value> tokens; | 54 std::vector<Token::Value> tokens; |
38 { | 55 { |
39 auto scanner = make_scanner(src_simple, &unicode_cache); | 56 auto scanner = make_scanner(src_simple); |
40 do { | 57 do { |
41 tokens.push_back(scanner->Next()); | 58 tokens.push_back(scanner->Next()); |
42 } while (scanner->current_token() != Token::EOS); | 59 } while (scanner->current_token() != Token::EOS); |
43 } | 60 } |
44 | 61 |
45 // For each position: | 62 // For each position: |
46 // - Scan through file, | 63 // - Scan through file, |
47 // - set a bookmark once the position is reached, | 64 // - set a bookmark once the position is reached, |
48 // - scan a bit more, | 65 // - scan a bit more, |
49 // - reset to the bookmark, and | 66 // - reset to the bookmark, and |
50 // - scan until the end. | 67 // - scan until the end. |
51 // At each step, compare to the reference token sequence generated above. | 68 // At each step, compare to the reference token sequence generated above. |
52 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { | 69 for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) { |
53 auto scanner = make_scanner(src_simple, &unicode_cache); | 70 auto scanner = make_scanner(src_simple); |
54 Scanner::BookmarkScope bookmark(scanner.get()); | 71 Scanner::BookmarkScope bookmark(scanner.get()); |
55 | 72 |
56 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { | 73 for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) { |
57 if (i == bookmark_pos) { | 74 if (i == bookmark_pos) { |
58 bookmark.Set(); | 75 bookmark.Set(); |
59 } | 76 } |
60 DCHECK_TOK(tokens[i], scanner->Next()); | 77 DCHECK_TOK(tokens[i], scanner->Next()); |
61 } | 78 } |
62 | 79 |
63 bookmark.Apply(); | 80 bookmark.Apply(); |
64 for (size_t i = bookmark_pos; i < tokens.size(); i++) { | 81 for (size_t i = bookmark_pos; i < tokens.size(); i++) { |
65 DCHECK_TOK(tokens[i], scanner->Next()); | 82 DCHECK_TOK(tokens[i], scanner->Next()); |
66 } | 83 } |
67 } | 84 } |
68 } | 85 } |
69 | 86 |
70 TEST(AllThePushbacks) { | 87 TEST(AllThePushbacks) { |
71 const struct { | 88 const struct { |
72 const char* src; | 89 const char* src; |
73 const Token::Value tokens[5]; // Large enough for any of the test cases. | 90 const Token::Value tokens[5]; // Large enough for any of the test cases. |
74 } test_cases[] = { | 91 } test_cases[] = { |
75 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 92 {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
76 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, | 93 {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}}, |
77 {"<!-x", | 94 {"<!-x", |
78 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, | 95 {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}}, |
79 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, | 96 {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}}, |
80 }; | 97 }; |
81 | 98 |
82 UnicodeCache unicode_cache; | |
83 for (const auto& test_case : test_cases) { | 99 for (const auto& test_case : test_cases) { |
84 auto scanner = make_scanner(test_case.src, &unicode_cache); | 100 auto scanner = make_scanner(test_case.src); |
85 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { | 101 for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) { |
86 DCHECK_TOK(test_case.tokens[i], scanner->Next()); | 102 DCHECK_TOK(test_case.tokens[i], scanner->Next()); |
87 } | 103 } |
88 DCHECK_TOK(Token::EOS, scanner->Next()); | 104 DCHECK_TOK(Token::EOS, scanner->Next()); |
89 } | 105 } |
90 } | 106 } |
OLD | NEW |