OLD | NEW |
1 // Copyright 2012 the V8 project authors. All rights reserved. | 1 // Copyright 2012 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
64 }; | 64 }; |
65 | 65 |
66 KeywordToken key_token; | 66 KeywordToken key_token; |
67 i::UnicodeCache unicode_cache; | 67 i::UnicodeCache unicode_cache; |
68 char buffer[32]; | 68 char buffer[32]; |
69 for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) { | 69 for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) { |
70 const char* keyword = key_token.keyword; | 70 const char* keyword = key_token.keyword; |
71 size_t length = strlen(key_token.keyword); | 71 size_t length = strlen(key_token.keyword); |
72 CHECK(static_cast<int>(sizeof(buffer)) >= length); | 72 CHECK(static_cast<int>(sizeof(buffer)) >= length); |
73 { | 73 { |
74 i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length); | 74 auto stream = i::ScannerStream::ForTesting(keyword, length); |
75 i::Scanner scanner(&unicode_cache); | 75 i::Scanner scanner(&unicode_cache); |
76 scanner.Initialize(&stream); | 76 scanner.Initialize(stream.get()); |
77 CHECK_EQ(key_token.token, scanner.Next()); | 77 CHECK_EQ(key_token.token, scanner.Next()); |
78 CHECK_EQ(i::Token::EOS, scanner.Next()); | 78 CHECK_EQ(i::Token::EOS, scanner.Next()); |
79 } | 79 } |
80 // Removing characters will make keyword matching fail. | 80 // Removing characters will make keyword matching fail. |
81 { | 81 { |
82 i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length - 1); | 82 auto stream = i::ScannerStream::ForTesting(keyword, length - 1); |
83 i::Scanner scanner(&unicode_cache); | 83 i::Scanner scanner(&unicode_cache); |
84 scanner.Initialize(&stream); | 84 scanner.Initialize(stream.get()); |
85 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); | 85 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); |
86 CHECK_EQ(i::Token::EOS, scanner.Next()); | 86 CHECK_EQ(i::Token::EOS, scanner.Next()); |
87 } | 87 } |
88 // Adding characters will make keyword matching fail. | 88 // Adding characters will make keyword matching fail. |
89 static const char chars_to_append[] = { 'z', '0', '_' }; | 89 static const char chars_to_append[] = { 'z', '0', '_' }; |
90 for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) { | 90 for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) { |
91 i::MemMove(buffer, keyword, length); | 91 i::MemMove(buffer, keyword, length); |
92 buffer[length] = chars_to_append[j]; | 92 buffer[length] = chars_to_append[j]; |
93 i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length + 1); | 93 auto stream = i::ScannerStream::ForTesting(buffer, length + 1); |
94 i::Scanner scanner(&unicode_cache); | 94 i::Scanner scanner(&unicode_cache); |
95 scanner.Initialize(&stream); | 95 scanner.Initialize(stream.get()); |
96 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); | 96 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); |
97 CHECK_EQ(i::Token::EOS, scanner.Next()); | 97 CHECK_EQ(i::Token::EOS, scanner.Next()); |
98 } | 98 } |
99 // Replacing characters will make keyword matching fail. | 99 // Replacing characters will make keyword matching fail. |
100 { | 100 { |
101 i::MemMove(buffer, keyword, length); | 101 i::MemMove(buffer, keyword, length); |
102 buffer[length - 1] = '_'; | 102 buffer[length - 1] = '_'; |
103 i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length); | 103 auto stream = i::ScannerStream::ForTesting(buffer, length); |
104 i::Scanner scanner(&unicode_cache); | 104 i::Scanner scanner(&unicode_cache); |
105 scanner.Initialize(&stream); | 105 scanner.Initialize(stream.get()); |
106 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); | 106 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); |
107 CHECK_EQ(i::Token::EOS, scanner.Next()); | 107 CHECK_EQ(i::Token::EOS, scanner.Next()); |
108 } | 108 } |
109 } | 109 } |
110 } | 110 } |
111 | 111 |
112 | 112 |
113 TEST(ScanHTMLEndComments) { | 113 TEST(ScanHTMLEndComments) { |
114 v8::V8::Initialize(); | 114 v8::V8::Initialize(); |
115 v8::Isolate* isolate = CcTest::isolate(); | 115 v8::Isolate* isolate = CcTest::isolate(); |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
159 NULL | 159 NULL |
160 }; | 160 }; |
161 // clang-format on | 161 // clang-format on |
162 | 162 |
163 // Parser/Scanner needs a stack limit. | 163 // Parser/Scanner needs a stack limit. |
164 CcTest::i_isolate()->stack_guard()->SetStackLimit( | 164 CcTest::i_isolate()->stack_guard()->SetStackLimit( |
165 i::GetCurrentStackPosition() - 128 * 1024); | 165 i::GetCurrentStackPosition() - 128 * 1024); |
166 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); | 166 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); |
167 for (int i = 0; tests[i]; i++) { | 167 for (int i = 0; tests[i]; i++) { |
168 const char* source = tests[i]; | 168 const char* source = tests[i]; |
169 i::ExternalOneByteStringUtf16CharacterStream stream(source); | 169 auto stream = i::ScannerStream::ForTesting(source); |
170 i::CompleteParserRecorder log; | 170 i::CompleteParserRecorder log; |
171 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 171 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
172 scanner.Initialize(&stream); | 172 scanner.Initialize(stream.get()); |
173 i::Zone zone(CcTest::i_isolate()->allocator()); | 173 i::Zone zone(CcTest::i_isolate()->allocator()); |
174 i::AstValueFactory ast_value_factory( | 174 i::AstValueFactory ast_value_factory( |
175 &zone, CcTest::i_isolate()->heap()->HashSeed()); | 175 &zone, CcTest::i_isolate()->heap()->HashSeed()); |
176 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 176 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
177 stack_limit); | 177 stack_limit); |
178 preparser.set_allow_lazy(true); | 178 preparser.set_allow_lazy(true); |
179 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 179 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
180 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 180 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
181 CHECK(!log.HasError()); | 181 CHECK(!log.HasError()); |
182 } | 182 } |
183 | 183 |
184 for (int i = 0; fail_tests[i]; i++) { | 184 for (int i = 0; fail_tests[i]; i++) { |
185 const char* source = fail_tests[i]; | 185 const char* source = fail_tests[i]; |
186 i::ExternalOneByteStringUtf16CharacterStream stream(source); | 186 auto stream = i::ScannerStream::ForTesting(source); |
187 i::CompleteParserRecorder log; | 187 i::CompleteParserRecorder log; |
188 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 188 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
189 scanner.Initialize(&stream); | 189 scanner.Initialize(stream.get()); |
190 i::Zone zone(CcTest::i_isolate()->allocator()); | 190 i::Zone zone(CcTest::i_isolate()->allocator()); |
191 i::AstValueFactory ast_value_factory( | 191 i::AstValueFactory ast_value_factory( |
192 &zone, CcTest::i_isolate()->heap()->HashSeed()); | 192 &zone, CcTest::i_isolate()->heap()->HashSeed()); |
193 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 193 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
194 stack_limit); | 194 stack_limit); |
195 preparser.set_allow_lazy(true); | 195 preparser.set_allow_lazy(true); |
196 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 196 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
197 // Even in the case of a syntax error, kPreParseSuccess is returned. | 197 // Even in the case of a syntax error, kPreParseSuccess is returned. |
198 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 198 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
199 CHECK(log.HasError()); | 199 CHECK(log.HasError()); |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
333 "var x = 42;", | 333 "var x = 42;", |
334 "function foo(x, y) { return x + y; }", | 334 "function foo(x, y) { return x + y; }", |
335 "%ArgleBargle(glop);", | 335 "%ArgleBargle(glop);", |
336 "var x = new new Function('this.x = 42');", | 336 "var x = new new Function('this.x = 42');", |
337 "var f = (x, y) => x + y;", | 337 "var f = (x, y) => x + y;", |
338 NULL | 338 NULL |
339 }; | 339 }; |
340 | 340 |
341 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); | 341 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); |
342 for (int i = 0; programs[i]; i++) { | 342 for (int i = 0; programs[i]; i++) { |
343 const char* program = programs[i]; | 343 auto stream = i::ScannerStream::ForTesting(programs[i]); |
344 i::ExternalOneByteStringUtf16CharacterStream stream(program); | |
345 i::CompleteParserRecorder log; | 344 i::CompleteParserRecorder log; |
346 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 345 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
347 scanner.Initialize(&stream); | 346 scanner.Initialize(stream.get()); |
348 | 347 |
349 i::Zone zone(CcTest::i_isolate()->allocator()); | 348 i::Zone zone(CcTest::i_isolate()->allocator()); |
350 i::AstValueFactory ast_value_factory( | 349 i::AstValueFactory ast_value_factory( |
351 &zone, CcTest::i_isolate()->heap()->HashSeed()); | 350 &zone, CcTest::i_isolate()->heap()->HashSeed()); |
352 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 351 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
353 stack_limit); | 352 stack_limit); |
354 preparser.set_allow_lazy(true); | 353 preparser.set_allow_lazy(true); |
355 preparser.set_allow_natives(true); | 354 preparser.set_allow_natives(true); |
356 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 355 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
357 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 356 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
358 CHECK(!log.HasError()); | 357 CHECK(!log.HasError()); |
359 } | 358 } |
360 } | 359 } |
361 | 360 |
362 | 361 |
363 TEST(StandAlonePreParserNoNatives) { | 362 TEST(StandAlonePreParserNoNatives) { |
364 v8::V8::Initialize(); | 363 v8::V8::Initialize(); |
365 | 364 |
366 CcTest::i_isolate()->stack_guard()->SetStackLimit( | 365 CcTest::i_isolate()->stack_guard()->SetStackLimit( |
367 i::GetCurrentStackPosition() - 128 * 1024); | 366 i::GetCurrentStackPosition() - 128 * 1024); |
368 | 367 |
369 const char* programs[] = { | 368 const char* programs[] = { |
370 "%ArgleBargle(glop);", | 369 "%ArgleBargle(glop);", |
371 "var x = %_IsSmi(42);", | 370 "var x = %_IsSmi(42);", |
372 NULL | 371 NULL |
373 }; | 372 }; |
374 | 373 |
375 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); | 374 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); |
376 for (int i = 0; programs[i]; i++) { | 375 for (int i = 0; programs[i]; i++) { |
377 const char* program = programs[i]; | 376 auto stream = i::ScannerStream::ForTesting(programs[i]); |
378 i::ExternalOneByteStringUtf16CharacterStream stream(program); | |
379 i::CompleteParserRecorder log; | 377 i::CompleteParserRecorder log; |
380 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 378 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
381 scanner.Initialize(&stream); | 379 scanner.Initialize(stream.get()); |
382 | 380 |
383 // Preparser defaults to disallowing natives syntax. | 381 // Preparser defaults to disallowing natives syntax. |
384 i::Zone zone(CcTest::i_isolate()->allocator()); | 382 i::Zone zone(CcTest::i_isolate()->allocator()); |
385 i::AstValueFactory ast_value_factory( | 383 i::AstValueFactory ast_value_factory( |
386 &zone, CcTest::i_isolate()->heap()->HashSeed()); | 384 &zone, CcTest::i_isolate()->heap()->HashSeed()); |
387 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 385 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
388 stack_limit); | 386 stack_limit); |
389 preparser.set_allow_lazy(true); | 387 preparser.set_allow_lazy(true); |
390 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 388 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
391 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 389 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
437 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - | 435 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - |
438 128 * 1024); | 436 128 * 1024); |
439 | 437 |
440 const char* program = "var x = 'something';\n" | 438 const char* program = "var x = 'something';\n" |
441 "escape: function() {}"; | 439 "escape: function() {}"; |
442 // Fails parsing expecting an identifier after "function". | 440 // Fails parsing expecting an identifier after "function". |
443 // Before fix, didn't check *ok after Expect(Token::Identifier, ok), | 441 // Before fix, didn't check *ok after Expect(Token::Identifier, ok), |
444 // and then used the invalid currently scanned literal. This always | 442 // and then used the invalid currently scanned literal. This always |
445 // failed in debug mode, and sometimes crashed in release mode. | 443 // failed in debug mode, and sometimes crashed in release mode. |
446 | 444 |
447 i::ExternalOneByteStringUtf16CharacterStream stream(program); | 445 auto stream = i::ScannerStream::ForTesting(program); |
448 i::CompleteParserRecorder log; | 446 i::CompleteParserRecorder log; |
449 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 447 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
450 scanner.Initialize(&stream); | 448 scanner.Initialize(stream.get()); |
451 i::Zone zone(CcTest::i_isolate()->allocator()); | 449 i::Zone zone(CcTest::i_isolate()->allocator()); |
452 i::AstValueFactory ast_value_factory(&zone, | 450 i::AstValueFactory ast_value_factory(&zone, |
453 CcTest::i_isolate()->heap()->HashSeed()); | 451 CcTest::i_isolate()->heap()->HashSeed()); |
454 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 452 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
455 CcTest::i_isolate()->stack_guard()->real_climit()); | 453 CcTest::i_isolate()->stack_guard()->real_climit()); |
456 preparser.set_allow_lazy(true); | 454 preparser.set_allow_lazy(true); |
457 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 455 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
458 // Even in the case of a syntax error, kPreParseSuccess is returned. | 456 // Even in the case of a syntax error, kPreParseSuccess is returned. |
459 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 457 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
460 CHECK(log.HasError()); | 458 CHECK(log.HasError()); |
461 } | 459 } |
462 | 460 |
463 | 461 |
464 TEST(Regress928) { | 462 TEST(Regress928) { |
465 v8::V8::Initialize(); | 463 v8::V8::Initialize(); |
466 i::Isolate* isolate = CcTest::i_isolate(); | 464 i::Isolate* isolate = CcTest::i_isolate(); |
467 i::Factory* factory = isolate->factory(); | |
468 | 465 |
469 // Preparsing didn't consider the catch clause of a try statement | 466 // Preparsing didn't consider the catch clause of a try statement |
470 // as with-content, which made it assume that a function inside | 467 // as with-content, which made it assume that a function inside |
471 // the block could be lazily compiled, and an extra, unexpected, | 468 // the block could be lazily compiled, and an extra, unexpected, |
472 // entry was added to the data. | 469 // entry was added to the data. |
473 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - | 470 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - |
474 128 * 1024); | 471 128 * 1024); |
475 | 472 |
476 const char* program = | 473 const char* program = |
477 "try { } catch (e) { var foo = function () { /* first */ } }" | 474 "try { } catch (e) { var foo = function () { /* first */ } }" |
478 "var bar = function () { /* second */ }"; | 475 "var bar = function () { /* second */ }"; |
479 | 476 |
480 v8::HandleScope handles(CcTest::isolate()); | 477 v8::HandleScope handles(CcTest::isolate()); |
481 i::Handle<i::String> source = factory->NewStringFromAsciiChecked(program); | 478 auto stream = i::ScannerStream::ForTesting(program); |
482 i::GenericStringUtf16CharacterStream stream(source, 0, source->length()); | |
483 i::CompleteParserRecorder log; | 479 i::CompleteParserRecorder log; |
484 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 480 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
485 scanner.Initialize(&stream); | 481 scanner.Initialize(stream.get()); |
486 i::Zone zone(CcTest::i_isolate()->allocator()); | 482 i::Zone zone(CcTest::i_isolate()->allocator()); |
487 i::AstValueFactory ast_value_factory(&zone, | 483 i::AstValueFactory ast_value_factory(&zone, |
488 CcTest::i_isolate()->heap()->HashSeed()); | 484 CcTest::i_isolate()->heap()->HashSeed()); |
489 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 485 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
490 CcTest::i_isolate()->stack_guard()->real_climit()); | 486 CcTest::i_isolate()->stack_guard()->real_climit()); |
491 preparser.set_allow_lazy(true); | 487 preparser.set_allow_lazy(true); |
492 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 488 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
493 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 489 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
494 i::ScriptData* sd = log.GetScriptData(); | 490 i::ScriptData* sd = log.GetScriptData(); |
495 i::ParseData* pd = i::ParseData::FromCachedData(sd); | 491 i::ParseData* pd = i::ParseData::FromCachedData(sd); |
(...skipping 25 matching lines...) Expand all Loading... |
521 CcTest::i_isolate()->stack_guard()->SetStackLimit( | 517 CcTest::i_isolate()->stack_guard()->SetStackLimit( |
522 i::GetCurrentStackPosition() - 128 * 1024); | 518 i::GetCurrentStackPosition() - 128 * 1024); |
523 | 519 |
524 size_t kProgramSize = 1024 * 1024; | 520 size_t kProgramSize = 1024 * 1024; |
525 std::unique_ptr<char[]> program(i::NewArray<char>(kProgramSize + 1)); | 521 std::unique_ptr<char[]> program(i::NewArray<char>(kProgramSize + 1)); |
526 memset(program.get(), '(', kProgramSize); | 522 memset(program.get(), '(', kProgramSize); |
527 program[kProgramSize] = '\0'; | 523 program[kProgramSize] = '\0'; |
528 | 524 |
529 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); | 525 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); |
530 | 526 |
531 i::ExternalOneByteStringUtf16CharacterStream stream(program.get(), | 527 auto stream = i::ScannerStream::ForTesting(program.get(), kProgramSize); |
532 kProgramSize); | |
533 i::CompleteParserRecorder log; | 528 i::CompleteParserRecorder log; |
534 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 529 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
535 scanner.Initialize(&stream); | 530 scanner.Initialize(stream.get()); |
536 | 531 |
537 i::Zone zone(CcTest::i_isolate()->allocator()); | 532 i::Zone zone(CcTest::i_isolate()->allocator()); |
538 i::AstValueFactory ast_value_factory(&zone, | 533 i::AstValueFactory ast_value_factory(&zone, |
539 CcTest::i_isolate()->heap()->HashSeed()); | 534 CcTest::i_isolate()->heap()->HashSeed()); |
540 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 535 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
541 stack_limit); | 536 stack_limit); |
542 preparser.set_allow_lazy(true); | 537 preparser.set_allow_lazy(true); |
543 i::PreParser::PreParseResult result = preparser.PreParseProgram(); | 538 i::PreParser::PreParseResult result = preparser.PreParseProgram(); |
544 CHECK_EQ(i::PreParser::kPreParseStackOverflow, result); | 539 CHECK_EQ(i::PreParser::kPreParseStackOverflow, result); |
545 } | 540 } |
546 | 541 |
547 | 542 |
548 class TestExternalResource: public v8::String::ExternalStringResource { | |
549 public: | |
550 explicit TestExternalResource(uint16_t* data, int length) | |
551 : data_(data), length_(static_cast<size_t>(length)) { } | |
552 | |
553 ~TestExternalResource() { } | |
554 | |
555 const uint16_t* data() const { | |
556 return data_; | |
557 } | |
558 | |
559 size_t length() const { | |
560 return length_; | |
561 } | |
562 private: | |
563 uint16_t* data_; | |
564 size_t length_; | |
565 }; | |
566 | |
567 | |
568 #define CHECK_EQU(v1, v2) CHECK_EQ(static_cast<int>(v1), static_cast<int>(v2)) | |
569 | |
570 void TestCharacterStream(const char* one_byte_source, unsigned length, | |
571 unsigned start = 0, unsigned end = 0) { | |
572 if (end == 0) end = length; | |
573 unsigned sub_length = end - start; | |
574 i::Isolate* isolate = CcTest::i_isolate(); | |
575 i::Factory* factory = isolate->factory(); | |
576 i::HandleScope test_scope(isolate); | |
577 std::unique_ptr<i::uc16[]> uc16_buffer(new i::uc16[length]); | |
578 for (unsigned i = 0; i < length; i++) { | |
579 uc16_buffer[i] = static_cast<i::uc16>(one_byte_source[i]); | |
580 } | |
581 i::Vector<const char> one_byte_vector(one_byte_source, | |
582 static_cast<int>(length)); | |
583 i::Handle<i::String> one_byte_string = | |
584 factory->NewStringFromAscii(one_byte_vector).ToHandleChecked(); | |
585 TestExternalResource resource(uc16_buffer.get(), length); | |
586 i::Handle<i::String> uc16_string( | |
587 factory->NewExternalStringFromTwoByte(&resource).ToHandleChecked()); | |
588 ScriptResource one_byte_resource(one_byte_source, length); | |
589 i::Handle<i::String> ext_one_byte_string( | |
590 factory->NewExternalStringFromOneByte(&one_byte_resource) | |
591 .ToHandleChecked()); | |
592 | |
593 i::ExternalTwoByteStringUtf16CharacterStream uc16_stream( | |
594 i::Handle<i::ExternalTwoByteString>::cast(uc16_string), start, end); | |
595 i::ExternalOneByteStringUtf16CharacterStream one_byte_stream( | |
596 i::Handle<i::ExternalOneByteString>::cast(ext_one_byte_string), start, | |
597 end); | |
598 i::GenericStringUtf16CharacterStream string_stream(one_byte_string, start, | |
599 end); | |
600 i::ExternalOneByteStringUtf16CharacterStream utf8_stream(one_byte_source, | |
601 end); | |
602 utf8_stream.SeekForward(start); | |
603 | |
604 unsigned i = start; | |
605 while (i < end) { | |
606 // Read streams one char at a time | |
607 CHECK_EQU(i, uc16_stream.pos()); | |
608 CHECK_EQU(i, string_stream.pos()); | |
609 CHECK_EQU(i, utf8_stream.pos()); | |
610 CHECK_EQU(i, one_byte_stream.pos()); | |
611 int32_t c0 = one_byte_source[i]; | |
612 int32_t c1 = uc16_stream.Advance(); | |
613 int32_t c2 = string_stream.Advance(); | |
614 int32_t c3 = utf8_stream.Advance(); | |
615 int32_t c4 = one_byte_stream.Advance(); | |
616 i++; | |
617 CHECK_EQ(c0, c1); | |
618 CHECK_EQ(c0, c2); | |
619 CHECK_EQ(c0, c3); | |
620 CHECK_EQ(c0, c4); | |
621 CHECK_EQU(i, uc16_stream.pos()); | |
622 CHECK_EQU(i, string_stream.pos()); | |
623 CHECK_EQU(i, utf8_stream.pos()); | |
624 CHECK_EQU(i, one_byte_stream.pos()); | |
625 } | |
626 while (i > start + sub_length / 4) { | |
627 // Pushback, re-read, pushback again. | |
628 int32_t c0 = one_byte_source[i - 1]; | |
629 CHECK_EQU(i, uc16_stream.pos()); | |
630 CHECK_EQU(i, string_stream.pos()); | |
631 CHECK_EQU(i, utf8_stream.pos()); | |
632 CHECK_EQU(i, one_byte_stream.pos()); | |
633 uc16_stream.PushBack(c0); | |
634 string_stream.PushBack(c0); | |
635 utf8_stream.PushBack(c0); | |
636 one_byte_stream.PushBack(c0); | |
637 i--; | |
638 CHECK_EQU(i, uc16_stream.pos()); | |
639 CHECK_EQU(i, string_stream.pos()); | |
640 CHECK_EQU(i, utf8_stream.pos()); | |
641 CHECK_EQU(i, one_byte_stream.pos()); | |
642 int32_t c1 = uc16_stream.Advance(); | |
643 int32_t c2 = string_stream.Advance(); | |
644 int32_t c3 = utf8_stream.Advance(); | |
645 int32_t c4 = one_byte_stream.Advance(); | |
646 i++; | |
647 CHECK_EQU(i, uc16_stream.pos()); | |
648 CHECK_EQU(i, string_stream.pos()); | |
649 CHECK_EQU(i, utf8_stream.pos()); | |
650 CHECK_EQU(i, one_byte_stream.pos()); | |
651 CHECK_EQ(c0, c1); | |
652 CHECK_EQ(c0, c2); | |
653 CHECK_EQ(c0, c3); | |
654 CHECK_EQ(c0, c4); | |
655 uc16_stream.PushBack(c0); | |
656 string_stream.PushBack(c0); | |
657 utf8_stream.PushBack(c0); | |
658 one_byte_stream.PushBack(c0); | |
659 i--; | |
660 CHECK_EQU(i, uc16_stream.pos()); | |
661 CHECK_EQU(i, string_stream.pos()); | |
662 CHECK_EQU(i, utf8_stream.pos()); | |
663 CHECK_EQU(i, one_byte_stream.pos()); | |
664 } | |
665 unsigned halfway = start + sub_length / 2; | |
666 uc16_stream.SeekForward(halfway - i); | |
667 string_stream.SeekForward(halfway - i); | |
668 utf8_stream.SeekForward(halfway - i); | |
669 one_byte_stream.SeekForward(halfway - i); | |
670 i = halfway; | |
671 CHECK_EQU(i, uc16_stream.pos()); | |
672 CHECK_EQU(i, string_stream.pos()); | |
673 CHECK_EQU(i, utf8_stream.pos()); | |
674 CHECK_EQU(i, one_byte_stream.pos()); | |
675 | |
676 while (i < end) { | |
677 // Read streams one char at a time | |
678 CHECK_EQU(i, uc16_stream.pos()); | |
679 CHECK_EQU(i, string_stream.pos()); | |
680 CHECK_EQU(i, utf8_stream.pos()); | |
681 CHECK_EQU(i, one_byte_stream.pos()); | |
682 int32_t c0 = one_byte_source[i]; | |
683 int32_t c1 = uc16_stream.Advance(); | |
684 int32_t c2 = string_stream.Advance(); | |
685 int32_t c3 = utf8_stream.Advance(); | |
686 int32_t c4 = one_byte_stream.Advance(); | |
687 i++; | |
688 CHECK_EQ(c0, c1); | |
689 CHECK_EQ(c0, c2); | |
690 CHECK_EQ(c0, c3); | |
691 CHECK_EQ(c0, c4); | |
692 CHECK_EQU(i, uc16_stream.pos()); | |
693 CHECK_EQU(i, string_stream.pos()); | |
694 CHECK_EQU(i, utf8_stream.pos()); | |
695 CHECK_EQU(i, one_byte_stream.pos()); | |
696 } | |
697 | |
698 int32_t c1 = uc16_stream.Advance(); | |
699 int32_t c2 = string_stream.Advance(); | |
700 int32_t c3 = utf8_stream.Advance(); | |
701 int32_t c4 = one_byte_stream.Advance(); | |
702 CHECK_LT(c1, 0); | |
703 CHECK_LT(c2, 0); | |
704 CHECK_LT(c3, 0); | |
705 CHECK_LT(c4, 0); | |
706 } | |
707 | |
708 #undef CHECK_EQU | |
709 | |
710 TEST(CharacterStreams) { | |
711 v8::Isolate* isolate = CcTest::isolate(); | |
712 v8::HandleScope handles(isolate); | |
713 v8::Local<v8::Context> context = v8::Context::New(isolate); | |
714 v8::Context::Scope context_scope(context); | |
715 | |
716 TestCharacterStream("abc\0\n\r\x7f", 7); | |
717 static const unsigned kBigStringSize = 4096; | |
718 char buffer[kBigStringSize + 1]; | |
719 for (unsigned i = 0; i < kBigStringSize; i++) { | |
720 buffer[i] = static_cast<char>(i & 0x7f); | |
721 } | |
722 TestCharacterStream(buffer, kBigStringSize); | |
723 | |
724 TestCharacterStream(buffer, kBigStringSize, 576, 3298); | |
725 | |
726 TestCharacterStream("\0", 1); | |
727 TestCharacterStream("", 0); | |
728 } | |
729 | |
730 | |
731 void TestStreamScanner(i::Utf16CharacterStream* stream, | 543 void TestStreamScanner(i::Utf16CharacterStream* stream, |
732 i::Token::Value* expected_tokens, | 544 i::Token::Value* expected_tokens, |
733 int skip_pos = 0, // Zero means not skipping. | 545 int skip_pos = 0, // Zero means not skipping. |
734 int skip_to = 0) { | 546 int skip_to = 0) { |
735 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 547 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
736 scanner.Initialize(stream); | 548 scanner.Initialize(stream); |
737 | 549 |
738 int i = 0; | 550 int i = 0; |
739 do { | 551 do { |
740 i::Token::Value expected = expected_tokens[i]; | 552 i::Token::Value expected = expected_tokens[i]; |
741 i::Token::Value actual = scanner.Next(); | 553 i::Token::Value actual = scanner.Next(); |
742 CHECK_EQ(i::Token::String(expected), i::Token::String(actual)); | 554 CHECK_EQ(i::Token::String(expected), i::Token::String(actual)); |
743 if (scanner.location().end_pos == skip_pos) { | 555 if (scanner.location().end_pos == skip_pos) { |
744 scanner.SeekForward(skip_to); | 556 scanner.SeekForward(skip_to); |
745 } | 557 } |
746 i++; | 558 i++; |
747 } while (expected_tokens[i] != i::Token::ILLEGAL); | 559 } while (expected_tokens[i] != i::Token::ILLEGAL); |
748 } | 560 } |
749 | 561 |
750 | 562 |
751 TEST(StreamScanner) { | 563 TEST(StreamScanner) { |
752 v8::V8::Initialize(); | 564 v8::V8::Initialize(); |
753 | |
754 const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib"; | 565 const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib"; |
755 i::ExternalOneByteStringUtf16CharacterStream stream1(str1); | 566 std::unique_ptr<i::Utf16CharacterStream> stream1( |
| 567 i::ScannerStream::ForTesting(str1)); |
756 i::Token::Value expectations1[] = { | 568 i::Token::Value expectations1[] = { |
757 i::Token::LBRACE, | 569 i::Token::LBRACE, |
758 i::Token::IDENTIFIER, | 570 i::Token::IDENTIFIER, |
759 i::Token::IDENTIFIER, | 571 i::Token::IDENTIFIER, |
760 i::Token::FOR, | 572 i::Token::FOR, |
761 i::Token::COLON, | 573 i::Token::COLON, |
762 i::Token::MUL, | 574 i::Token::MUL, |
763 i::Token::DIV, | 575 i::Token::DIV, |
764 i::Token::LT, | 576 i::Token::LT, |
765 i::Token::SUB, | 577 i::Token::SUB, |
766 i::Token::IDENTIFIER, | 578 i::Token::IDENTIFIER, |
767 i::Token::EOS, | 579 i::Token::EOS, |
768 i::Token::ILLEGAL | 580 i::Token::ILLEGAL |
769 }; | 581 }; |
770 TestStreamScanner(&stream1, expectations1, 0, 0); | 582 TestStreamScanner(stream1.get(), expectations1, 0, 0); |
771 | 583 |
772 const char* str2 = "case default const {THIS\nPART\nSKIPPED} do"; | 584 const char* str2 = "case default const {THIS\nPART\nSKIPPED} do"; |
773 i::ExternalOneByteStringUtf16CharacterStream stream2(str2); | 585 std::unique_ptr<i::Utf16CharacterStream> stream2( |
| 586 i::ScannerStream::ForTesting(str2)); |
774 i::Token::Value expectations2[] = { | 587 i::Token::Value expectations2[] = { |
775 i::Token::CASE, | 588 i::Token::CASE, |
776 i::Token::DEFAULT, | 589 i::Token::DEFAULT, |
777 i::Token::CONST, | 590 i::Token::CONST, |
778 i::Token::LBRACE, | 591 i::Token::LBRACE, |
779 // Skipped part here | 592 // Skipped part here |
780 i::Token::RBRACE, | 593 i::Token::RBRACE, |
781 i::Token::DO, | 594 i::Token::DO, |
782 i::Token::EOS, | 595 i::Token::EOS, |
783 i::Token::ILLEGAL | 596 i::Token::ILLEGAL |
784 }; | 597 }; |
785 CHECK_EQ('{', str2[19]); | 598 CHECK_EQ('{', str2[19]); |
786 CHECK_EQ('}', str2[37]); | 599 CHECK_EQ('}', str2[37]); |
787 TestStreamScanner(&stream2, expectations2, 20, 37); | 600 TestStreamScanner(stream2.get(), expectations2, 20, 37); |
788 | 601 |
789 const char* str3 = "{}}}}"; | 602 const char* str3 = "{}}}}"; |
790 i::Token::Value expectations3[] = { | 603 i::Token::Value expectations3[] = { |
791 i::Token::LBRACE, | 604 i::Token::LBRACE, |
792 i::Token::RBRACE, | 605 i::Token::RBRACE, |
793 i::Token::RBRACE, | 606 i::Token::RBRACE, |
794 i::Token::RBRACE, | 607 i::Token::RBRACE, |
795 i::Token::RBRACE, | 608 i::Token::RBRACE, |
796 i::Token::EOS, | 609 i::Token::EOS, |
797 i::Token::ILLEGAL | 610 i::Token::ILLEGAL |
798 }; | 611 }; |
799 // Skip zero-four RBRACEs. | 612 // Skip zero-four RBRACEs. |
800 for (int i = 0; i <= 4; i++) { | 613 for (int i = 0; i <= 4; i++) { |
801 expectations3[6 - i] = i::Token::ILLEGAL; | 614 expectations3[6 - i] = i::Token::ILLEGAL; |
802 expectations3[5 - i] = i::Token::EOS; | 615 expectations3[5 - i] = i::Token::EOS; |
803 i::ExternalOneByteStringUtf16CharacterStream stream3(str3); | 616 std::unique_ptr<i::Utf16CharacterStream> stream3( |
804 TestStreamScanner(&stream3, expectations3, 1, 1 + i); | 617 i::ScannerStream::ForTesting(str3)); |
| 618 TestStreamScanner(stream3.get(), expectations3, 1, 1 + i); |
805 } | 619 } |
806 } | 620 } |
807 | 621 |
808 | |
809 void TestScanRegExp(const char* re_source, const char* expected) { | 622 void TestScanRegExp(const char* re_source, const char* expected) { |
810 i::ExternalOneByteStringUtf16CharacterStream stream(re_source); | 623 auto stream = i::ScannerStream::ForTesting(re_source); |
811 i::HandleScope scope(CcTest::i_isolate()); | 624 i::HandleScope scope(CcTest::i_isolate()); |
812 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); | 625 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); |
813 scanner.Initialize(&stream); | 626 scanner.Initialize(stream.get()); |
814 | 627 |
815 i::Token::Value start = scanner.peek(); | 628 i::Token::Value start = scanner.peek(); |
816 CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV); | 629 CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV); |
817 CHECK(scanner.ScanRegExpPattern()); | 630 CHECK(scanner.ScanRegExpPattern()); |
818 scanner.Next(); // Current token is now the regexp literal. | 631 scanner.Next(); // Current token is now the regexp literal. |
819 i::Zone zone(CcTest::i_isolate()->allocator()); | 632 i::Zone zone(CcTest::i_isolate()->allocator()); |
820 i::AstValueFactory ast_value_factory(&zone, | 633 i::AstValueFactory ast_value_factory(&zone, |
821 CcTest::i_isolate()->heap()->HashSeed()); | 634 CcTest::i_isolate()->heap()->HashSeed()); |
822 ast_value_factory.Internalize(CcTest::i_isolate()); | 635 ast_value_factory.Internalize(CcTest::i_isolate()); |
823 i::Handle<i::String> val = | 636 i::Handle<i::String> val = |
(...skipping 702 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1526 i::Factory* factory = isolate->factory(); | 1339 i::Factory* factory = isolate->factory(); |
1527 | 1340 |
1528 uintptr_t stack_limit = isolate->stack_guard()->real_climit(); | 1341 uintptr_t stack_limit = isolate->stack_guard()->real_climit(); |
1529 int preparser_materialized_literals = -1; | 1342 int preparser_materialized_literals = -1; |
1530 int parser_materialized_literals = -2; | 1343 int parser_materialized_literals = -2; |
1531 | 1344 |
1532 // Preparse the data. | 1345 // Preparse the data. |
1533 i::CompleteParserRecorder log; | 1346 i::CompleteParserRecorder log; |
1534 if (test_preparser) { | 1347 if (test_preparser) { |
1535 i::Scanner scanner(isolate->unicode_cache()); | 1348 i::Scanner scanner(isolate->unicode_cache()); |
1536 i::GenericStringUtf16CharacterStream stream(source, 0, source->length()); | 1349 std::unique_ptr<i::Utf16CharacterStream> stream( |
| 1350 i::ScannerStream::For(source)); |
1537 i::Zone zone(CcTest::i_isolate()->allocator()); | 1351 i::Zone zone(CcTest::i_isolate()->allocator()); |
1538 i::AstValueFactory ast_value_factory( | 1352 i::AstValueFactory ast_value_factory( |
1539 &zone, CcTest::i_isolate()->heap()->HashSeed()); | 1353 &zone, CcTest::i_isolate()->heap()->HashSeed()); |
1540 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, | 1354 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, |
1541 stack_limit); | 1355 stack_limit); |
1542 SetParserFlags(&preparser, flags); | 1356 SetParserFlags(&preparser, flags); |
1543 scanner.Initialize(&stream); | 1357 scanner.Initialize(stream.get()); |
1544 i::PreParser::PreParseResult result = | 1358 i::PreParser::PreParseResult result = |
1545 preparser.PreParseProgram(&preparser_materialized_literals, is_module); | 1359 preparser.PreParseProgram(&preparser_materialized_literals, is_module); |
1546 CHECK_EQ(i::PreParser::kPreParseSuccess, result); | 1360 CHECK_EQ(i::PreParser::kPreParseSuccess, result); |
1547 } | 1361 } |
1548 bool preparse_error = log.HasError(); | 1362 bool preparse_error = log.HasError(); |
1549 | 1363 |
1550 // Parse the data | 1364 // Parse the data |
1551 i::FunctionLiteral* function; | 1365 i::FunctionLiteral* function; |
1552 { | 1366 { |
1553 i::Handle<i::Script> script = factory->NewScript(source); | 1367 i::Handle<i::Script> script = factory->NewScript(source); |
(...skipping 6769 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
8323 "(a,);", | 8137 "(a,);", |
8324 "(a,b,c,);", | 8138 "(a,b,c,);", |
8325 NULL | 8139 NULL |
8326 }; | 8140 }; |
8327 // clang-format on | 8141 // clang-format on |
8328 | 8142 |
8329 static const ParserFlag always_flags[] = {kAllowHarmonyTrailingCommas}; | 8143 static const ParserFlag always_flags[] = {kAllowHarmonyTrailingCommas}; |
8330 RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags, | 8144 RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags, |
8331 arraysize(always_flags)); | 8145 arraysize(always_flags)); |
8332 } | 8146 } |
OLD | NEW |