Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(589)

Side by Side Diff: test/cctest/test-parsing.cc

Issue 2314663002: Rework scanner-character-streams. (Closed)
Patch Set: Add comments to document Utf8ExternalStreamingStream's internal state. Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2012 the V8 project authors. All rights reserved. 1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Redistribution and use in source and binary forms, with or without 2 // Redistribution and use in source and binary forms, with or without
3 // modification, are permitted provided that the following conditions are 3 // modification, are permitted provided that the following conditions are
4 // met: 4 // met:
5 // 5 //
6 // * Redistributions of source code must retain the above copyright 6 // * Redistributions of source code must retain the above copyright
7 // notice, this list of conditions and the following disclaimer. 7 // notice, this list of conditions and the following disclaimer.
8 // * Redistributions in binary form must reproduce the above 8 // * Redistributions in binary form must reproduce the above
9 // copyright notice, this list of conditions and the following 9 // copyright notice, this list of conditions and the following
10 // disclaimer in the documentation and/or other materials provided 10 // disclaimer in the documentation and/or other materials provided
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
64 }; 64 };
65 65
66 KeywordToken key_token; 66 KeywordToken key_token;
67 i::UnicodeCache unicode_cache; 67 i::UnicodeCache unicode_cache;
68 char buffer[32]; 68 char buffer[32];
69 for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) { 69 for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
70 const char* keyword = key_token.keyword; 70 const char* keyword = key_token.keyword;
71 size_t length = strlen(key_token.keyword); 71 size_t length = strlen(key_token.keyword);
72 CHECK(static_cast<int>(sizeof(buffer)) >= length); 72 CHECK(static_cast<int>(sizeof(buffer)) >= length);
73 { 73 {
74 i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length); 74 auto stream = i::ScannerStream::ForTesting(keyword, length);
75 i::Scanner scanner(&unicode_cache); 75 i::Scanner scanner(&unicode_cache);
76 scanner.Initialize(&stream); 76 scanner.Initialize(stream.get());
77 CHECK_EQ(key_token.token, scanner.Next()); 77 CHECK_EQ(key_token.token, scanner.Next());
78 CHECK_EQ(i::Token::EOS, scanner.Next()); 78 CHECK_EQ(i::Token::EOS, scanner.Next());
79 } 79 }
80 // Removing characters will make keyword matching fail. 80 // Removing characters will make keyword matching fail.
81 { 81 {
82 i::ExternalOneByteStringUtf16CharacterStream stream(keyword, length - 1); 82 auto stream = i::ScannerStream::ForTesting(keyword, length - 1);
83 i::Scanner scanner(&unicode_cache); 83 i::Scanner scanner(&unicode_cache);
84 scanner.Initialize(&stream); 84 scanner.Initialize(stream.get());
85 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); 85 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
86 CHECK_EQ(i::Token::EOS, scanner.Next()); 86 CHECK_EQ(i::Token::EOS, scanner.Next());
87 } 87 }
88 // Adding characters will make keyword matching fail. 88 // Adding characters will make keyword matching fail.
89 static const char chars_to_append[] = { 'z', '0', '_' }; 89 static const char chars_to_append[] = { 'z', '0', '_' };
90 for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) { 90 for (int j = 0; j < static_cast<int>(arraysize(chars_to_append)); ++j) {
91 i::MemMove(buffer, keyword, length); 91 i::MemMove(buffer, keyword, length);
92 buffer[length] = chars_to_append[j]; 92 buffer[length] = chars_to_append[j];
93 i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length + 1); 93 auto stream = i::ScannerStream::ForTesting(buffer, length + 1);
94 i::Scanner scanner(&unicode_cache); 94 i::Scanner scanner(&unicode_cache);
95 scanner.Initialize(&stream); 95 scanner.Initialize(stream.get());
96 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); 96 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
97 CHECK_EQ(i::Token::EOS, scanner.Next()); 97 CHECK_EQ(i::Token::EOS, scanner.Next());
98 } 98 }
99 // Replacing characters will make keyword matching fail. 99 // Replacing characters will make keyword matching fail.
100 { 100 {
101 i::MemMove(buffer, keyword, length); 101 i::MemMove(buffer, keyword, length);
102 buffer[length - 1] = '_'; 102 buffer[length - 1] = '_';
103 i::ExternalOneByteStringUtf16CharacterStream stream(buffer, length); 103 auto stream = i::ScannerStream::ForTesting(buffer, length);
104 i::Scanner scanner(&unicode_cache); 104 i::Scanner scanner(&unicode_cache);
105 scanner.Initialize(&stream); 105 scanner.Initialize(stream.get());
106 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next()); 106 CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
107 CHECK_EQ(i::Token::EOS, scanner.Next()); 107 CHECK_EQ(i::Token::EOS, scanner.Next());
108 } 108 }
109 } 109 }
110 } 110 }
111 111
112 112
113 TEST(ScanHTMLEndComments) { 113 TEST(ScanHTMLEndComments) {
114 v8::V8::Initialize(); 114 v8::V8::Initialize();
115 v8::Isolate* isolate = CcTest::isolate(); 115 v8::Isolate* isolate = CcTest::isolate();
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
159 NULL 159 NULL
160 }; 160 };
161 // clang-format on 161 // clang-format on
162 162
163 // Parser/Scanner needs a stack limit. 163 // Parser/Scanner needs a stack limit.
164 CcTest::i_isolate()->stack_guard()->SetStackLimit( 164 CcTest::i_isolate()->stack_guard()->SetStackLimit(
165 i::GetCurrentStackPosition() - 128 * 1024); 165 i::GetCurrentStackPosition() - 128 * 1024);
166 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); 166 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
167 for (int i = 0; tests[i]; i++) { 167 for (int i = 0; tests[i]; i++) {
168 const char* source = tests[i]; 168 const char* source = tests[i];
169 i::ExternalOneByteStringUtf16CharacterStream stream(source); 169 auto stream = i::ScannerStream::ForTesting(source);
170 i::CompleteParserRecorder log; 170 i::CompleteParserRecorder log;
171 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 171 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
172 scanner.Initialize(&stream); 172 scanner.Initialize(stream.get());
173 i::Zone zone(CcTest::i_isolate()->allocator()); 173 i::Zone zone(CcTest::i_isolate()->allocator());
174 i::AstValueFactory ast_value_factory( 174 i::AstValueFactory ast_value_factory(
175 &zone, CcTest::i_isolate()->heap()->HashSeed()); 175 &zone, CcTest::i_isolate()->heap()->HashSeed());
176 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 176 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
177 stack_limit); 177 stack_limit);
178 preparser.set_allow_lazy(true); 178 preparser.set_allow_lazy(true);
179 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 179 i::PreParser::PreParseResult result = preparser.PreParseProgram();
180 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 180 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
181 CHECK(!log.HasError()); 181 CHECK(!log.HasError());
182 } 182 }
183 183
184 for (int i = 0; fail_tests[i]; i++) { 184 for (int i = 0; fail_tests[i]; i++) {
185 const char* source = fail_tests[i]; 185 const char* source = fail_tests[i];
186 i::ExternalOneByteStringUtf16CharacterStream stream(source); 186 auto stream = i::ScannerStream::ForTesting(source);
187 i::CompleteParserRecorder log; 187 i::CompleteParserRecorder log;
188 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 188 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
189 scanner.Initialize(&stream); 189 scanner.Initialize(stream.get());
190 i::Zone zone(CcTest::i_isolate()->allocator()); 190 i::Zone zone(CcTest::i_isolate()->allocator());
191 i::AstValueFactory ast_value_factory( 191 i::AstValueFactory ast_value_factory(
192 &zone, CcTest::i_isolate()->heap()->HashSeed()); 192 &zone, CcTest::i_isolate()->heap()->HashSeed());
193 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 193 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
194 stack_limit); 194 stack_limit);
195 preparser.set_allow_lazy(true); 195 preparser.set_allow_lazy(true);
196 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 196 i::PreParser::PreParseResult result = preparser.PreParseProgram();
197 // Even in the case of a syntax error, kPreParseSuccess is returned. 197 // Even in the case of a syntax error, kPreParseSuccess is returned.
198 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 198 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
199 CHECK(log.HasError()); 199 CHECK(log.HasError());
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after
333 "var x = 42;", 333 "var x = 42;",
334 "function foo(x, y) { return x + y; }", 334 "function foo(x, y) { return x + y; }",
335 "%ArgleBargle(glop);", 335 "%ArgleBargle(glop);",
336 "var x = new new Function('this.x = 42');", 336 "var x = new new Function('this.x = 42');",
337 "var f = (x, y) => x + y;", 337 "var f = (x, y) => x + y;",
338 NULL 338 NULL
339 }; 339 };
340 340
341 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); 341 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
342 for (int i = 0; programs[i]; i++) { 342 for (int i = 0; programs[i]; i++) {
343 const char* program = programs[i]; 343 auto stream = i::ScannerStream::ForTesting(programs[i]);
344 i::ExternalOneByteStringUtf16CharacterStream stream(program);
345 i::CompleteParserRecorder log; 344 i::CompleteParserRecorder log;
346 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 345 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
347 scanner.Initialize(&stream); 346 scanner.Initialize(stream.get());
348 347
349 i::Zone zone(CcTest::i_isolate()->allocator()); 348 i::Zone zone(CcTest::i_isolate()->allocator());
350 i::AstValueFactory ast_value_factory( 349 i::AstValueFactory ast_value_factory(
351 &zone, CcTest::i_isolate()->heap()->HashSeed()); 350 &zone, CcTest::i_isolate()->heap()->HashSeed());
352 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 351 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
353 stack_limit); 352 stack_limit);
354 preparser.set_allow_lazy(true); 353 preparser.set_allow_lazy(true);
355 preparser.set_allow_natives(true); 354 preparser.set_allow_natives(true);
356 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 355 i::PreParser::PreParseResult result = preparser.PreParseProgram();
357 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 356 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
358 CHECK(!log.HasError()); 357 CHECK(!log.HasError());
359 } 358 }
360 } 359 }
361 360
362 361
363 TEST(StandAlonePreParserNoNatives) { 362 TEST(StandAlonePreParserNoNatives) {
364 v8::V8::Initialize(); 363 v8::V8::Initialize();
365 364
366 CcTest::i_isolate()->stack_guard()->SetStackLimit( 365 CcTest::i_isolate()->stack_guard()->SetStackLimit(
367 i::GetCurrentStackPosition() - 128 * 1024); 366 i::GetCurrentStackPosition() - 128 * 1024);
368 367
369 const char* programs[] = { 368 const char* programs[] = {
370 "%ArgleBargle(glop);", 369 "%ArgleBargle(glop);",
371 "var x = %_IsSmi(42);", 370 "var x = %_IsSmi(42);",
372 NULL 371 NULL
373 }; 372 };
374 373
375 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); 374 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
376 for (int i = 0; programs[i]; i++) { 375 for (int i = 0; programs[i]; i++) {
377 const char* program = programs[i]; 376 auto stream = i::ScannerStream::ForTesting(programs[i]);
378 i::ExternalOneByteStringUtf16CharacterStream stream(program);
379 i::CompleteParserRecorder log; 377 i::CompleteParserRecorder log;
380 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 378 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
381 scanner.Initialize(&stream); 379 scanner.Initialize(stream.get());
382 380
383 // Preparser defaults to disallowing natives syntax. 381 // Preparser defaults to disallowing natives syntax.
384 i::Zone zone(CcTest::i_isolate()->allocator()); 382 i::Zone zone(CcTest::i_isolate()->allocator());
385 i::AstValueFactory ast_value_factory( 383 i::AstValueFactory ast_value_factory(
386 &zone, CcTest::i_isolate()->heap()->HashSeed()); 384 &zone, CcTest::i_isolate()->heap()->HashSeed());
387 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 385 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
388 stack_limit); 386 stack_limit);
389 preparser.set_allow_lazy(true); 387 preparser.set_allow_lazy(true);
390 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 388 i::PreParser::PreParseResult result = preparser.PreParseProgram();
391 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 389 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
437 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - 435 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() -
438 128 * 1024); 436 128 * 1024);
439 437
440 const char* program = "var x = 'something';\n" 438 const char* program = "var x = 'something';\n"
441 "escape: function() {}"; 439 "escape: function() {}";
442 // Fails parsing expecting an identifier after "function". 440 // Fails parsing expecting an identifier after "function".
443 // Before fix, didn't check *ok after Expect(Token::Identifier, ok), 441 // Before fix, didn't check *ok after Expect(Token::Identifier, ok),
444 // and then used the invalid currently scanned literal. This always 442 // and then used the invalid currently scanned literal. This always
445 // failed in debug mode, and sometimes crashed in release mode. 443 // failed in debug mode, and sometimes crashed in release mode.
446 444
447 i::ExternalOneByteStringUtf16CharacterStream stream(program); 445 auto stream = i::ScannerStream::ForTesting(program);
448 i::CompleteParserRecorder log; 446 i::CompleteParserRecorder log;
449 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 447 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
450 scanner.Initialize(&stream); 448 scanner.Initialize(stream.get());
451 i::Zone zone(CcTest::i_isolate()->allocator()); 449 i::Zone zone(CcTest::i_isolate()->allocator());
452 i::AstValueFactory ast_value_factory(&zone, 450 i::AstValueFactory ast_value_factory(&zone,
453 CcTest::i_isolate()->heap()->HashSeed()); 451 CcTest::i_isolate()->heap()->HashSeed());
454 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 452 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
455 CcTest::i_isolate()->stack_guard()->real_climit()); 453 CcTest::i_isolate()->stack_guard()->real_climit());
456 preparser.set_allow_lazy(true); 454 preparser.set_allow_lazy(true);
457 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 455 i::PreParser::PreParseResult result = preparser.PreParseProgram();
458 // Even in the case of a syntax error, kPreParseSuccess is returned. 456 // Even in the case of a syntax error, kPreParseSuccess is returned.
459 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 457 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
460 CHECK(log.HasError()); 458 CHECK(log.HasError());
461 } 459 }
462 460
463 461
464 TEST(Regress928) { 462 TEST(Regress928) {
465 v8::V8::Initialize(); 463 v8::V8::Initialize();
466 i::Isolate* isolate = CcTest::i_isolate(); 464 i::Isolate* isolate = CcTest::i_isolate();
467 i::Factory* factory = isolate->factory();
468 465
469 // Preparsing didn't consider the catch clause of a try statement 466 // Preparsing didn't consider the catch clause of a try statement
470 // as with-content, which made it assume that a function inside 467 // as with-content, which made it assume that a function inside
471 // the block could be lazily compiled, and an extra, unexpected, 468 // the block could be lazily compiled, and an extra, unexpected,
472 // entry was added to the data. 469 // entry was added to the data.
473 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() - 470 isolate->stack_guard()->SetStackLimit(i::GetCurrentStackPosition() -
474 128 * 1024); 471 128 * 1024);
475 472
476 const char* program = 473 const char* program =
477 "try { } catch (e) { var foo = function () { /* first */ } }" 474 "try { } catch (e) { var foo = function () { /* first */ } }"
478 "var bar = function () { /* second */ }"; 475 "var bar = function () { /* second */ }";
479 476
480 v8::HandleScope handles(CcTest::isolate()); 477 v8::HandleScope handles(CcTest::isolate());
481 i::Handle<i::String> source = factory->NewStringFromAsciiChecked(program); 478 auto stream = i::ScannerStream::ForTesting(program);
482 i::GenericStringUtf16CharacterStream stream(source, 0, source->length());
483 i::CompleteParserRecorder log; 479 i::CompleteParserRecorder log;
484 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 480 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
485 scanner.Initialize(&stream); 481 scanner.Initialize(stream.get());
486 i::Zone zone(CcTest::i_isolate()->allocator()); 482 i::Zone zone(CcTest::i_isolate()->allocator());
487 i::AstValueFactory ast_value_factory(&zone, 483 i::AstValueFactory ast_value_factory(&zone,
488 CcTest::i_isolate()->heap()->HashSeed()); 484 CcTest::i_isolate()->heap()->HashSeed());
489 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 485 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
490 CcTest::i_isolate()->stack_guard()->real_climit()); 486 CcTest::i_isolate()->stack_guard()->real_climit());
491 preparser.set_allow_lazy(true); 487 preparser.set_allow_lazy(true);
492 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 488 i::PreParser::PreParseResult result = preparser.PreParseProgram();
493 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 489 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
494 i::ScriptData* sd = log.GetScriptData(); 490 i::ScriptData* sd = log.GetScriptData();
495 i::ParseData* pd = i::ParseData::FromCachedData(sd); 491 i::ParseData* pd = i::ParseData::FromCachedData(sd);
(...skipping 25 matching lines...) Expand all
521 CcTest::i_isolate()->stack_guard()->SetStackLimit( 517 CcTest::i_isolate()->stack_guard()->SetStackLimit(
522 i::GetCurrentStackPosition() - 128 * 1024); 518 i::GetCurrentStackPosition() - 128 * 1024);
523 519
524 size_t kProgramSize = 1024 * 1024; 520 size_t kProgramSize = 1024 * 1024;
525 std::unique_ptr<char[]> program(i::NewArray<char>(kProgramSize + 1)); 521 std::unique_ptr<char[]> program(i::NewArray<char>(kProgramSize + 1));
526 memset(program.get(), '(', kProgramSize); 522 memset(program.get(), '(', kProgramSize);
527 program[kProgramSize] = '\0'; 523 program[kProgramSize] = '\0';
528 524
529 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit(); 525 uintptr_t stack_limit = CcTest::i_isolate()->stack_guard()->real_climit();
530 526
531 i::ExternalOneByteStringUtf16CharacterStream stream(program.get(), 527 auto stream = i::ScannerStream::ForTesting(program.get(), kProgramSize);
532 kProgramSize);
533 i::CompleteParserRecorder log; 528 i::CompleteParserRecorder log;
534 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 529 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
535 scanner.Initialize(&stream); 530 scanner.Initialize(stream.get());
536 531
537 i::Zone zone(CcTest::i_isolate()->allocator()); 532 i::Zone zone(CcTest::i_isolate()->allocator());
538 i::AstValueFactory ast_value_factory(&zone, 533 i::AstValueFactory ast_value_factory(&zone,
539 CcTest::i_isolate()->heap()->HashSeed()); 534 CcTest::i_isolate()->heap()->HashSeed());
540 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 535 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
541 stack_limit); 536 stack_limit);
542 preparser.set_allow_lazy(true); 537 preparser.set_allow_lazy(true);
543 i::PreParser::PreParseResult result = preparser.PreParseProgram(); 538 i::PreParser::PreParseResult result = preparser.PreParseProgram();
544 CHECK_EQ(i::PreParser::kPreParseStackOverflow, result); 539 CHECK_EQ(i::PreParser::kPreParseStackOverflow, result);
545 } 540 }
546 541
547 542
548 class TestExternalResource: public v8::String::ExternalStringResource {
549 public:
550 explicit TestExternalResource(uint16_t* data, int length)
551 : data_(data), length_(static_cast<size_t>(length)) { }
552
553 ~TestExternalResource() { }
554
555 const uint16_t* data() const {
556 return data_;
557 }
558
559 size_t length() const {
560 return length_;
561 }
562 private:
563 uint16_t* data_;
564 size_t length_;
565 };
566
567
568 #define CHECK_EQU(v1, v2) CHECK_EQ(static_cast<int>(v1), static_cast<int>(v2))
569
570 void TestCharacterStream(const char* one_byte_source, unsigned length,
571 unsigned start = 0, unsigned end = 0) {
572 if (end == 0) end = length;
573 unsigned sub_length = end - start;
574 i::Isolate* isolate = CcTest::i_isolate();
575 i::Factory* factory = isolate->factory();
576 i::HandleScope test_scope(isolate);
577 std::unique_ptr<i::uc16[]> uc16_buffer(new i::uc16[length]);
578 for (unsigned i = 0; i < length; i++) {
579 uc16_buffer[i] = static_cast<i::uc16>(one_byte_source[i]);
580 }
581 i::Vector<const char> one_byte_vector(one_byte_source,
582 static_cast<int>(length));
583 i::Handle<i::String> one_byte_string =
584 factory->NewStringFromAscii(one_byte_vector).ToHandleChecked();
585 TestExternalResource resource(uc16_buffer.get(), length);
586 i::Handle<i::String> uc16_string(
587 factory->NewExternalStringFromTwoByte(&resource).ToHandleChecked());
588 ScriptResource one_byte_resource(one_byte_source, length);
589 i::Handle<i::String> ext_one_byte_string(
590 factory->NewExternalStringFromOneByte(&one_byte_resource)
591 .ToHandleChecked());
592
593 i::ExternalTwoByteStringUtf16CharacterStream uc16_stream(
594 i::Handle<i::ExternalTwoByteString>::cast(uc16_string), start, end);
595 i::ExternalOneByteStringUtf16CharacterStream one_byte_stream(
596 i::Handle<i::ExternalOneByteString>::cast(ext_one_byte_string), start,
597 end);
598 i::GenericStringUtf16CharacterStream string_stream(one_byte_string, start,
599 end);
600 i::ExternalOneByteStringUtf16CharacterStream utf8_stream(one_byte_source,
601 end);
602 utf8_stream.SeekForward(start);
603
604 unsigned i = start;
605 while (i < end) {
606 // Read streams one char at a time
607 CHECK_EQU(i, uc16_stream.pos());
608 CHECK_EQU(i, string_stream.pos());
609 CHECK_EQU(i, utf8_stream.pos());
610 CHECK_EQU(i, one_byte_stream.pos());
611 int32_t c0 = one_byte_source[i];
612 int32_t c1 = uc16_stream.Advance();
613 int32_t c2 = string_stream.Advance();
614 int32_t c3 = utf8_stream.Advance();
615 int32_t c4 = one_byte_stream.Advance();
616 i++;
617 CHECK_EQ(c0, c1);
618 CHECK_EQ(c0, c2);
619 CHECK_EQ(c0, c3);
620 CHECK_EQ(c0, c4);
621 CHECK_EQU(i, uc16_stream.pos());
622 CHECK_EQU(i, string_stream.pos());
623 CHECK_EQU(i, utf8_stream.pos());
624 CHECK_EQU(i, one_byte_stream.pos());
625 }
626 while (i > start + sub_length / 4) {
627 // Pushback, re-read, pushback again.
628 int32_t c0 = one_byte_source[i - 1];
629 CHECK_EQU(i, uc16_stream.pos());
630 CHECK_EQU(i, string_stream.pos());
631 CHECK_EQU(i, utf8_stream.pos());
632 CHECK_EQU(i, one_byte_stream.pos());
633 uc16_stream.PushBack(c0);
634 string_stream.PushBack(c0);
635 utf8_stream.PushBack(c0);
636 one_byte_stream.PushBack(c0);
637 i--;
638 CHECK_EQU(i, uc16_stream.pos());
639 CHECK_EQU(i, string_stream.pos());
640 CHECK_EQU(i, utf8_stream.pos());
641 CHECK_EQU(i, one_byte_stream.pos());
642 int32_t c1 = uc16_stream.Advance();
643 int32_t c2 = string_stream.Advance();
644 int32_t c3 = utf8_stream.Advance();
645 int32_t c4 = one_byte_stream.Advance();
646 i++;
647 CHECK_EQU(i, uc16_stream.pos());
648 CHECK_EQU(i, string_stream.pos());
649 CHECK_EQU(i, utf8_stream.pos());
650 CHECK_EQU(i, one_byte_stream.pos());
651 CHECK_EQ(c0, c1);
652 CHECK_EQ(c0, c2);
653 CHECK_EQ(c0, c3);
654 CHECK_EQ(c0, c4);
655 uc16_stream.PushBack(c0);
656 string_stream.PushBack(c0);
657 utf8_stream.PushBack(c0);
658 one_byte_stream.PushBack(c0);
659 i--;
660 CHECK_EQU(i, uc16_stream.pos());
661 CHECK_EQU(i, string_stream.pos());
662 CHECK_EQU(i, utf8_stream.pos());
663 CHECK_EQU(i, one_byte_stream.pos());
664 }
665 unsigned halfway = start + sub_length / 2;
666 uc16_stream.SeekForward(halfway - i);
667 string_stream.SeekForward(halfway - i);
668 utf8_stream.SeekForward(halfway - i);
669 one_byte_stream.SeekForward(halfway - i);
670 i = halfway;
671 CHECK_EQU(i, uc16_stream.pos());
672 CHECK_EQU(i, string_stream.pos());
673 CHECK_EQU(i, utf8_stream.pos());
674 CHECK_EQU(i, one_byte_stream.pos());
675
676 while (i < end) {
677 // Read streams one char at a time
678 CHECK_EQU(i, uc16_stream.pos());
679 CHECK_EQU(i, string_stream.pos());
680 CHECK_EQU(i, utf8_stream.pos());
681 CHECK_EQU(i, one_byte_stream.pos());
682 int32_t c0 = one_byte_source[i];
683 int32_t c1 = uc16_stream.Advance();
684 int32_t c2 = string_stream.Advance();
685 int32_t c3 = utf8_stream.Advance();
686 int32_t c4 = one_byte_stream.Advance();
687 i++;
688 CHECK_EQ(c0, c1);
689 CHECK_EQ(c0, c2);
690 CHECK_EQ(c0, c3);
691 CHECK_EQ(c0, c4);
692 CHECK_EQU(i, uc16_stream.pos());
693 CHECK_EQU(i, string_stream.pos());
694 CHECK_EQU(i, utf8_stream.pos());
695 CHECK_EQU(i, one_byte_stream.pos());
696 }
697
698 int32_t c1 = uc16_stream.Advance();
699 int32_t c2 = string_stream.Advance();
700 int32_t c3 = utf8_stream.Advance();
701 int32_t c4 = one_byte_stream.Advance();
702 CHECK_LT(c1, 0);
703 CHECK_LT(c2, 0);
704 CHECK_LT(c3, 0);
705 CHECK_LT(c4, 0);
706 }
707
708 #undef CHECK_EQU
709
710 TEST(CharacterStreams) {
711 v8::Isolate* isolate = CcTest::isolate();
712 v8::HandleScope handles(isolate);
713 v8::Local<v8::Context> context = v8::Context::New(isolate);
714 v8::Context::Scope context_scope(context);
715
716 TestCharacterStream("abc\0\n\r\x7f", 7);
717 static const unsigned kBigStringSize = 4096;
718 char buffer[kBigStringSize + 1];
719 for (unsigned i = 0; i < kBigStringSize; i++) {
720 buffer[i] = static_cast<char>(i & 0x7f);
721 }
722 TestCharacterStream(buffer, kBigStringSize);
723
724 TestCharacterStream(buffer, kBigStringSize, 576, 3298);
725
726 TestCharacterStream("\0", 1);
727 TestCharacterStream("", 0);
728 }
729
730
731 void TestStreamScanner(i::Utf16CharacterStream* stream, 543 void TestStreamScanner(i::Utf16CharacterStream* stream,
732 i::Token::Value* expected_tokens, 544 i::Token::Value* expected_tokens,
733 int skip_pos = 0, // Zero means not skipping. 545 int skip_pos = 0, // Zero means not skipping.
734 int skip_to = 0) { 546 int skip_to = 0) {
735 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 547 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
736 scanner.Initialize(stream); 548 scanner.Initialize(stream);
737 549
738 int i = 0; 550 int i = 0;
739 do { 551 do {
740 i::Token::Value expected = expected_tokens[i]; 552 i::Token::Value expected = expected_tokens[i];
741 i::Token::Value actual = scanner.Next(); 553 i::Token::Value actual = scanner.Next();
742 CHECK_EQ(i::Token::String(expected), i::Token::String(actual)); 554 CHECK_EQ(i::Token::String(expected), i::Token::String(actual));
743 if (scanner.location().end_pos == skip_pos) { 555 if (scanner.location().end_pos == skip_pos) {
744 scanner.SeekForward(skip_to); 556 scanner.SeekForward(skip_to);
745 } 557 }
746 i++; 558 i++;
747 } while (expected_tokens[i] != i::Token::ILLEGAL); 559 } while (expected_tokens[i] != i::Token::ILLEGAL);
748 } 560 }
749 561
750 562
751 TEST(StreamScanner) { 563 TEST(StreamScanner) {
752 v8::V8::Initialize(); 564 v8::V8::Initialize();
753
754 const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib"; 565 const char* str1 = "{ foo get for : */ <- \n\n /*foo*/ bib";
755 i::ExternalOneByteStringUtf16CharacterStream stream1(str1); 566 std::unique_ptr<i::Utf16CharacterStream> stream1(
567 i::ScannerStream::ForTesting(str1));
756 i::Token::Value expectations1[] = { 568 i::Token::Value expectations1[] = {
757 i::Token::LBRACE, 569 i::Token::LBRACE,
758 i::Token::IDENTIFIER, 570 i::Token::IDENTIFIER,
759 i::Token::IDENTIFIER, 571 i::Token::IDENTIFIER,
760 i::Token::FOR, 572 i::Token::FOR,
761 i::Token::COLON, 573 i::Token::COLON,
762 i::Token::MUL, 574 i::Token::MUL,
763 i::Token::DIV, 575 i::Token::DIV,
764 i::Token::LT, 576 i::Token::LT,
765 i::Token::SUB, 577 i::Token::SUB,
766 i::Token::IDENTIFIER, 578 i::Token::IDENTIFIER,
767 i::Token::EOS, 579 i::Token::EOS,
768 i::Token::ILLEGAL 580 i::Token::ILLEGAL
769 }; 581 };
770 TestStreamScanner(&stream1, expectations1, 0, 0); 582 TestStreamScanner(stream1.get(), expectations1, 0, 0);
771 583
772 const char* str2 = "case default const {THIS\nPART\nSKIPPED} do"; 584 const char* str2 = "case default const {THIS\nPART\nSKIPPED} do";
773 i::ExternalOneByteStringUtf16CharacterStream stream2(str2); 585 std::unique_ptr<i::Utf16CharacterStream> stream2(
586 i::ScannerStream::ForTesting(str2));
774 i::Token::Value expectations2[] = { 587 i::Token::Value expectations2[] = {
775 i::Token::CASE, 588 i::Token::CASE,
776 i::Token::DEFAULT, 589 i::Token::DEFAULT,
777 i::Token::CONST, 590 i::Token::CONST,
778 i::Token::LBRACE, 591 i::Token::LBRACE,
779 // Skipped part here 592 // Skipped part here
780 i::Token::RBRACE, 593 i::Token::RBRACE,
781 i::Token::DO, 594 i::Token::DO,
782 i::Token::EOS, 595 i::Token::EOS,
783 i::Token::ILLEGAL 596 i::Token::ILLEGAL
784 }; 597 };
785 CHECK_EQ('{', str2[19]); 598 CHECK_EQ('{', str2[19]);
786 CHECK_EQ('}', str2[37]); 599 CHECK_EQ('}', str2[37]);
787 TestStreamScanner(&stream2, expectations2, 20, 37); 600 TestStreamScanner(stream2.get(), expectations2, 20, 37);
788 601
789 const char* str3 = "{}}}}"; 602 const char* str3 = "{}}}}";
790 i::Token::Value expectations3[] = { 603 i::Token::Value expectations3[] = {
791 i::Token::LBRACE, 604 i::Token::LBRACE,
792 i::Token::RBRACE, 605 i::Token::RBRACE,
793 i::Token::RBRACE, 606 i::Token::RBRACE,
794 i::Token::RBRACE, 607 i::Token::RBRACE,
795 i::Token::RBRACE, 608 i::Token::RBRACE,
796 i::Token::EOS, 609 i::Token::EOS,
797 i::Token::ILLEGAL 610 i::Token::ILLEGAL
798 }; 611 };
799 // Skip zero-four RBRACEs. 612 // Skip zero-four RBRACEs.
800 for (int i = 0; i <= 4; i++) { 613 for (int i = 0; i <= 4; i++) {
801 expectations3[6 - i] = i::Token::ILLEGAL; 614 expectations3[6 - i] = i::Token::ILLEGAL;
802 expectations3[5 - i] = i::Token::EOS; 615 expectations3[5 - i] = i::Token::EOS;
803 i::ExternalOneByteStringUtf16CharacterStream stream3(str3); 616 std::unique_ptr<i::Utf16CharacterStream> stream3(
804 TestStreamScanner(&stream3, expectations3, 1, 1 + i); 617 i::ScannerStream::ForTesting(str3));
618 TestStreamScanner(stream3.get(), expectations3, 1, 1 + i);
805 } 619 }
806 } 620 }
807 621
808
809 void TestScanRegExp(const char* re_source, const char* expected) { 622 void TestScanRegExp(const char* re_source, const char* expected) {
810 i::ExternalOneByteStringUtf16CharacterStream stream(re_source); 623 auto stream = i::ScannerStream::ForTesting(re_source);
811 i::HandleScope scope(CcTest::i_isolate()); 624 i::HandleScope scope(CcTest::i_isolate());
812 i::Scanner scanner(CcTest::i_isolate()->unicode_cache()); 625 i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
813 scanner.Initialize(&stream); 626 scanner.Initialize(stream.get());
814 627
815 i::Token::Value start = scanner.peek(); 628 i::Token::Value start = scanner.peek();
816 CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV); 629 CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
817 CHECK(scanner.ScanRegExpPattern()); 630 CHECK(scanner.ScanRegExpPattern());
818 scanner.Next(); // Current token is now the regexp literal. 631 scanner.Next(); // Current token is now the regexp literal.
819 i::Zone zone(CcTest::i_isolate()->allocator()); 632 i::Zone zone(CcTest::i_isolate()->allocator());
820 i::AstValueFactory ast_value_factory(&zone, 633 i::AstValueFactory ast_value_factory(&zone,
821 CcTest::i_isolate()->heap()->HashSeed()); 634 CcTest::i_isolate()->heap()->HashSeed());
822 ast_value_factory.Internalize(CcTest::i_isolate()); 635 ast_value_factory.Internalize(CcTest::i_isolate());
823 i::Handle<i::String> val = 636 i::Handle<i::String> val =
(...skipping 702 matching lines...) Expand 10 before | Expand all | Expand 10 after
1526 i::Factory* factory = isolate->factory(); 1339 i::Factory* factory = isolate->factory();
1527 1340
1528 uintptr_t stack_limit = isolate->stack_guard()->real_climit(); 1341 uintptr_t stack_limit = isolate->stack_guard()->real_climit();
1529 int preparser_materialized_literals = -1; 1342 int preparser_materialized_literals = -1;
1530 int parser_materialized_literals = -2; 1343 int parser_materialized_literals = -2;
1531 1344
1532 // Preparse the data. 1345 // Preparse the data.
1533 i::CompleteParserRecorder log; 1346 i::CompleteParserRecorder log;
1534 if (test_preparser) { 1347 if (test_preparser) {
1535 i::Scanner scanner(isolate->unicode_cache()); 1348 i::Scanner scanner(isolate->unicode_cache());
1536 i::GenericStringUtf16CharacterStream stream(source, 0, source->length()); 1349 std::unique_ptr<i::Utf16CharacterStream> stream(
1350 i::ScannerStream::For(source));
1537 i::Zone zone(CcTest::i_isolate()->allocator()); 1351 i::Zone zone(CcTest::i_isolate()->allocator());
1538 i::AstValueFactory ast_value_factory( 1352 i::AstValueFactory ast_value_factory(
1539 &zone, CcTest::i_isolate()->heap()->HashSeed()); 1353 &zone, CcTest::i_isolate()->heap()->HashSeed());
1540 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log, 1354 i::PreParser preparser(&zone, &scanner, &ast_value_factory, &log,
1541 stack_limit); 1355 stack_limit);
1542 SetParserFlags(&preparser, flags); 1356 SetParserFlags(&preparser, flags);
1543 scanner.Initialize(&stream); 1357 scanner.Initialize(stream.get());
1544 i::PreParser::PreParseResult result = 1358 i::PreParser::PreParseResult result =
1545 preparser.PreParseProgram(&preparser_materialized_literals, is_module); 1359 preparser.PreParseProgram(&preparser_materialized_literals, is_module);
1546 CHECK_EQ(i::PreParser::kPreParseSuccess, result); 1360 CHECK_EQ(i::PreParser::kPreParseSuccess, result);
1547 } 1361 }
1548 bool preparse_error = log.HasError(); 1362 bool preparse_error = log.HasError();
1549 1363
1550 // Parse the data 1364 // Parse the data
1551 i::FunctionLiteral* function; 1365 i::FunctionLiteral* function;
1552 { 1366 {
1553 i::Handle<i::Script> script = factory->NewScript(source); 1367 i::Handle<i::Script> script = factory->NewScript(source);
(...skipping 6769 matching lines...) Expand 10 before | Expand all | Expand 10 after
8323 "(a,);", 8137 "(a,);",
8324 "(a,b,c,);", 8138 "(a,b,c,);",
8325 NULL 8139 NULL
8326 }; 8140 };
8327 // clang-format on 8141 // clang-format on
8328 8142
8329 static const ParserFlag always_flags[] = {kAllowHarmonyTrailingCommas}; 8143 static const ParserFlag always_flags[] = {kAllowHarmonyTrailingCommas};
8330 RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags, 8144 RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags,
8331 arraysize(always_flags)); 8145 arraysize(always_flags));
8332 } 8146 }
OLDNEW
« src/parsing/scanner-character-streams.cc ('K') | « test/cctest/test-api.cc ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698