OLD | NEW |
1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
42 unibrow::Predicate<unibrow::LineTerminator, 128> Scanner::kIsLineTerminator; | 42 unibrow::Predicate<unibrow::LineTerminator, 128> Scanner::kIsLineTerminator; |
43 unibrow::Predicate<unibrow::WhiteSpace, 128> Scanner::kIsWhiteSpace; | 43 unibrow::Predicate<unibrow::WhiteSpace, 128> Scanner::kIsWhiteSpace; |
44 | 44 |
45 | 45 |
46 StaticResource<Scanner::Utf8Decoder> Scanner::utf8_decoder_; | 46 StaticResource<Scanner::Utf8Decoder> Scanner::utf8_decoder_; |
47 | 47 |
48 | 48 |
49 // ---------------------------------------------------------------------------- | 49 // ---------------------------------------------------------------------------- |
50 // UTF8Buffer | 50 // UTF8Buffer |
51 | 51 |
52 UTF8Buffer::UTF8Buffer() : data_(NULL), limit_(NULL) { } | 52 UTF8Buffer::UTF8Buffer() { |
| 53 static const int kInitialCapacity = 1 * KB; |
| 54 data_ = NewArray<char>(kInitialCapacity); |
| 55 limit_ = ComputeLimit(data_, kInitialCapacity); |
| 56 Reset(); |
| 57 ASSERT(Capacity() == kInitialCapacity && pos() == 0); |
| 58 } |
53 | 59 |
54 | 60 |
55 UTF8Buffer::~UTF8Buffer() { | 61 UTF8Buffer::~UTF8Buffer() { |
56 DeleteArray(data_); | 62 DeleteArray(data_); |
57 } | 63 } |
58 | 64 |
59 | 65 |
60 void UTF8Buffer::AddCharSlow(uc32 c) { | 66 void UTF8Buffer::AddCharSlow(uc32 c) { |
61 static const int kCapacityGrowthLimit = 1 * MB; | 67 static const int kCapacityGrowthLimit = 1 * MB; |
62 if (cursor_ > limit_) { | 68 if (cursor_ > limit_) { |
63 int old_capacity = Capacity(); | 69 int old_capacity = Capacity(); |
64 int old_position = pos(); | 70 int old_position = pos(); |
65 int new_capacity = | 71 int new_capacity = |
66 Min(old_capacity * 3, old_capacity + kCapacityGrowthLimit); | 72 Min(old_capacity * 2, old_capacity + kCapacityGrowthLimit); |
67 char* new_data = NewArray<char>(new_capacity); | 73 char* new_data = NewArray<char>(new_capacity); |
68 memcpy(new_data, data_, old_position); | 74 memcpy(new_data, data_, old_position); |
69 DeleteArray(data_); | 75 DeleteArray(data_); |
70 data_ = new_data; | 76 data_ = new_data; |
71 cursor_ = new_data + old_position; | 77 cursor_ = new_data + old_position; |
72 limit_ = ComputeLimit(new_data, new_capacity); | 78 limit_ = ComputeLimit(new_data, new_capacity); |
73 ASSERT(Capacity() == new_capacity && pos() == old_position); | 79 ASSERT(Capacity() == new_capacity && pos() == old_position); |
74 } | 80 } |
75 if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) { | 81 if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) { |
76 *cursor_++ = c; // Common case: 7-bit ASCII. | 82 *cursor_++ = c; // Common case: 7-bit ASCII. |
(...skipping 256 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
333 two_byte_string_buffer_.Initialize( | 339 two_byte_string_buffer_.Initialize( |
334 Handle<ExternalTwoByteString>::cast(source)); | 340 Handle<ExternalTwoByteString>::cast(source)); |
335 source_ = &two_byte_string_buffer_; | 341 source_ = &two_byte_string_buffer_; |
336 } else { | 342 } else { |
337 char_stream_buffer_.Initialize(source, stream); | 343 char_stream_buffer_.Initialize(source, stream); |
338 source_ = &char_stream_buffer_; | 344 source_ = &char_stream_buffer_; |
339 } | 345 } |
340 | 346 |
341 position_ = position; | 347 position_ = position; |
342 | 348 |
| 349 // Reset literals buffer |
| 350 literals_.Reset(); |
| 351 |
343 // Set c0_ (one character ahead) | 352 // Set c0_ (one character ahead) |
344 ASSERT(kCharacterLookaheadBufferSize == 1); | 353 ASSERT(kCharacterLookaheadBufferSize == 1); |
345 Advance(); | 354 Advance(); |
346 | 355 |
347 // Skip initial whitespace allowing HTML comment ends just like | 356 // Skip initial whitespace allowing HTML comment ends just like |
348 // after a newline and scan first token. | 357 // after a newline and scan first token. |
349 has_line_terminator_before_next_ = true; | 358 has_line_terminator_before_next_ = true; |
350 SkipWhiteSpace(); | 359 SkipWhiteSpace(); |
351 Scan(); | 360 Scan(); |
352 } | 361 } |
353 | 362 |
354 | 363 |
355 Handle<String> Scanner::SubString(int start, int end) { | 364 Handle<String> Scanner::SubString(int start, int end) { |
356 return source_->SubString(start - position_, end - position_); | 365 return source_->SubString(start - position_, end - position_); |
357 } | 366 } |
358 | 367 |
359 | 368 |
360 Token::Value Scanner::Next() { | 369 Token::Value Scanner::Next() { |
361 // BUG 1215673: Find a thread safe way to set a stack limit in | 370 // BUG 1215673: Find a thread safe way to set a stack limit in |
362 // pre-parse mode. Otherwise, we cannot safely pre-parse from other | 371 // pre-parse mode. Otherwise, we cannot safely pre-parse from other |
363 // threads. | 372 // threads. |
364 current_ = next_; | 373 current_ = next_; |
365 // Check for stack-overflow before returning any tokens. | 374 // Check for stack-overflow before returning any tokens. |
366 StackLimitCheck check; | 375 StackLimitCheck check; |
367 if (check.HasOverflowed()) { | 376 if (check.HasOverflowed()) { |
368 stack_overflow_ = true; | 377 stack_overflow_ = true; |
369 next_.token = Token::ILLEGAL; | 378 next_.token = Token::ILLEGAL; |
370 next_.literal_buffer = NULL; | |
371 } else { | 379 } else { |
372 Scan(); | 380 Scan(); |
373 } | 381 } |
374 return current_.token; | 382 return current_.token; |
375 } | 383 } |
376 | 384 |
377 | 385 |
378 void Scanner::StartLiteral() { | 386 void Scanner::StartLiteral() { |
379 // Use the first buffer unless it's currently in use by the current_ token. | 387 next_.literal_pos = literals_.pos(); |
380 // In most cases we won't have two literals/identifiers in a row, so | |
381 // the second buffer won't be used very often and is unlikely to grow much. | |
382 UTF8Buffer* free_buffer = | |
383 (current_.literal_buffer != &literal_buffer_1_) ? &literal_buffer_1_ | |
384 : &literal_buffer_2_; | |
385 next_.literal_buffer = free_buffer; | |
386 free_buffer->Reset(); | |
387 } | 388 } |
388 | 389 |
389 | 390 |
390 void Scanner::AddChar(uc32 c) { | 391 void Scanner::AddChar(uc32 c) { |
391 next_.literal_buffer->AddChar(c); | 392 literals_.AddChar(c); |
392 } | 393 } |
393 | 394 |
394 | 395 |
395 void Scanner::TerminateLiteral() { | 396 void Scanner::TerminateLiteral() { |
| 397 next_.literal_end = literals_.pos(); |
396 AddChar(0); | 398 AddChar(0); |
397 } | 399 } |
398 | 400 |
399 | 401 |
400 void Scanner::AddCharAdvance() { | 402 void Scanner::AddCharAdvance() { |
401 AddChar(c0_); | 403 AddChar(c0_); |
402 Advance(); | 404 Advance(); |
403 } | 405 } |
404 | 406 |
405 | 407 |
(...skipping 681 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1087 } | 1089 } |
1088 AddCharAdvance(); | 1090 AddCharAdvance(); |
1089 } | 1091 } |
1090 TerminateLiteral(); | 1092 TerminateLiteral(); |
1091 | 1093 |
1092 next_.location.end_pos = source_pos() - 1; | 1094 next_.location.end_pos = source_pos() - 1; |
1093 return true; | 1095 return true; |
1094 } | 1096 } |
1095 | 1097 |
1096 } } // namespace v8::internal | 1098 } } // namespace v8::internal |
OLD | NEW |