| OLD | NEW |
| 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
| 2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
| 3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
| 4 // met: | 4 // met: |
| 5 // | 5 // |
| 6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
| 7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
| 8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
| 9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
| 10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 42 unibrow::Predicate<unibrow::LineTerminator, 128> Scanner::kIsLineTerminator; | 42 unibrow::Predicate<unibrow::LineTerminator, 128> Scanner::kIsLineTerminator; |
| 43 unibrow::Predicate<unibrow::WhiteSpace, 128> Scanner::kIsWhiteSpace; | 43 unibrow::Predicate<unibrow::WhiteSpace, 128> Scanner::kIsWhiteSpace; |
| 44 | 44 |
| 45 | 45 |
| 46 StaticResource<Scanner::Utf8Decoder> Scanner::utf8_decoder_; | 46 StaticResource<Scanner::Utf8Decoder> Scanner::utf8_decoder_; |
| 47 | 47 |
| 48 | 48 |
| 49 // ---------------------------------------------------------------------------- | 49 // ---------------------------------------------------------------------------- |
| 50 // UTF8Buffer | 50 // UTF8Buffer |
| 51 | 51 |
| 52 UTF8Buffer::UTF8Buffer() : data_(NULL), limit_(NULL) { } | 52 UTF8Buffer::UTF8Buffer() { |
| 53 static const int kInitialCapacity = 1 * KB; |
| 54 data_ = NewArray<char>(kInitialCapacity); |
| 55 limit_ = ComputeLimit(data_, kInitialCapacity); |
| 56 Reset(); |
| 57 ASSERT(Capacity() == kInitialCapacity && pos() == 0); |
| 58 } |
| 53 | 59 |
| 54 | 60 |
| 55 UTF8Buffer::~UTF8Buffer() { | 61 UTF8Buffer::~UTF8Buffer() { |
| 56 DeleteArray(data_); | 62 DeleteArray(data_); |
| 57 } | 63 } |
| 58 | 64 |
| 59 | 65 |
| 60 void UTF8Buffer::AddCharSlow(uc32 c) { | 66 void UTF8Buffer::AddCharSlow(uc32 c) { |
| 61 static const int kCapacityGrowthLimit = 1 * MB; | 67 static const int kCapacityGrowthLimit = 1 * MB; |
| 62 if (cursor_ > limit_) { | 68 if (cursor_ > limit_) { |
| 63 int old_capacity = Capacity(); | 69 int old_capacity = Capacity(); |
| 64 int old_position = pos(); | 70 int old_position = pos(); |
| 65 int new_capacity = | 71 int new_capacity = |
| 66 Min(old_capacity * 3, old_capacity + kCapacityGrowthLimit); | 72 Min(old_capacity * 2, old_capacity + kCapacityGrowthLimit); |
| 67 char* new_data = NewArray<char>(new_capacity); | 73 char* new_data = NewArray<char>(new_capacity); |
| 68 memcpy(new_data, data_, old_position); | 74 memcpy(new_data, data_, old_position); |
| 69 DeleteArray(data_); | 75 DeleteArray(data_); |
| 70 data_ = new_data; | 76 data_ = new_data; |
| 71 cursor_ = new_data + old_position; | 77 cursor_ = new_data + old_position; |
| 72 limit_ = ComputeLimit(new_data, new_capacity); | 78 limit_ = ComputeLimit(new_data, new_capacity); |
| 73 ASSERT(Capacity() == new_capacity && pos() == old_position); | 79 ASSERT(Capacity() == new_capacity && pos() == old_position); |
| 74 } | 80 } |
| 75 if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) { | 81 if (static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) { |
| 76 *cursor_++ = c; // Common case: 7-bit ASCII. | 82 *cursor_++ = c; // Common case: 7-bit ASCII. |
| (...skipping 256 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 333 two_byte_string_buffer_.Initialize( | 339 two_byte_string_buffer_.Initialize( |
| 334 Handle<ExternalTwoByteString>::cast(source)); | 340 Handle<ExternalTwoByteString>::cast(source)); |
| 335 source_ = &two_byte_string_buffer_; | 341 source_ = &two_byte_string_buffer_; |
| 336 } else { | 342 } else { |
| 337 char_stream_buffer_.Initialize(source, stream); | 343 char_stream_buffer_.Initialize(source, stream); |
| 338 source_ = &char_stream_buffer_; | 344 source_ = &char_stream_buffer_; |
| 339 } | 345 } |
| 340 | 346 |
| 341 position_ = position; | 347 position_ = position; |
| 342 | 348 |
| 349 // Reset literals buffer |
| 350 literals_.Reset(); |
| 351 |
| 343 // Set c0_ (one character ahead) | 352 // Set c0_ (one character ahead) |
| 344 ASSERT(kCharacterLookaheadBufferSize == 1); | 353 ASSERT(kCharacterLookaheadBufferSize == 1); |
| 345 Advance(); | 354 Advance(); |
| 346 | 355 |
| 347 // Skip initial whitespace allowing HTML comment ends just like | 356 // Skip initial whitespace allowing HTML comment ends just like |
| 348 // after a newline and scan first token. | 357 // after a newline and scan first token. |
| 349 has_line_terminator_before_next_ = true; | 358 has_line_terminator_before_next_ = true; |
| 350 SkipWhiteSpace(); | 359 SkipWhiteSpace(); |
| 351 Scan(); | 360 Scan(); |
| 352 } | 361 } |
| 353 | 362 |
| 354 | 363 |
| 355 Handle<String> Scanner::SubString(int start, int end) { | 364 Handle<String> Scanner::SubString(int start, int end) { |
| 356 return source_->SubString(start - position_, end - position_); | 365 return source_->SubString(start - position_, end - position_); |
| 357 } | 366 } |
| 358 | 367 |
| 359 | 368 |
| 360 Token::Value Scanner::Next() { | 369 Token::Value Scanner::Next() { |
| 361 // BUG 1215673: Find a thread safe way to set a stack limit in | 370 // BUG 1215673: Find a thread safe way to set a stack limit in |
| 362 // pre-parse mode. Otherwise, we cannot safely pre-parse from other | 371 // pre-parse mode. Otherwise, we cannot safely pre-parse from other |
| 363 // threads. | 372 // threads. |
| 364 current_ = next_; | 373 current_ = next_; |
| 365 // Check for stack-overflow before returning any tokens. | 374 // Check for stack-overflow before returning any tokens. |
| 366 StackLimitCheck check; | 375 StackLimitCheck check; |
| 367 if (check.HasOverflowed()) { | 376 if (check.HasOverflowed()) { |
| 368 stack_overflow_ = true; | 377 stack_overflow_ = true; |
| 369 next_.token = Token::ILLEGAL; | 378 next_.token = Token::ILLEGAL; |
| 370 next_.literal_buffer = NULL; | |
| 371 } else { | 379 } else { |
| 372 Scan(); | 380 Scan(); |
| 373 } | 381 } |
| 374 return current_.token; | 382 return current_.token; |
| 375 } | 383 } |
| 376 | 384 |
| 377 | 385 |
| 378 void Scanner::StartLiteral() { | 386 void Scanner::StartLiteral() { |
| 379 // Use the first buffer unless it's currently in use by the current_ token. | 387 next_.literal_pos = literals_.pos(); |
| 380 // In most cases we won't have two literals/identifiers in a row, so | |
| 381 // the second buffer won't be used very often and is unlikely to grow much. | |
| 382 UTF8Buffer* free_buffer = | |
| 383 (current_.literal_buffer != &literal_buffer_1_) ? &literal_buffer_1_ | |
| 384 : &literal_buffer_2_; | |
| 385 next_.literal_buffer = free_buffer; | |
| 386 free_buffer->Reset(); | |
| 387 } | 388 } |
| 388 | 389 |
| 389 | 390 |
| 390 void Scanner::AddChar(uc32 c) { | 391 void Scanner::AddChar(uc32 c) { |
| 391 next_.literal_buffer->AddChar(c); | 392 literals_.AddChar(c); |
| 392 } | 393 } |
| 393 | 394 |
| 394 | 395 |
| 395 void Scanner::TerminateLiteral() { | 396 void Scanner::TerminateLiteral() { |
| 397 next_.literal_end = literals_.pos(); |
| 396 AddChar(0); | 398 AddChar(0); |
| 397 } | 399 } |
| 398 | 400 |
| 399 | 401 |
| 400 void Scanner::AddCharAdvance() { | 402 void Scanner::AddCharAdvance() { |
| 401 AddChar(c0_); | 403 AddChar(c0_); |
| 402 Advance(); | 404 Advance(); |
| 403 } | 405 } |
| 404 | 406 |
| 405 | 407 |
| (...skipping 681 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1087 } | 1089 } |
| 1088 AddCharAdvance(); | 1090 AddCharAdvance(); |
| 1089 } | 1091 } |
| 1090 TerminateLiteral(); | 1092 TerminateLiteral(); |
| 1091 | 1093 |
| 1092 next_.location.end_pos = source_pos() - 1; | 1094 next_.location.end_pos = source_pos() - 1; |
| 1093 return true; | 1095 return true; |
| 1094 } | 1096 } |
| 1095 | 1097 |
| 1096 } } // namespace v8::internal | 1098 } } // namespace v8::internal |
| OLD | NEW |