| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "src/parsing/scanner-character-streams.h" | 5 #include "src/parsing/scanner-character-streams.h" |
| 6 | 6 |
| 7 #include "include/v8.h" | 7 #include "include/v8.h" |
| 8 #include "src/globals.h" | 8 #include "src/globals.h" |
| 9 #include "src/handles.h" | 9 #include "src/handles.h" |
| 10 #include "src/list-inl.h" // TODO(mstarzinger): Temporary cycle breaker! | 10 #include "src/list-inl.h" // TODO(mstarzinger): Temporary cycle breaker! |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 42 return to_fill; | 42 return to_fill; |
| 43 } | 43 } |
| 44 | 44 |
| 45 } // namespace | 45 } // namespace |
| 46 | 46 |
| 47 | 47 |
| 48 // ---------------------------------------------------------------------------- | 48 // ---------------------------------------------------------------------------- |
| 49 // BufferedUtf16CharacterStreams | 49 // BufferedUtf16CharacterStreams |
| 50 | 50 |
| 51 BufferedUtf16CharacterStream::BufferedUtf16CharacterStream() | 51 BufferedUtf16CharacterStream::BufferedUtf16CharacterStream() |
| 52 : Utf16CharacterStream(), | 52 : Utf16CharacterStream(), pushback_limit_(NULL) { |
| 53 pushback_limit_(NULL) { | |
| 54 // Initialize buffer as being empty. First read will fill the buffer. | 53 // Initialize buffer as being empty. First read will fill the buffer. |
| 55 buffer_cursor_ = buffer_; | 54 buffer_cursor_ = buffer_; |
| 56 buffer_end_ = buffer_; | 55 buffer_end_ = buffer_; |
| 57 } | 56 } |
| 58 | 57 |
| 59 | 58 |
| 60 BufferedUtf16CharacterStream::~BufferedUtf16CharacterStream() { } | 59 BufferedUtf16CharacterStream::~BufferedUtf16CharacterStream() {} |
| 61 | 60 |
| 62 void BufferedUtf16CharacterStream::PushBack(uc32 character) { | 61 void BufferedUtf16CharacterStream::PushBack(uc32 character) { |
| 63 if (character == kEndOfInput) { | 62 if (character == kEndOfInput) { |
| 64 pos_--; | 63 pos_--; |
| 65 return; | 64 return; |
| 66 } | 65 } |
| 67 if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) { | 66 if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) { |
| 68 // buffer_ is writable, buffer_cursor_ is const pointer. | 67 // buffer_ is writable, buffer_cursor_ is const pointer. |
| 69 buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character); | 68 buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character); |
| 70 pos_--; | 69 pos_--; |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 130 | 129 |
| 131 | 130 |
| 132 GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( | 131 GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( |
| 133 Handle<String> data, size_t start_position, size_t end_position) | 132 Handle<String> data, size_t start_position, size_t end_position) |
| 134 : string_(data), length_(end_position), bookmark_(kNoBookmark) { | 133 : string_(data), length_(end_position), bookmark_(kNoBookmark) { |
| 135 DCHECK(end_position >= start_position); | 134 DCHECK(end_position >= start_position); |
| 136 pos_ = start_position; | 135 pos_ = start_position; |
| 137 } | 136 } |
| 138 | 137 |
| 139 | 138 |
| 140 GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() { } | 139 GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() {} |
| 141 | 140 |
| 142 | 141 |
| 143 bool GenericStringUtf16CharacterStream::SetBookmark() { | 142 bool GenericStringUtf16CharacterStream::SetBookmark() { |
| 144 bookmark_ = pos_; | 143 bookmark_ = pos_; |
| 145 return true; | 144 return true; |
| 146 } | 145 } |
| 147 | 146 |
| 148 | 147 |
| 149 void GenericStringUtf16CharacterStream::ResetToBookmark() { | 148 void GenericStringUtf16CharacterStream::ResetToBookmark() { |
| 150 DCHECK(bookmark_ != kNoBookmark); | 149 DCHECK(bookmark_ != kNoBookmark); |
| (...skipping 29 matching lines...) Expand all Loading... |
| 180 size_t length) | 179 size_t length) |
| 181 : BufferedUtf16CharacterStream(), | 180 : BufferedUtf16CharacterStream(), |
| 182 raw_data_(data), | 181 raw_data_(data), |
| 183 raw_data_length_(length), | 182 raw_data_length_(length), |
| 184 raw_data_pos_(0), | 183 raw_data_pos_(0), |
| 185 raw_character_position_(0) { | 184 raw_character_position_(0) { |
| 186 ReadBlock(); | 185 ReadBlock(); |
| 187 } | 186 } |
| 188 | 187 |
| 189 | 188 |
| 190 Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() { } | 189 Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() {} |
| 191 | 190 |
| 192 | 191 |
| 193 size_t Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, size_t length, | 192 size_t Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, size_t length, |
| 194 const byte* src, size_t* src_pos, | 193 const byte* src, size_t* src_pos, |
| 195 size_t src_length) { | 194 size_t src_length) { |
| 196 static const unibrow::uchar kMaxUtf16Character = 0xffff; | 195 static const unibrow::uchar kMaxUtf16Character = 0xffff; |
| 197 size_t i = 0; | 196 size_t i = 0; |
| 198 // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer | 197 // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer |
| 199 // one character early (in the normal case), because we need to have at least | 198 // one character early (in the normal case), because we need to have at least |
| 200 // two free spaces in the buffer to be sure that the next character will fit. | 199 // two free spaces in the buffer to be sure that the next character will fit. |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 261 | 260 |
| 262 // Move the cursor back to point at the preceding UTF-8 character start | 261 // Move the cursor back to point at the preceding UTF-8 character start |
| 263 // in the buffer. | 262 // in the buffer. |
| 264 static inline void Utf8CharacterBack(const byte* buffer, size_t* cursor) { | 263 static inline void Utf8CharacterBack(const byte* buffer, size_t* cursor) { |
| 265 byte character = buffer[--*cursor]; | 264 byte character = buffer[--*cursor]; |
| 266 if (character > unibrow::Utf8::kMaxOneByteChar) { | 265 if (character > unibrow::Utf8::kMaxOneByteChar) { |
| 267 DCHECK(IsUtf8MultiCharacterFollower(character)); | 266 DCHECK(IsUtf8MultiCharacterFollower(character)); |
| 268 // Last byte of a multi-byte character encoding. Step backwards until | 267 // Last byte of a multi-byte character encoding. Step backwards until |
| 269 // pointing to the first byte of the encoding, recognized by having the | 268 // pointing to the first byte of the encoding, recognized by having the |
| 270 // top two bits set. | 269 // top two bits set. |
| 271 while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { } | 270 while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { |
| 271 } |
| 272 DCHECK(IsUtf8MultiCharacterStart(buffer[*cursor])); | 272 DCHECK(IsUtf8MultiCharacterStart(buffer[*cursor])); |
| 273 } | 273 } |
| 274 } | 274 } |
| 275 | 275 |
| 276 | 276 |
| 277 // Move the cursor forward to point at the next following UTF-8 character start | 277 // Move the cursor forward to point at the next following UTF-8 character start |
| 278 // in the buffer. | 278 // in the buffer. |
| 279 static inline void Utf8CharacterForward(const byte* buffer, size_t* cursor) { | 279 static inline void Utf8CharacterForward(const byte* buffer, size_t* cursor) { |
| 280 byte character = buffer[(*cursor)++]; | 280 byte character = buffer[(*cursor)++]; |
| 281 if (character > unibrow::Utf8::kMaxOneByteChar) { | 281 if (character > unibrow::Utf8::kMaxOneByteChar) { |
| (...skipping 267 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 549 for (size_t i = 0; i < utf8_split_char_buffer_length_; ++i) { | 549 for (size_t i = 0; i < utf8_split_char_buffer_length_; ++i) { |
| 550 utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i]; | 550 utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i]; |
| 551 } | 551 } |
| 552 } | 552 } |
| 553 | 553 |
| 554 | 554 |
| 555 // ---------------------------------------------------------------------------- | 555 // ---------------------------------------------------------------------------- |
| 556 // ExternalTwoByteStringUtf16CharacterStream | 556 // ExternalTwoByteStringUtf16CharacterStream |
| 557 | 557 |
| 558 ExternalTwoByteStringUtf16CharacterStream:: | 558 ExternalTwoByteStringUtf16CharacterStream:: |
| 559 ~ExternalTwoByteStringUtf16CharacterStream() { } | 559 ~ExternalTwoByteStringUtf16CharacterStream() {} |
| 560 | 560 |
| 561 | 561 |
| 562 ExternalTwoByteStringUtf16CharacterStream:: | 562 ExternalTwoByteStringUtf16CharacterStream:: |
| 563 ExternalTwoByteStringUtf16CharacterStream( | 563 ExternalTwoByteStringUtf16CharacterStream( |
| 564 Handle<ExternalTwoByteString> data, int start_position, | 564 Handle<ExternalTwoByteString> data, int start_position, |
| 565 int end_position) | 565 int end_position) |
| 566 : Utf16CharacterStream(), | 566 : Utf16CharacterStream(), |
| 567 source_(data), | 567 source_(data), |
| 568 raw_data_(data->GetTwoByteData(start_position)), | 568 raw_data_(data->GetTwoByteData(start_position)), |
| 569 bookmark_(kNoBookmark) { | 569 bookmark_(kNoBookmark) { |
| 570 buffer_cursor_ = raw_data_, | 570 buffer_cursor_ = raw_data_, |
| 571 buffer_end_ = raw_data_ + (end_position - start_position); | 571 buffer_end_ = raw_data_ + (end_position - start_position); |
| 572 pos_ = start_position; | 572 pos_ = start_position; |
| 573 } | 573 } |
| 574 | 574 |
| 575 | 575 |
| 576 bool ExternalTwoByteStringUtf16CharacterStream::SetBookmark() { | 576 bool ExternalTwoByteStringUtf16CharacterStream::SetBookmark() { |
| 577 bookmark_ = pos_; | 577 bookmark_ = pos_; |
| 578 return true; | 578 return true; |
| 579 } | 579 } |
| 580 | 580 |
| 581 | 581 |
| 582 void ExternalTwoByteStringUtf16CharacterStream::ResetToBookmark() { | 582 void ExternalTwoByteStringUtf16CharacterStream::ResetToBookmark() { |
| 583 DCHECK(bookmark_ != kNoBookmark); | 583 DCHECK(bookmark_ != kNoBookmark); |
| 584 pos_ = bookmark_; | 584 pos_ = bookmark_; |
| 585 buffer_cursor_ = raw_data_ + bookmark_; | 585 buffer_cursor_ = raw_data_ + bookmark_; |
| 586 } | 586 } |
| 587 } // namespace internal | 587 } // namespace internal |
| 588 } // namespace v8 | 588 } // namespace v8 |
| OLD | NEW |