OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "src/parsing/scanner-character-streams.h" | 5 #include "src/parsing/scanner-character-streams.h" |
6 | 6 |
7 #include "include/v8.h" | 7 #include "include/v8.h" |
8 #include "src/globals.h" | 8 #include "src/globals.h" |
9 #include "src/handles.h" | 9 #include "src/handles.h" |
10 #include "src/list-inl.h" // TODO(mstarzinger): Temporary cycle breaker! | 10 #include "src/list-inl.h" // TODO(mstarzinger): Temporary cycle breaker! |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
42 return to_fill; | 42 return to_fill; |
43 } | 43 } |
44 | 44 |
45 } // namespace | 45 } // namespace |
46 | 46 |
47 | 47 |
48 // ---------------------------------------------------------------------------- | 48 // ---------------------------------------------------------------------------- |
49 // BufferedUtf16CharacterStreams | 49 // BufferedUtf16CharacterStreams |
50 | 50 |
51 BufferedUtf16CharacterStream::BufferedUtf16CharacterStream() | 51 BufferedUtf16CharacterStream::BufferedUtf16CharacterStream() |
52 : Utf16CharacterStream(), | 52 : Utf16CharacterStream(), pushback_limit_(NULL) { |
53 pushback_limit_(NULL) { | |
54 // Initialize buffer as being empty. First read will fill the buffer. | 53 // Initialize buffer as being empty. First read will fill the buffer. |
55 buffer_cursor_ = buffer_; | 54 buffer_cursor_ = buffer_; |
56 buffer_end_ = buffer_; | 55 buffer_end_ = buffer_; |
57 } | 56 } |
58 | 57 |
59 | 58 |
60 BufferedUtf16CharacterStream::~BufferedUtf16CharacterStream() { } | 59 BufferedUtf16CharacterStream::~BufferedUtf16CharacterStream() {} |
61 | 60 |
62 void BufferedUtf16CharacterStream::PushBack(uc32 character) { | 61 void BufferedUtf16CharacterStream::PushBack(uc32 character) { |
63 if (character == kEndOfInput) { | 62 if (character == kEndOfInput) { |
64 pos_--; | 63 pos_--; |
65 return; | 64 return; |
66 } | 65 } |
67 if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) { | 66 if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) { |
68 // buffer_ is writable, buffer_cursor_ is const pointer. | 67 // buffer_ is writable, buffer_cursor_ is const pointer. |
69 buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character); | 68 buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character); |
70 pos_--; | 69 pos_--; |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
130 | 129 |
131 | 130 |
132 GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( | 131 GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( |
133 Handle<String> data, size_t start_position, size_t end_position) | 132 Handle<String> data, size_t start_position, size_t end_position) |
134 : string_(data), length_(end_position), bookmark_(kNoBookmark) { | 133 : string_(data), length_(end_position), bookmark_(kNoBookmark) { |
135 DCHECK(end_position >= start_position); | 134 DCHECK(end_position >= start_position); |
136 pos_ = start_position; | 135 pos_ = start_position; |
137 } | 136 } |
138 | 137 |
139 | 138 |
140 GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() { } | 139 GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() {} |
141 | 140 |
142 | 141 |
143 bool GenericStringUtf16CharacterStream::SetBookmark() { | 142 bool GenericStringUtf16CharacterStream::SetBookmark() { |
144 bookmark_ = pos_; | 143 bookmark_ = pos_; |
145 return true; | 144 return true; |
146 } | 145 } |
147 | 146 |
148 | 147 |
149 void GenericStringUtf16CharacterStream::ResetToBookmark() { | 148 void GenericStringUtf16CharacterStream::ResetToBookmark() { |
150 DCHECK(bookmark_ != kNoBookmark); | 149 DCHECK(bookmark_ != kNoBookmark); |
(...skipping 29 matching lines...) Expand all Loading... |
180 size_t length) | 179 size_t length) |
181 : BufferedUtf16CharacterStream(), | 180 : BufferedUtf16CharacterStream(), |
182 raw_data_(data), | 181 raw_data_(data), |
183 raw_data_length_(length), | 182 raw_data_length_(length), |
184 raw_data_pos_(0), | 183 raw_data_pos_(0), |
185 raw_character_position_(0) { | 184 raw_character_position_(0) { |
186 ReadBlock(); | 185 ReadBlock(); |
187 } | 186 } |
188 | 187 |
189 | 188 |
190 Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() { } | 189 Utf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() {} |
191 | 190 |
192 | 191 |
193 size_t Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, size_t length, | 192 size_t Utf8ToUtf16CharacterStream::CopyChars(uint16_t* dest, size_t length, |
194 const byte* src, size_t* src_pos, | 193 const byte* src, size_t* src_pos, |
195 size_t src_length) { | 194 size_t src_length) { |
196 static const unibrow::uchar kMaxUtf16Character = 0xffff; | 195 static const unibrow::uchar kMaxUtf16Character = 0xffff; |
197 size_t i = 0; | 196 size_t i = 0; |
198 // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer | 197 // Because of the UTF-16 lead and trail surrogates, we stop filling the buffer |
199 // one character early (in the normal case), because we need to have at least | 198 // one character early (in the normal case), because we need to have at least |
200 // two free spaces in the buffer to be sure that the next character will fit. | 199 // two free spaces in the buffer to be sure that the next character will fit. |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
261 | 260 |
262 // Move the cursor back to point at the preceding UTF-8 character start | 261 // Move the cursor back to point at the preceding UTF-8 character start |
263 // in the buffer. | 262 // in the buffer. |
264 static inline void Utf8CharacterBack(const byte* buffer, size_t* cursor) { | 263 static inline void Utf8CharacterBack(const byte* buffer, size_t* cursor) { |
265 byte character = buffer[--*cursor]; | 264 byte character = buffer[--*cursor]; |
266 if (character > unibrow::Utf8::kMaxOneByteChar) { | 265 if (character > unibrow::Utf8::kMaxOneByteChar) { |
267 DCHECK(IsUtf8MultiCharacterFollower(character)); | 266 DCHECK(IsUtf8MultiCharacterFollower(character)); |
268 // Last byte of a multi-byte character encoding. Step backwards until | 267 // Last byte of a multi-byte character encoding. Step backwards until |
269 // pointing to the first byte of the encoding, recognized by having the | 268 // pointing to the first byte of the encoding, recognized by having the |
270 // top two bits set. | 269 // top two bits set. |
271 while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { } | 270 while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { |
| 271 } |
272 DCHECK(IsUtf8MultiCharacterStart(buffer[*cursor])); | 272 DCHECK(IsUtf8MultiCharacterStart(buffer[*cursor])); |
273 } | 273 } |
274 } | 274 } |
275 | 275 |
276 | 276 |
277 // Move the cursor forward to point at the next following UTF-8 character start | 277 // Move the cursor forward to point at the next following UTF-8 character start |
278 // in the buffer. | 278 // in the buffer. |
279 static inline void Utf8CharacterForward(const byte* buffer, size_t* cursor) { | 279 static inline void Utf8CharacterForward(const byte* buffer, size_t* cursor) { |
280 byte character = buffer[(*cursor)++]; | 280 byte character = buffer[(*cursor)++]; |
281 if (character > unibrow::Utf8::kMaxOneByteChar) { | 281 if (character > unibrow::Utf8::kMaxOneByteChar) { |
(...skipping 267 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
549 for (size_t i = 0; i < utf8_split_char_buffer_length_; ++i) { | 549 for (size_t i = 0; i < utf8_split_char_buffer_length_; ++i) { |
550 utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i]; | 550 utf8_split_char_buffer_[i] = current_data_[current_data_length_ + i]; |
551 } | 551 } |
552 } | 552 } |
553 | 553 |
554 | 554 |
555 // ---------------------------------------------------------------------------- | 555 // ---------------------------------------------------------------------------- |
556 // ExternalTwoByteStringUtf16CharacterStream | 556 // ExternalTwoByteStringUtf16CharacterStream |
557 | 557 |
558 ExternalTwoByteStringUtf16CharacterStream:: | 558 ExternalTwoByteStringUtf16CharacterStream:: |
559 ~ExternalTwoByteStringUtf16CharacterStream() { } | 559 ~ExternalTwoByteStringUtf16CharacterStream() {} |
560 | 560 |
561 | 561 |
562 ExternalTwoByteStringUtf16CharacterStream:: | 562 ExternalTwoByteStringUtf16CharacterStream:: |
563 ExternalTwoByteStringUtf16CharacterStream( | 563 ExternalTwoByteStringUtf16CharacterStream( |
564 Handle<ExternalTwoByteString> data, int start_position, | 564 Handle<ExternalTwoByteString> data, int start_position, |
565 int end_position) | 565 int end_position) |
566 : Utf16CharacterStream(), | 566 : Utf16CharacterStream(), |
567 source_(data), | 567 source_(data), |
568 raw_data_(data->GetTwoByteData(start_position)), | 568 raw_data_(data->GetTwoByteData(start_position)), |
569 bookmark_(kNoBookmark) { | 569 bookmark_(kNoBookmark) { |
570 buffer_cursor_ = raw_data_, | 570 buffer_cursor_ = raw_data_, |
571 buffer_end_ = raw_data_ + (end_position - start_position); | 571 buffer_end_ = raw_data_ + (end_position - start_position); |
572 pos_ = start_position; | 572 pos_ = start_position; |
573 } | 573 } |
574 | 574 |
575 | 575 |
576 bool ExternalTwoByteStringUtf16CharacterStream::SetBookmark() { | 576 bool ExternalTwoByteStringUtf16CharacterStream::SetBookmark() { |
577 bookmark_ = pos_; | 577 bookmark_ = pos_; |
578 return true; | 578 return true; |
579 } | 579 } |
580 | 580 |
581 | 581 |
582 void ExternalTwoByteStringUtf16CharacterStream::ResetToBookmark() { | 582 void ExternalTwoByteStringUtf16CharacterStream::ResetToBookmark() { |
583 DCHECK(bookmark_ != kNoBookmark); | 583 DCHECK(bookmark_ != kNoBookmark); |
584 pos_ = bookmark_; | 584 pos_ = bookmark_; |
585 buffer_cursor_ = raw_data_ + bookmark_; | 585 buffer_cursor_ = raw_data_ + bookmark_; |
586 } | 586 } |
587 } // namespace internal | 587 } // namespace internal |
588 } // namespace v8 | 588 } // namespace v8 |
OLD | NEW |