OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 | 8 |
9 #include <cmath> | 9 #include <cmath> |
10 | 10 |
(...skipping 220 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
231 Token::ILLEGAL | 231 Token::ILLEGAL |
232 }; | 232 }; |
233 | 233 |
234 | 234 |
235 Token::Value Scanner::Next() { | 235 Token::Value Scanner::Next() { |
236 if (next_.token == Token::EOS) { | 236 if (next_.token == Token::EOS) { |
237 next_.location.beg_pos = current_.location.beg_pos; | 237 next_.location.beg_pos = current_.location.beg_pos; |
238 next_.location.end_pos = current_.location.end_pos; | 238 next_.location.end_pos = current_.location.end_pos; |
239 } | 239 } |
240 current_ = next_; | 240 current_ = next_; |
| 241 if (uber_next_.token != Token::UNINITIALIZED) { |
| 242 next_ = uber_next_; |
| 243 uber_next_.token = Token::UNINITIALIZED; |
| 244 return current_.token; |
| 245 } |
241 has_line_terminator_before_next_ = false; | 246 has_line_terminator_before_next_ = false; |
242 has_multiline_comment_before_next_ = false; | 247 has_multiline_comment_before_next_ = false; |
243 if (static_cast<unsigned>(c0_) <= 0x7f) { | 248 if (static_cast<unsigned>(c0_) <= 0x7f) { |
244 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); | 249 Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); |
245 if (token != Token::ILLEGAL) { | 250 if (token != Token::ILLEGAL) { |
246 int pos = source_pos(); | 251 int pos = source_pos(); |
247 next_.token = token; | 252 next_.token = token; |
248 next_.location.beg_pos = pos; | 253 next_.location.beg_pos = pos; |
249 next_.location.end_pos = pos + 1; | 254 next_.location.end_pos = pos + 1; |
250 Advance(); | 255 Advance(); |
251 return current_.token; | 256 return current_.token; |
252 } | 257 } |
253 } | 258 } |
254 Scan(); | 259 Scan(); |
255 return current_.token; | 260 return current_.token; |
256 } | 261 } |
257 | 262 |
258 | 263 |
| 264 Token::Value Scanner::PeekAhead() { |
| 265 if (uber_next_.token != Token::UNINITIALIZED) { |
| 266 return uber_next_.token; |
| 267 } |
| 268 TokenDesc prev = current_; |
| 269 Next(); |
| 270 Token::Value ret = next_.token; |
| 271 uber_next_ = next_; |
| 272 next_ = current_; |
| 273 current_ = prev; |
| 274 return ret; |
| 275 } |
| 276 |
| 277 |
259 // TODO(yangguo): check whether this is actually necessary. | 278 // TODO(yangguo): check whether this is actually necessary. |
260 static inline bool IsLittleEndianByteOrderMark(uc32 c) { | 279 static inline bool IsLittleEndianByteOrderMark(uc32 c) { |
261 // The Unicode value U+FFFE is guaranteed never to be assigned as a | 280 // The Unicode value U+FFFE is guaranteed never to be assigned as a |
262 // Unicode character; this implies that in a Unicode context the | 281 // Unicode character; this implies that in a Unicode context the |
263 // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF | 282 // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF |
264 // character expressed in little-endian byte order (since it could | 283 // character expressed in little-endian byte order (since it could |
265 // not be a U+FFFE character expressed in big-endian byte | 284 // not be a U+FFFE character expressed in big-endian byte |
266 // order). Nevertheless, we check for it to be compatible with | 285 // order). Nevertheless, we check for it to be compatible with |
267 // Spidermonkey. | 286 // Spidermonkey. |
268 return c == 0xFFFE; | 287 return c == 0xFFFE; |
(...skipping 1346 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1615 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1634 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1616 } | 1635 } |
1617 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1636 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1618 | 1637 |
1619 backing_store_.AddBlock(bytes); | 1638 backing_store_.AddBlock(bytes); |
1620 return backing_store_.EndSequence().start(); | 1639 return backing_store_.EndSequence().start(); |
1621 } | 1640 } |
1622 | 1641 |
1623 } // namespace internal | 1642 } // namespace internal |
1624 } // namespace v8 | 1643 } // namespace v8 |
OLD | NEW |