OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
9 | 9 |
10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
(...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
204 INLINE(void AddChar(uint32_t code_unit)) { | 204 INLINE(void AddChar(uint32_t code_unit)) { |
205 if (position_ >= backing_store_.length()) ExpandBuffer(); | 205 if (position_ >= backing_store_.length()) ExpandBuffer(); |
206 if (is_one_byte_) { | 206 if (is_one_byte_) { |
207 if (code_unit <= unibrow::Latin1::kMaxChar) { | 207 if (code_unit <= unibrow::Latin1::kMaxChar) { |
208 backing_store_[position_] = static_cast<byte>(code_unit); | 208 backing_store_[position_] = static_cast<byte>(code_unit); |
209 position_ += kOneByteSize; | 209 position_ += kOneByteSize; |
210 return; | 210 return; |
211 } | 211 } |
212 ConvertToTwoByte(); | 212 ConvertToTwoByte(); |
213 } | 213 } |
214 ASSERT(code_unit < 0x10000u); | 214 DCHECK(code_unit < 0x10000u); |
215 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit; | 215 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit; |
216 position_ += kUC16Size; | 216 position_ += kUC16Size; |
217 } | 217 } |
218 | 218 |
219 bool is_one_byte() const { return is_one_byte_; } | 219 bool is_one_byte() const { return is_one_byte_; } |
220 | 220 |
221 bool is_contextual_keyword(Vector<const char> keyword) const { | 221 bool is_contextual_keyword(Vector<const char> keyword) const { |
222 return is_one_byte() && keyword.length() == position_ && | 222 return is_one_byte() && keyword.length() == position_ && |
223 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); | 223 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); |
224 } | 224 } |
225 | 225 |
226 Vector<const uint16_t> two_byte_literal() const { | 226 Vector<const uint16_t> two_byte_literal() const { |
227 ASSERT(!is_one_byte_); | 227 DCHECK(!is_one_byte_); |
228 ASSERT((position_ & 0x1) == 0); | 228 DCHECK((position_ & 0x1) == 0); |
229 return Vector<const uint16_t>( | 229 return Vector<const uint16_t>( |
230 reinterpret_cast<const uint16_t*>(backing_store_.start()), | 230 reinterpret_cast<const uint16_t*>(backing_store_.start()), |
231 position_ >> 1); | 231 position_ >> 1); |
232 } | 232 } |
233 | 233 |
234 Vector<const uint8_t> one_byte_literal() const { | 234 Vector<const uint8_t> one_byte_literal() const { |
235 ASSERT(is_one_byte_); | 235 DCHECK(is_one_byte_); |
236 return Vector<const uint8_t>( | 236 return Vector<const uint8_t>( |
237 reinterpret_cast<const uint8_t*>(backing_store_.start()), | 237 reinterpret_cast<const uint8_t*>(backing_store_.start()), |
238 position_); | 238 position_); |
239 } | 239 } |
240 | 240 |
241 int length() const { | 241 int length() const { |
242 return is_one_byte_ ? position_ : (position_ >> 1); | 242 return is_one_byte_ ? position_ : (position_ >> 1); |
243 } | 243 } |
244 | 244 |
245 void Reset() { | 245 void Reset() { |
(...skipping 15 matching lines...) Expand all Loading... |
261 } | 261 } |
262 | 262 |
263 void ExpandBuffer() { | 263 void ExpandBuffer() { |
264 Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity)); | 264 Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity)); |
265 MemCopy(new_store.start(), backing_store_.start(), position_); | 265 MemCopy(new_store.start(), backing_store_.start(), position_); |
266 backing_store_.Dispose(); | 266 backing_store_.Dispose(); |
267 backing_store_ = new_store; | 267 backing_store_ = new_store; |
268 } | 268 } |
269 | 269 |
270 void ConvertToTwoByte() { | 270 void ConvertToTwoByte() { |
271 ASSERT(is_one_byte_); | 271 DCHECK(is_one_byte_); |
272 Vector<byte> new_store; | 272 Vector<byte> new_store; |
273 int new_content_size = position_ * kUC16Size; | 273 int new_content_size = position_ * kUC16Size; |
274 if (new_content_size >= backing_store_.length()) { | 274 if (new_content_size >= backing_store_.length()) { |
275 // Ensure room for all currently read code units as UC16 as well | 275 // Ensure room for all currently read code units as UC16 as well |
276 // as the code unit about to be stored. | 276 // as the code unit about to be stored. |
277 new_store = Vector<byte>::New(NewCapacity(new_content_size)); | 277 new_store = Vector<byte>::New(NewCapacity(new_content_size)); |
278 } else { | 278 } else { |
279 new_store = backing_store_; | 279 new_store = backing_store_; |
280 } | 280 } |
281 uint8_t* src = backing_store_.start(); | 281 uint8_t* src = backing_store_.start(); |
(...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
365 bool literal_contains_escapes() const { | 365 bool literal_contains_escapes() const { |
366 Location location = current_.location; | 366 Location location = current_.location; |
367 int source_length = (location.end_pos - location.beg_pos); | 367 int source_length = (location.end_pos - location.beg_pos); |
368 if (current_.token == Token::STRING) { | 368 if (current_.token == Token::STRING) { |
369 // Subtract delimiters. | 369 // Subtract delimiters. |
370 source_length -= 2; | 370 source_length -= 2; |
371 } | 371 } |
372 return current_.literal_chars->length() != source_length; | 372 return current_.literal_chars->length() != source_length; |
373 } | 373 } |
374 bool is_literal_contextual_keyword(Vector<const char> keyword) { | 374 bool is_literal_contextual_keyword(Vector<const char> keyword) { |
375 ASSERT_NOT_NULL(current_.literal_chars); | 375 DCHECK_NOT_NULL(current_.literal_chars); |
376 return current_.literal_chars->is_contextual_keyword(keyword); | 376 return current_.literal_chars->is_contextual_keyword(keyword); |
377 } | 377 } |
378 bool is_next_contextual_keyword(Vector<const char> keyword) { | 378 bool is_next_contextual_keyword(Vector<const char> keyword) { |
379 ASSERT_NOT_NULL(next_.literal_chars); | 379 DCHECK_NOT_NULL(next_.literal_chars); |
380 return next_.literal_chars->is_contextual_keyword(keyword); | 380 return next_.literal_chars->is_contextual_keyword(keyword); |
381 } | 381 } |
382 | 382 |
383 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 383 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
384 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 384 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
385 | 385 |
386 double DoubleValue(); | 386 double DoubleValue(); |
387 bool UnescapedLiteralMatches(const char* data, int length) { | 387 bool UnescapedLiteralMatches(const char* data, int length) { |
388 if (is_literal_one_byte() && | 388 if (is_literal_one_byte() && |
389 literal_length() == length && | 389 literal_length() == length && |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
484 | 484 |
485 // Literal buffer support | 485 // Literal buffer support |
486 inline void StartLiteral() { | 486 inline void StartLiteral() { |
487 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 487 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? |
488 &literal_buffer2_ : &literal_buffer1_; | 488 &literal_buffer2_ : &literal_buffer1_; |
489 free_buffer->Reset(); | 489 free_buffer->Reset(); |
490 next_.literal_chars = free_buffer; | 490 next_.literal_chars = free_buffer; |
491 } | 491 } |
492 | 492 |
493 INLINE(void AddLiteralChar(uc32 c)) { | 493 INLINE(void AddLiteralChar(uc32 c)) { |
494 ASSERT_NOT_NULL(next_.literal_chars); | 494 DCHECK_NOT_NULL(next_.literal_chars); |
495 next_.literal_chars->AddChar(c); | 495 next_.literal_chars->AddChar(c); |
496 } | 496 } |
497 | 497 |
498 // Complete scanning of a literal. | 498 // Complete scanning of a literal. |
499 inline void TerminateLiteral() { | 499 inline void TerminateLiteral() { |
500 // Does nothing in the current implementation. | 500 // Does nothing in the current implementation. |
501 } | 501 } |
502 | 502 |
503 // Stops scanning of a literal and drop the collected characters, | 503 // Stops scanning of a literal and drop the collected characters, |
504 // e.g., due to an encountered error. | 504 // e.g., due to an encountered error. |
(...skipping 28 matching lines...) Expand all Loading... |
533 } | 533 } |
534 } | 534 } |
535 | 535 |
536 // Returns the literal string, if any, for the current token (the | 536 // Returns the literal string, if any, for the current token (the |
537 // token last returned by Next()). The string is 0-terminated. | 537 // token last returned by Next()). The string is 0-terminated. |
538 // Literal strings are collected for identifiers, strings, and | 538 // Literal strings are collected for identifiers, strings, and |
539 // numbers. | 539 // numbers. |
540 // These functions only give the correct result if the literal | 540 // These functions only give the correct result if the literal |
541 // was scanned between calls to StartLiteral() and TerminateLiteral(). | 541 // was scanned between calls to StartLiteral() and TerminateLiteral(). |
542 Vector<const uint8_t> literal_one_byte_string() { | 542 Vector<const uint8_t> literal_one_byte_string() { |
543 ASSERT_NOT_NULL(current_.literal_chars); | 543 DCHECK_NOT_NULL(current_.literal_chars); |
544 return current_.literal_chars->one_byte_literal(); | 544 return current_.literal_chars->one_byte_literal(); |
545 } | 545 } |
546 Vector<const uint16_t> literal_two_byte_string() { | 546 Vector<const uint16_t> literal_two_byte_string() { |
547 ASSERT_NOT_NULL(current_.literal_chars); | 547 DCHECK_NOT_NULL(current_.literal_chars); |
548 return current_.literal_chars->two_byte_literal(); | 548 return current_.literal_chars->two_byte_literal(); |
549 } | 549 } |
550 bool is_literal_one_byte() { | 550 bool is_literal_one_byte() { |
551 ASSERT_NOT_NULL(current_.literal_chars); | 551 DCHECK_NOT_NULL(current_.literal_chars); |
552 return current_.literal_chars->is_one_byte(); | 552 return current_.literal_chars->is_one_byte(); |
553 } | 553 } |
554 int literal_length() const { | 554 int literal_length() const { |
555 ASSERT_NOT_NULL(current_.literal_chars); | 555 DCHECK_NOT_NULL(current_.literal_chars); |
556 return current_.literal_chars->length(); | 556 return current_.literal_chars->length(); |
557 } | 557 } |
558 // Returns the literal string for the next token (the token that | 558 // Returns the literal string for the next token (the token that |
559 // would be returned if Next() were called). | 559 // would be returned if Next() were called). |
560 Vector<const uint8_t> next_literal_one_byte_string() { | 560 Vector<const uint8_t> next_literal_one_byte_string() { |
561 ASSERT_NOT_NULL(next_.literal_chars); | 561 DCHECK_NOT_NULL(next_.literal_chars); |
562 return next_.literal_chars->one_byte_literal(); | 562 return next_.literal_chars->one_byte_literal(); |
563 } | 563 } |
564 Vector<const uint16_t> next_literal_two_byte_string() { | 564 Vector<const uint16_t> next_literal_two_byte_string() { |
565 ASSERT_NOT_NULL(next_.literal_chars); | 565 DCHECK_NOT_NULL(next_.literal_chars); |
566 return next_.literal_chars->two_byte_literal(); | 566 return next_.literal_chars->two_byte_literal(); |
567 } | 567 } |
568 bool is_next_literal_one_byte() { | 568 bool is_next_literal_one_byte() { |
569 ASSERT_NOT_NULL(next_.literal_chars); | 569 DCHECK_NOT_NULL(next_.literal_chars); |
570 return next_.literal_chars->is_one_byte(); | 570 return next_.literal_chars->is_one_byte(); |
571 } | 571 } |
572 int next_literal_length() const { | 572 int next_literal_length() const { |
573 ASSERT_NOT_NULL(next_.literal_chars); | 573 DCHECK_NOT_NULL(next_.literal_chars); |
574 return next_.literal_chars->length(); | 574 return next_.literal_chars->length(); |
575 } | 575 } |
576 | 576 |
577 uc32 ScanHexNumber(int expected_length); | 577 uc32 ScanHexNumber(int expected_length); |
578 | 578 |
579 // Scans a single JavaScript token. | 579 // Scans a single JavaScript token. |
580 void Scan(); | 580 void Scan(); |
581 | 581 |
582 bool SkipWhiteSpace(); | 582 bool SkipWhiteSpace(); |
583 Token::Value SkipSingleLineComment(); | 583 Token::Value SkipSingleLineComment(); |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
645 bool harmony_scoping_; | 645 bool harmony_scoping_; |
646 // Whether we scan 'module', 'import', 'export' as keywords. | 646 // Whether we scan 'module', 'import', 'export' as keywords. |
647 bool harmony_modules_; | 647 bool harmony_modules_; |
648 // Whether we scan 0o777 and 0b111 as numbers. | 648 // Whether we scan 0o777 and 0b111 as numbers. |
649 bool harmony_numeric_literals_; | 649 bool harmony_numeric_literals_; |
650 }; | 650 }; |
651 | 651 |
652 } } // namespace v8::internal | 652 } } // namespace v8::internal |
653 | 653 |
654 #endif // V8_SCANNER_H_ | 654 #endif // V8_SCANNER_H_ |
OLD | NEW |