| OLD | NEW |
| 1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
| 6 | 6 |
| 7 #ifndef V8_SCANNER_H_ | 7 #ifndef V8_SCANNER_H_ |
| 8 #define V8_SCANNER_H_ | 8 #define V8_SCANNER_H_ |
| 9 | 9 |
| 10 #include "src/allocation.h" | 10 #include "src/allocation.h" |
| (...skipping 193 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 204 INLINE(void AddChar(uint32_t code_unit)) { | 204 INLINE(void AddChar(uint32_t code_unit)) { |
| 205 if (position_ >= backing_store_.length()) ExpandBuffer(); | 205 if (position_ >= backing_store_.length()) ExpandBuffer(); |
| 206 if (is_one_byte_) { | 206 if (is_one_byte_) { |
| 207 if (code_unit <= unibrow::Latin1::kMaxChar) { | 207 if (code_unit <= unibrow::Latin1::kMaxChar) { |
| 208 backing_store_[position_] = static_cast<byte>(code_unit); | 208 backing_store_[position_] = static_cast<byte>(code_unit); |
| 209 position_ += kOneByteSize; | 209 position_ += kOneByteSize; |
| 210 return; | 210 return; |
| 211 } | 211 } |
| 212 ConvertToTwoByte(); | 212 ConvertToTwoByte(); |
| 213 } | 213 } |
| 214 ASSERT(code_unit < 0x10000u); | 214 DCHECK(code_unit < 0x10000u); |
| 215 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit; | 215 *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit; |
| 216 position_ += kUC16Size; | 216 position_ += kUC16Size; |
| 217 } | 217 } |
| 218 | 218 |
| 219 bool is_one_byte() const { return is_one_byte_; } | 219 bool is_one_byte() const { return is_one_byte_; } |
| 220 | 220 |
| 221 bool is_contextual_keyword(Vector<const char> keyword) const { | 221 bool is_contextual_keyword(Vector<const char> keyword) const { |
| 222 return is_one_byte() && keyword.length() == position_ && | 222 return is_one_byte() && keyword.length() == position_ && |
| 223 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); | 223 (memcmp(keyword.start(), backing_store_.start(), position_) == 0); |
| 224 } | 224 } |
| 225 | 225 |
| 226 Vector<const uint16_t> two_byte_literal() const { | 226 Vector<const uint16_t> two_byte_literal() const { |
| 227 ASSERT(!is_one_byte_); | 227 DCHECK(!is_one_byte_); |
| 228 ASSERT((position_ & 0x1) == 0); | 228 DCHECK((position_ & 0x1) == 0); |
| 229 return Vector<const uint16_t>( | 229 return Vector<const uint16_t>( |
| 230 reinterpret_cast<const uint16_t*>(backing_store_.start()), | 230 reinterpret_cast<const uint16_t*>(backing_store_.start()), |
| 231 position_ >> 1); | 231 position_ >> 1); |
| 232 } | 232 } |
| 233 | 233 |
| 234 Vector<const uint8_t> one_byte_literal() const { | 234 Vector<const uint8_t> one_byte_literal() const { |
| 235 ASSERT(is_one_byte_); | 235 DCHECK(is_one_byte_); |
| 236 return Vector<const uint8_t>( | 236 return Vector<const uint8_t>( |
| 237 reinterpret_cast<const uint8_t*>(backing_store_.start()), | 237 reinterpret_cast<const uint8_t*>(backing_store_.start()), |
| 238 position_); | 238 position_); |
| 239 } | 239 } |
| 240 | 240 |
| 241 int length() const { | 241 int length() const { |
| 242 return is_one_byte_ ? position_ : (position_ >> 1); | 242 return is_one_byte_ ? position_ : (position_ >> 1); |
| 243 } | 243 } |
| 244 | 244 |
| 245 void Reset() { | 245 void Reset() { |
| (...skipping 15 matching lines...) Expand all Loading... |
| 261 } | 261 } |
| 262 | 262 |
| 263 void ExpandBuffer() { | 263 void ExpandBuffer() { |
| 264 Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity)); | 264 Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity)); |
| 265 MemCopy(new_store.start(), backing_store_.start(), position_); | 265 MemCopy(new_store.start(), backing_store_.start(), position_); |
| 266 backing_store_.Dispose(); | 266 backing_store_.Dispose(); |
| 267 backing_store_ = new_store; | 267 backing_store_ = new_store; |
| 268 } | 268 } |
| 269 | 269 |
| 270 void ConvertToTwoByte() { | 270 void ConvertToTwoByte() { |
| 271 ASSERT(is_one_byte_); | 271 DCHECK(is_one_byte_); |
| 272 Vector<byte> new_store; | 272 Vector<byte> new_store; |
| 273 int new_content_size = position_ * kUC16Size; | 273 int new_content_size = position_ * kUC16Size; |
| 274 if (new_content_size >= backing_store_.length()) { | 274 if (new_content_size >= backing_store_.length()) { |
| 275 // Ensure room for all currently read code units as UC16 as well | 275 // Ensure room for all currently read code units as UC16 as well |
| 276 // as the code unit about to be stored. | 276 // as the code unit about to be stored. |
| 277 new_store = Vector<byte>::New(NewCapacity(new_content_size)); | 277 new_store = Vector<byte>::New(NewCapacity(new_content_size)); |
| 278 } else { | 278 } else { |
| 279 new_store = backing_store_; | 279 new_store = backing_store_; |
| 280 } | 280 } |
| 281 uint8_t* src = backing_store_.start(); | 281 uint8_t* src = backing_store_.start(); |
| (...skipping 83 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 365 bool literal_contains_escapes() const { | 365 bool literal_contains_escapes() const { |
| 366 Location location = current_.location; | 366 Location location = current_.location; |
| 367 int source_length = (location.end_pos - location.beg_pos); | 367 int source_length = (location.end_pos - location.beg_pos); |
| 368 if (current_.token == Token::STRING) { | 368 if (current_.token == Token::STRING) { |
| 369 // Subtract delimiters. | 369 // Subtract delimiters. |
| 370 source_length -= 2; | 370 source_length -= 2; |
| 371 } | 371 } |
| 372 return current_.literal_chars->length() != source_length; | 372 return current_.literal_chars->length() != source_length; |
| 373 } | 373 } |
| 374 bool is_literal_contextual_keyword(Vector<const char> keyword) { | 374 bool is_literal_contextual_keyword(Vector<const char> keyword) { |
| 375 ASSERT_NOT_NULL(current_.literal_chars); | 375 DCHECK_NOT_NULL(current_.literal_chars); |
| 376 return current_.literal_chars->is_contextual_keyword(keyword); | 376 return current_.literal_chars->is_contextual_keyword(keyword); |
| 377 } | 377 } |
| 378 bool is_next_contextual_keyword(Vector<const char> keyword) { | 378 bool is_next_contextual_keyword(Vector<const char> keyword) { |
| 379 ASSERT_NOT_NULL(next_.literal_chars); | 379 DCHECK_NOT_NULL(next_.literal_chars); |
| 380 return next_.literal_chars->is_contextual_keyword(keyword); | 380 return next_.literal_chars->is_contextual_keyword(keyword); |
| 381 } | 381 } |
| 382 | 382 |
| 383 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); | 383 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); |
| 384 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); | 384 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); |
| 385 | 385 |
| 386 double DoubleValue(); | 386 double DoubleValue(); |
| 387 bool UnescapedLiteralMatches(const char* data, int length) { | 387 bool UnescapedLiteralMatches(const char* data, int length) { |
| 388 if (is_literal_one_byte() && | 388 if (is_literal_one_byte() && |
| 389 literal_length() == length && | 389 literal_length() == length && |
| (...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 484 | 484 |
| 485 // Literal buffer support | 485 // Literal buffer support |
| 486 inline void StartLiteral() { | 486 inline void StartLiteral() { |
| 487 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? | 487 LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? |
| 488 &literal_buffer2_ : &literal_buffer1_; | 488 &literal_buffer2_ : &literal_buffer1_; |
| 489 free_buffer->Reset(); | 489 free_buffer->Reset(); |
| 490 next_.literal_chars = free_buffer; | 490 next_.literal_chars = free_buffer; |
| 491 } | 491 } |
| 492 | 492 |
| 493 INLINE(void AddLiteralChar(uc32 c)) { | 493 INLINE(void AddLiteralChar(uc32 c)) { |
| 494 ASSERT_NOT_NULL(next_.literal_chars); | 494 DCHECK_NOT_NULL(next_.literal_chars); |
| 495 next_.literal_chars->AddChar(c); | 495 next_.literal_chars->AddChar(c); |
| 496 } | 496 } |
| 497 | 497 |
| 498 // Complete scanning of a literal. | 498 // Complete scanning of a literal. |
| 499 inline void TerminateLiteral() { | 499 inline void TerminateLiteral() { |
| 500 // Does nothing in the current implementation. | 500 // Does nothing in the current implementation. |
| 501 } | 501 } |
| 502 | 502 |
| 503 // Stops scanning of a literal and drop the collected characters, | 503 // Stops scanning of a literal and drop the collected characters, |
| 504 // e.g., due to an encountered error. | 504 // e.g., due to an encountered error. |
| (...skipping 28 matching lines...) Expand all Loading... |
| 533 } | 533 } |
| 534 } | 534 } |
| 535 | 535 |
| 536 // Returns the literal string, if any, for the current token (the | 536 // Returns the literal string, if any, for the current token (the |
| 537 // token last returned by Next()). The string is 0-terminated. | 537 // token last returned by Next()). The string is 0-terminated. |
| 538 // Literal strings are collected for identifiers, strings, and | 538 // Literal strings are collected for identifiers, strings, and |
| 539 // numbers. | 539 // numbers. |
| 540 // These functions only give the correct result if the literal | 540 // These functions only give the correct result if the literal |
| 541 // was scanned between calls to StartLiteral() and TerminateLiteral(). | 541 // was scanned between calls to StartLiteral() and TerminateLiteral(). |
| 542 Vector<const uint8_t> literal_one_byte_string() { | 542 Vector<const uint8_t> literal_one_byte_string() { |
| 543 ASSERT_NOT_NULL(current_.literal_chars); | 543 DCHECK_NOT_NULL(current_.literal_chars); |
| 544 return current_.literal_chars->one_byte_literal(); | 544 return current_.literal_chars->one_byte_literal(); |
| 545 } | 545 } |
| 546 Vector<const uint16_t> literal_two_byte_string() { | 546 Vector<const uint16_t> literal_two_byte_string() { |
| 547 ASSERT_NOT_NULL(current_.literal_chars); | 547 DCHECK_NOT_NULL(current_.literal_chars); |
| 548 return current_.literal_chars->two_byte_literal(); | 548 return current_.literal_chars->two_byte_literal(); |
| 549 } | 549 } |
| 550 bool is_literal_one_byte() { | 550 bool is_literal_one_byte() { |
| 551 ASSERT_NOT_NULL(current_.literal_chars); | 551 DCHECK_NOT_NULL(current_.literal_chars); |
| 552 return current_.literal_chars->is_one_byte(); | 552 return current_.literal_chars->is_one_byte(); |
| 553 } | 553 } |
| 554 int literal_length() const { | 554 int literal_length() const { |
| 555 ASSERT_NOT_NULL(current_.literal_chars); | 555 DCHECK_NOT_NULL(current_.literal_chars); |
| 556 return current_.literal_chars->length(); | 556 return current_.literal_chars->length(); |
| 557 } | 557 } |
| 558 // Returns the literal string for the next token (the token that | 558 // Returns the literal string for the next token (the token that |
| 559 // would be returned if Next() were called). | 559 // would be returned if Next() were called). |
| 560 Vector<const uint8_t> next_literal_one_byte_string() { | 560 Vector<const uint8_t> next_literal_one_byte_string() { |
| 561 ASSERT_NOT_NULL(next_.literal_chars); | 561 DCHECK_NOT_NULL(next_.literal_chars); |
| 562 return next_.literal_chars->one_byte_literal(); | 562 return next_.literal_chars->one_byte_literal(); |
| 563 } | 563 } |
| 564 Vector<const uint16_t> next_literal_two_byte_string() { | 564 Vector<const uint16_t> next_literal_two_byte_string() { |
| 565 ASSERT_NOT_NULL(next_.literal_chars); | 565 DCHECK_NOT_NULL(next_.literal_chars); |
| 566 return next_.literal_chars->two_byte_literal(); | 566 return next_.literal_chars->two_byte_literal(); |
| 567 } | 567 } |
| 568 bool is_next_literal_one_byte() { | 568 bool is_next_literal_one_byte() { |
| 569 ASSERT_NOT_NULL(next_.literal_chars); | 569 DCHECK_NOT_NULL(next_.literal_chars); |
| 570 return next_.literal_chars->is_one_byte(); | 570 return next_.literal_chars->is_one_byte(); |
| 571 } | 571 } |
| 572 int next_literal_length() const { | 572 int next_literal_length() const { |
| 573 ASSERT_NOT_NULL(next_.literal_chars); | 573 DCHECK_NOT_NULL(next_.literal_chars); |
| 574 return next_.literal_chars->length(); | 574 return next_.literal_chars->length(); |
| 575 } | 575 } |
| 576 | 576 |
| 577 uc32 ScanHexNumber(int expected_length); | 577 uc32 ScanHexNumber(int expected_length); |
| 578 | 578 |
| 579 // Scans a single JavaScript token. | 579 // Scans a single JavaScript token. |
| 580 void Scan(); | 580 void Scan(); |
| 581 | 581 |
| 582 bool SkipWhiteSpace(); | 582 bool SkipWhiteSpace(); |
| 583 Token::Value SkipSingleLineComment(); | 583 Token::Value SkipSingleLineComment(); |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 645 bool harmony_scoping_; | 645 bool harmony_scoping_; |
| 646 // Whether we scan 'module', 'import', 'export' as keywords. | 646 // Whether we scan 'module', 'import', 'export' as keywords. |
| 647 bool harmony_modules_; | 647 bool harmony_modules_; |
| 648 // Whether we scan 0o777 and 0b111 as numbers. | 648 // Whether we scan 0o777 and 0b111 as numbers. |
| 649 bool harmony_numeric_literals_; | 649 bool harmony_numeric_literals_; |
| 650 }; | 650 }; |
| 651 | 651 |
| 652 } } // namespace v8::internal | 652 } } // namespace v8::internal |
| 653 | 653 |
| 654 #endif // V8_SCANNER_H_ | 654 #endif // V8_SCANNER_H_ |
| OLD | NEW |