| OLD | NEW |
| 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
| 2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
| 3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
| 4 // met: | 4 // met: |
| 5 // | 5 // |
| 6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
| 7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
| 8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
| 9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
| 10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
| (...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 244 state_ = new_state; | 244 state_ = new_state; |
| 245 token_ = keyword_token; | 245 token_ = keyword_token; |
| 246 return true; | 246 return true; |
| 247 } | 247 } |
| 248 return false; | 248 return false; |
| 249 } | 249 } |
| 250 | 250 |
| 251 void Step(uc32 input); | 251 void Step(uc32 input); |
| 252 }; | 252 }; |
| 253 | 253 |
| 254 class ScannerData { |
| 255 public: |
| 256 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace_; |
| 257 private: |
| 258 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart_; |
| 259 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart_; |
| 260 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator_; |
| 261 |
| 262 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 263 StaticResource<Utf8Decoder> utf8_decoder_; |
| 264 |
| 265 friend class V8Context; |
| 266 friend class Scanner; |
| 267 ScannerData(); |
| 268 DISALLOW_COPY_AND_ASSIGN(ScannerData); |
| 269 }; |
| 254 | 270 |
| 255 class Scanner { | 271 class Scanner { |
| 256 public: | 272 public: |
| 257 | 273 |
| 258 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 274 typedef ScannerData::Utf8Decoder Utf8Decoder; |
| 259 | 275 |
| 260 // Construction | 276 // Construction |
| 261 explicit Scanner(bool is_pre_parsing); | 277 explicit Scanner(bool is_pre_parsing); |
| 262 | 278 |
| 263 // Initialize the Scanner to scan source: | 279 // Initialize the Scanner to scan source: |
| 264 void Init(Handle<String> source, | 280 void Init(Handle<String> source, |
| 265 unibrow::CharacterStream* stream, | 281 unibrow::CharacterStream* stream, |
| 266 int position); | 282 int position); |
| 267 | 283 |
| 268 // Returns the next token. | 284 // Returns the next token. |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 327 | 343 |
| 328 // Seek forward to the given position. This operation does not | 344 // Seek forward to the given position. This operation does not |
| 329 // work in general, for instance when there are pushed back | 345 // work in general, for instance when there are pushed back |
| 330 // characters, but works for seeking forward until simple delimiter | 346 // characters, but works for seeking forward until simple delimiter |
| 331 // tokens, which is what it is used for. | 347 // tokens, which is what it is used for. |
| 332 void SeekForward(int pos); | 348 void SeekForward(int pos); |
| 333 | 349 |
| 334 Handle<String> SubString(int start_pos, int end_pos); | 350 Handle<String> SubString(int start_pos, int end_pos); |
| 335 bool stack_overflow() { return stack_overflow_; } | 351 bool stack_overflow() { return stack_overflow_; } |
| 336 | 352 |
| 337 static StaticResource<Utf8Decoder>* utf8_decoder() { return &utf8_decoder_; } | 353 static StaticResource<Utf8Decoder>* utf8_decoder() { |
| 354 return &v8_context()->scanner_data_.utf8_decoder_; |
| 355 } |
| 338 | 356 |
| 339 // Tells whether the buffer contains an identifier (no escapes). | 357 // Tells whether the buffer contains an identifier (no escapes). |
| 340 // Used for checking if a property name is an identifier. | 358 // Used for checking if a property name is an identifier. |
| 341 static bool IsIdentifier(unibrow::CharacterStream* buffer); | 359 static bool IsIdentifier(unibrow::CharacterStream* buffer); |
| 342 | 360 |
| 343 static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; | |
| 344 static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; | |
| 345 static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; | |
| 346 static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; | |
| 347 | |
| 348 static const int kCharacterLookaheadBufferSize = 1; | 361 static const int kCharacterLookaheadBufferSize = 1; |
| 349 | 362 |
| 350 private: | 363 private: |
| 351 CharacterStreamUTF16Buffer char_stream_buffer_; | 364 CharacterStreamUTF16Buffer char_stream_buffer_; |
| 352 TwoByteStringUTF16Buffer two_byte_string_buffer_; | 365 TwoByteStringUTF16Buffer two_byte_string_buffer_; |
| 353 | 366 |
| 354 // Source. | 367 // Source. |
| 355 UTF16Buffer* source_; | 368 UTF16Buffer* source_; |
| 356 int position_; | 369 int position_; |
| 357 | 370 |
| 358 // Buffer to hold literal values (identifiers, strings, numbers) | 371 // Buffer to hold literal values (identifiers, strings, numbers) |
| 359 // using 0-terminated UTF-8 encoding. | 372 // using 0-terminated UTF-8 encoding. |
| 360 UTF8Buffer literal_buffer_1_; | 373 UTF8Buffer literal_buffer_1_; |
| 361 UTF8Buffer literal_buffer_2_; | 374 UTF8Buffer literal_buffer_2_; |
| 362 | 375 |
| 363 bool stack_overflow_; | 376 bool stack_overflow_; |
| 364 static StaticResource<Utf8Decoder> utf8_decoder_; | |
| 365 | |
| 366 // One Unicode character look-ahead; c0_ < 0 at the end of the input. | 377 // One Unicode character look-ahead; c0_ < 0 at the end of the input. |
| 367 uc32 c0_; | 378 uc32 c0_; |
| 368 | 379 |
| 369 // The current and look-ahead token. | 380 // The current and look-ahead token. |
| 370 struct TokenDesc { | 381 struct TokenDesc { |
| 371 Token::Value token; | 382 Token::Value token; |
| 372 Location location; | 383 Location location; |
| 373 UTF8Buffer* literal_buffer; | 384 UTF8Buffer* literal_buffer; |
| 374 }; | 385 }; |
| 375 | 386 |
| 376 TokenDesc current_; // desc for current token (as returned by Next()) | 387 TokenDesc current_; // desc for current token (as returned by Next()) |
| 377 TokenDesc next_; // desc for next token (one token look-ahead) | 388 TokenDesc next_; // desc for next token (one token look-ahead) |
| 378 bool has_line_terminator_before_next_; | 389 bool has_line_terminator_before_next_; |
| 379 bool is_pre_parsing_; | 390 bool is_pre_parsing_; |
| 391 ScannerData& scanner_data_; |
| 380 | 392 |
| 381 // Literal buffer support | 393 // Literal buffer support |
| 382 void StartLiteral(); | 394 void StartLiteral(); |
| 383 void AddChar(uc32 ch); | 395 void AddChar(uc32 ch); |
| 384 void AddCharAdvance(); | 396 void AddCharAdvance(); |
| 385 void TerminateLiteral(); | 397 void TerminateLiteral(); |
| 386 | 398 |
| 387 // Low-level scanning support. | 399 // Low-level scanning support. |
| 388 void Advance() { c0_ = source_->Advance(); } | 400 void Advance() { c0_ = source_->Advance(); } |
| 389 void PushBack(uc32 ch) { | 401 void PushBack(uc32 ch) { |
| (...skipping 26 matching lines...) Expand all Loading... |
| 416 } | 428 } |
| 417 | 429 |
| 418 // Decodes a unicode escape-sequence which is part of an identifier. | 430 // Decodes a unicode escape-sequence which is part of an identifier. |
| 419 // If the escape sequence cannot be decoded the result is kBadRune. | 431 // If the escape sequence cannot be decoded the result is kBadRune. |
| 420 uc32 ScanIdentifierUnicodeEscape(); | 432 uc32 ScanIdentifierUnicodeEscape(); |
| 421 }; | 433 }; |
| 422 | 434 |
| 423 } } // namespace v8::internal | 435 } } // namespace v8::internal |
| 424 | 436 |
| 425 #endif // V8_SCANNER_H_ | 437 #endif // V8_SCANNER_H_ |
| OLD | NEW |