OLD | NEW |
1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 233 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
244 state_ = new_state; | 244 state_ = new_state; |
245 token_ = keyword_token; | 245 token_ = keyword_token; |
246 return true; | 246 return true; |
247 } | 247 } |
248 return false; | 248 return false; |
249 } | 249 } |
250 | 250 |
251 void Step(uc32 input); | 251 void Step(uc32 input); |
252 }; | 252 }; |
253 | 253 |
| 254 class ScannerData { |
| 255 public: |
| 256 unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace_; |
| 257 private: |
| 258 unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart_; |
| 259 unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart_; |
| 260 unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator_; |
| 261 |
| 262 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 263 StaticResource<Utf8Decoder> utf8_decoder_; |
| 264 |
| 265 friend class V8Context; |
| 266 friend class Scanner; |
| 267 ScannerData(); |
| 268 DISALLOW_COPY_AND_ASSIGN(ScannerData); |
| 269 }; |
254 | 270 |
255 class Scanner { | 271 class Scanner { |
256 public: | 272 public: |
257 | 273 |
258 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 274 typedef ScannerData::Utf8Decoder Utf8Decoder; |
259 | 275 |
260 // Construction | 276 // Construction |
261 explicit Scanner(bool is_pre_parsing); | 277 explicit Scanner(bool is_pre_parsing); |
262 | 278 |
263 // Initialize the Scanner to scan source: | 279 // Initialize the Scanner to scan source: |
264 void Init(Handle<String> source, | 280 void Init(Handle<String> source, |
265 unibrow::CharacterStream* stream, | 281 unibrow::CharacterStream* stream, |
266 int position); | 282 int position); |
267 | 283 |
268 // Returns the next token. | 284 // Returns the next token. |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
327 | 343 |
328 // Seek forward to the given position. This operation does not | 344 // Seek forward to the given position. This operation does not |
329 // work in general, for instance when there are pushed back | 345 // work in general, for instance when there are pushed back |
330 // characters, but works for seeking forward until simple delimiter | 346 // characters, but works for seeking forward until simple delimiter |
331 // tokens, which is what it is used for. | 347 // tokens, which is what it is used for. |
332 void SeekForward(int pos); | 348 void SeekForward(int pos); |
333 | 349 |
334 Handle<String> SubString(int start_pos, int end_pos); | 350 Handle<String> SubString(int start_pos, int end_pos); |
335 bool stack_overflow() { return stack_overflow_; } | 351 bool stack_overflow() { return stack_overflow_; } |
336 | 352 |
337 static StaticResource<Utf8Decoder>* utf8_decoder() { return &utf8_decoder_; } | 353 static StaticResource<Utf8Decoder>* utf8_decoder() { |
| 354 return &v8_context()->scanner_data_.utf8_decoder_; |
| 355 } |
338 | 356 |
339 // Tells whether the buffer contains an identifier (no escapes). | 357 // Tells whether the buffer contains an identifier (no escapes). |
340 // Used for checking if a property name is an identifier. | 358 // Used for checking if a property name is an identifier. |
341 static bool IsIdentifier(unibrow::CharacterStream* buffer); | 359 static bool IsIdentifier(unibrow::CharacterStream* buffer); |
342 | 360 |
343 static unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; | |
344 static unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; | |
345 static unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; | |
346 static unibrow::Predicate<unibrow::WhiteSpace, 128> kIsWhiteSpace; | |
347 | |
348 static const int kCharacterLookaheadBufferSize = 1; | 361 static const int kCharacterLookaheadBufferSize = 1; |
349 | 362 |
350 private: | 363 private: |
351 CharacterStreamUTF16Buffer char_stream_buffer_; | 364 CharacterStreamUTF16Buffer char_stream_buffer_; |
352 TwoByteStringUTF16Buffer two_byte_string_buffer_; | 365 TwoByteStringUTF16Buffer two_byte_string_buffer_; |
353 | 366 |
354 // Source. | 367 // Source. |
355 UTF16Buffer* source_; | 368 UTF16Buffer* source_; |
356 int position_; | 369 int position_; |
357 | 370 |
358 // Buffer to hold literal values (identifiers, strings, numbers) | 371 // Buffer to hold literal values (identifiers, strings, numbers) |
359 // using 0-terminated UTF-8 encoding. | 372 // using 0-terminated UTF-8 encoding. |
360 UTF8Buffer literal_buffer_1_; | 373 UTF8Buffer literal_buffer_1_; |
361 UTF8Buffer literal_buffer_2_; | 374 UTF8Buffer literal_buffer_2_; |
362 | 375 |
363 bool stack_overflow_; | 376 bool stack_overflow_; |
364 static StaticResource<Utf8Decoder> utf8_decoder_; | |
365 | |
366 // One Unicode character look-ahead; c0_ < 0 at the end of the input. | 377 // One Unicode character look-ahead; c0_ < 0 at the end of the input. |
367 uc32 c0_; | 378 uc32 c0_; |
368 | 379 |
369 // The current and look-ahead token. | 380 // The current and look-ahead token. |
370 struct TokenDesc { | 381 struct TokenDesc { |
371 Token::Value token; | 382 Token::Value token; |
372 Location location; | 383 Location location; |
373 UTF8Buffer* literal_buffer; | 384 UTF8Buffer* literal_buffer; |
374 }; | 385 }; |
375 | 386 |
376 TokenDesc current_; // desc for current token (as returned by Next()) | 387 TokenDesc current_; // desc for current token (as returned by Next()) |
377 TokenDesc next_; // desc for next token (one token look-ahead) | 388 TokenDesc next_; // desc for next token (one token look-ahead) |
378 bool has_line_terminator_before_next_; | 389 bool has_line_terminator_before_next_; |
379 bool is_pre_parsing_; | 390 bool is_pre_parsing_; |
| 391 ScannerData& scanner_data_; |
380 | 392 |
381 // Literal buffer support | 393 // Literal buffer support |
382 void StartLiteral(); | 394 void StartLiteral(); |
383 void AddChar(uc32 ch); | 395 void AddChar(uc32 ch); |
384 void AddCharAdvance(); | 396 void AddCharAdvance(); |
385 void TerminateLiteral(); | 397 void TerminateLiteral(); |
386 | 398 |
387 // Low-level scanning support. | 399 // Low-level scanning support. |
388 void Advance() { c0_ = source_->Advance(); } | 400 void Advance() { c0_ = source_->Advance(); } |
389 void PushBack(uc32 ch) { | 401 void PushBack(uc32 ch) { |
(...skipping 26 matching lines...) Expand all Loading... |
416 } | 428 } |
417 | 429 |
418 // Decodes a unicode escape-sequence which is part of an identifier. | 430 // Decodes a unicode escape-sequence which is part of an identifier. |
419 // If the escape sequence cannot be decoded the result is kBadRune. | 431 // If the escape sequence cannot be decoded the result is kBadRune. |
420 uc32 ScanIdentifierUnicodeEscape(); | 432 uc32 ScanIdentifierUnicodeEscape(); |
421 }; | 433 }; |
422 | 434 |
423 } } // namespace v8::internal | 435 } } // namespace v8::internal |
424 | 436 |
425 #endif // V8_SCANNER_H_ | 437 #endif // V8_SCANNER_H_ |
OLD | NEW |