OLD | NEW |
---|---|
1 // Copyright 2006-2008 the V8 project authors. All rights reserved. | 1 // Copyright 2006-2008 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 234 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
245 token_ = keyword_token; | 245 token_ = keyword_token; |
246 return true; | 246 return true; |
247 } | 247 } |
248 return false; | 248 return false; |
249 } | 249 } |
250 | 250 |
251 void Step(uc32 input); | 251 void Step(uc32 input); |
252 }; | 252 }; |
253 | 253 |
254 | 254 |
255 enum ParserMode { PARSE, PREPARSE }; | |
256 enum ParserLanguage { JAVASCRIPT, JSON }; | |
257 | |
258 | |
255 class Scanner { | 259 class Scanner { |
256 public: | 260 public: |
257 | |
258 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 261 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
259 | 262 |
260 // Construction | 263 // Construction |
261 explicit Scanner(bool is_pre_parsing); | 264 explicit Scanner(ParserMode parse_mode); |
262 | 265 |
263 // Initialize the Scanner to scan source: | 266 // Initialize the Scanner to scan source: |
264 void Init(Handle<String> source, | 267 void Init(Handle<String> source, |
265 unibrow::CharacterStream* stream, | 268 unibrow::CharacterStream* stream, |
266 int position); | 269 int position, |
270 ParserLanguage language); | |
267 | 271 |
268 // Returns the next token. | 272 // Returns the next token. |
269 Token::Value Next(); | 273 Token::Value Next(); |
270 | 274 |
271 // One token look-ahead (past the token returned by Next()). | 275 // One token look-ahead (past the token returned by Next()). |
272 Token::Value peek() const { return next_.token; } | 276 Token::Value peek() const { return next_.token; } |
273 | 277 |
274 // Returns true if there was a line terminator before the peek'ed token. | 278 // Returns true if there was a line terminator before the peek'ed token. |
275 bool has_line_terminator_before_next() const { | 279 bool has_line_terminator_before_next() const { |
276 return has_line_terminator_before_next_; | 280 return has_line_terminator_before_next_; |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
370 struct TokenDesc { | 374 struct TokenDesc { |
371 Token::Value token; | 375 Token::Value token; |
372 Location location; | 376 Location location; |
373 UTF8Buffer* literal_buffer; | 377 UTF8Buffer* literal_buffer; |
374 }; | 378 }; |
375 | 379 |
376 TokenDesc current_; // desc for current token (as returned by Next()) | 380 TokenDesc current_; // desc for current token (as returned by Next()) |
377 TokenDesc next_; // desc for next token (one token look-ahead) | 381 TokenDesc next_; // desc for next token (one token look-ahead) |
378 bool has_line_terminator_before_next_; | 382 bool has_line_terminator_before_next_; |
379 bool is_pre_parsing_; | 383 bool is_pre_parsing_; |
384 bool is_parsing_json_; | |
380 | 385 |
381 // Literal buffer support | 386 // Literal buffer support |
382 void StartLiteral(); | 387 void StartLiteral(); |
383 void AddChar(uc32 ch); | 388 void AddChar(uc32 ch); |
384 void AddCharAdvance(); | 389 void AddCharAdvance(); |
385 void TerminateLiteral(); | 390 void TerminateLiteral(); |
386 | 391 |
387 // Low-level scanning support. | 392 // Low-level scanning support. |
388 void Advance() { c0_ = source_->Advance(); } | 393 void Advance() { c0_ = source_->Advance(); } |
389 void PushBack(uc32 ch) { | 394 void PushBack(uc32 ch) { |
390 source_->PushBack(ch); | 395 source_->PushBack(ch); |
391 c0_ = ch; | 396 c0_ = ch; |
392 } | 397 } |
393 | 398 |
394 bool SkipWhiteSpace(); | 399 bool SkipWhiteSpace() { |
400 if (is_parsing_json_) { | |
401 return SkipJsonWhiteSpace(); | |
402 } else { | |
403 return SkipJavaScriptWhiteSpace(); | |
404 } | |
405 } | |
406 bool SkipJavaScriptWhiteSpace(); | |
407 bool SkipJsonWhiteSpace(); | |
395 Token::Value SkipSingleLineComment(); | 408 Token::Value SkipSingleLineComment(); |
396 Token::Value SkipMultiLineComment(); | 409 Token::Value SkipMultiLineComment(); |
397 | 410 |
398 inline Token::Value Select(Token::Value tok); | 411 inline Token::Value Select(Token::Value tok); |
399 inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_); | 412 inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_); |
400 | 413 |
401 void Scan(); | 414 inline void Scan() { |
415 if (is_parsing_json_) { | |
416 ScanJson(); | |
417 } else { | |
418 ScanJavaScript(); | |
419 } | |
420 } | |
421 void ScanJson(); | |
422 void ScanJavaScript(); | |
423 | |
424 Token::Value ScanJsonNumber(); | |
Mads Ager (chromium)
2010/02/01 08:42:19
For each of these Scan functions, please add a com
| |
425 Token::Value ScanJsonString(); | |
426 Token::Value ScanJsonIdentifier(const char* text, Token::Value token); | |
427 | |
402 void ScanDecimalDigits(); | 428 void ScanDecimalDigits(); |
403 Token::Value ScanNumber(bool seen_period); | 429 Token::Value ScanNumber(bool seen_period); |
404 Token::Value ScanIdentifier(); | 430 Token::Value ScanIdentifier(); |
405 uc32 ScanHexEscape(uc32 c, int length); | 431 uc32 ScanHexEscape(uc32 c, int length); |
406 uc32 ScanOctalEscape(uc32 c, int length); | 432 uc32 ScanOctalEscape(uc32 c, int length); |
407 void ScanEscape(); | 433 void ScanEscape(); |
408 Token::Value ScanString(); | 434 Token::Value ScanString(); |
409 | 435 |
410 // Scans a possible HTML comment -- begins with '<!'. | 436 // Scans a possible HTML comment -- begins with '<!'. |
411 Token::Value ScanHtmlComment(); | 437 Token::Value ScanHtmlComment(); |
412 | 438 |
413 // Return the current source position. | 439 // Return the current source position. |
414 int source_pos() { | 440 int source_pos() { |
415 return source_->pos() - kCharacterLookaheadBufferSize + position_; | 441 return source_->pos() - kCharacterLookaheadBufferSize + position_; |
416 } | 442 } |
417 | 443 |
418 // Decodes a unicode escape-sequence which is part of an identifier. | 444 // Decodes a unicode escape-sequence which is part of an identifier. |
419 // If the escape sequence cannot be decoded the result is kBadRune. | 445 // If the escape sequence cannot be decoded the result is kBadRune. |
420 uc32 ScanIdentifierUnicodeEscape(); | 446 uc32 ScanIdentifierUnicodeEscape(); |
421 }; | 447 }; |
422 | 448 |
423 } } // namespace v8::internal | 449 } } // namespace v8::internal |
424 | 450 |
425 #endif // V8_SCANNER_H_ | 451 #endif // V8_SCANNER_H_ |
OLD | NEW |