| OLD | NEW |
| 1 // Copyright 2010 the V8 project authors. All rights reserved. | 1 // Copyright 2010 the V8 project authors. All rights reserved. |
| 2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
| 3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
| 4 // met: | 4 // met: |
| 5 // | 5 // |
| 6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
| 7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
| 8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
| 9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
| 10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
| (...skipping 229 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 240 Vector<byte> backing_store_; | 240 Vector<byte> backing_store_; |
| 241 }; | 241 }; |
| 242 | 242 |
| 243 | 243 |
| 244 // ---------------------------------------------------------------------------- | 244 // ---------------------------------------------------------------------------- |
| 245 // Scanner base-class. | 245 // Scanner base-class. |
| 246 | 246 |
| 247 // Generic functionality used by both JSON and JavaScript scanners. | 247 // Generic functionality used by both JSON and JavaScript scanners. |
| 248 class Scanner { | 248 class Scanner { |
| 249 public: | 249 public: |
| 250 // -1 is outside of the range of any real source code. |
| 251 static const int kNoOctalLocation = -1; |
| 252 |
| 250 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; | 253 typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder; |
| 251 | 254 |
| 252 class LiteralScope { | 255 class LiteralScope { |
| 253 public: | 256 public: |
| 254 explicit LiteralScope(Scanner* self); | 257 explicit LiteralScope(Scanner* self); |
| 255 ~LiteralScope(); | 258 ~LiteralScope(); |
| 256 void Complete(); | 259 void Complete(); |
| 257 | 260 |
| 258 private: | 261 private: |
| 259 Scanner* scanner_; | 262 Scanner* scanner_; |
| (...skipping 13 matching lines...) Expand all Loading... |
| 273 Location() : beg_pos(0), end_pos(0) { } | 276 Location() : beg_pos(0), end_pos(0) { } |
| 274 int beg_pos; | 277 int beg_pos; |
| 275 int end_pos; | 278 int end_pos; |
| 276 }; | 279 }; |
| 277 | 280 |
| 278 // Returns the location information for the current token | 281 // Returns the location information for the current token |
| 279 // (the token returned by Next()). | 282 // (the token returned by Next()). |
| 280 Location location() const { return current_.location; } | 283 Location location() const { return current_.location; } |
| 281 Location peek_location() const { return next_.location; } | 284 Location peek_location() const { return next_.location; } |
| 282 | 285 |
| 286 // Returns the location of the last seen octal literal |
| 287 int octal_position() const { return octal_pos_; } |
| 288 void clear_octal_position() { octal_pos_ = -1; } |
| 289 |
| 283 // Returns the literal string, if any, for the current token (the | 290 // Returns the literal string, if any, for the current token (the |
| 284 // token returned by Next()). The string is 0-terminated and in | 291 // token returned by Next()). The string is 0-terminated and in |
| 285 // UTF-8 format; they may contain 0-characters. Literal strings are | 292 // UTF-8 format; they may contain 0-characters. Literal strings are |
| 286 // collected for identifiers, strings, and numbers. | 293 // collected for identifiers, strings, and numbers. |
| 287 // These functions only give the correct result if the literal | 294 // These functions only give the correct result if the literal |
| 288 // was scanned between calls to StartLiteral() and TerminateLiteral(). | 295 // was scanned between calls to StartLiteral() and TerminateLiteral(). |
| 289 bool is_literal_ascii() { | 296 bool is_literal_ascii() { |
| 290 ASSERT_NOT_NULL(current_.literal_chars); | 297 ASSERT_NOT_NULL(current_.literal_chars); |
| 291 return current_.literal_chars->is_ascii(); | 298 return current_.literal_chars->is_ascii(); |
| 292 } | 299 } |
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 403 // Buffers collecting literal strings, numbers, etc. | 410 // Buffers collecting literal strings, numbers, etc. |
| 404 LiteralBuffer literal_buffer1_; | 411 LiteralBuffer literal_buffer1_; |
| 405 LiteralBuffer literal_buffer2_; | 412 LiteralBuffer literal_buffer2_; |
| 406 | 413 |
| 407 TokenDesc current_; // desc for current token (as returned by Next()) | 414 TokenDesc current_; // desc for current token (as returned by Next()) |
| 408 TokenDesc next_; // desc for next token (one token look-ahead) | 415 TokenDesc next_; // desc for next token (one token look-ahead) |
| 409 | 416 |
| 410 // Input stream. Must be initialized to an UC16CharacterStream. | 417 // Input stream. Must be initialized to an UC16CharacterStream. |
| 411 UC16CharacterStream* source_; | 418 UC16CharacterStream* source_; |
| 412 | 419 |
| 420 // Start position of the octal literal last scanned. |
| 421 int octal_pos_; |
| 413 | 422 |
| 414 // One Unicode character look-ahead; c0_ < 0 at the end of the input. | 423 // One Unicode character look-ahead; c0_ < 0 at the end of the input. |
| 415 uc32 c0_; | 424 uc32 c0_; |
| 416 }; | 425 }; |
| 417 | 426 |
| 418 // ---------------------------------------------------------------------------- | 427 // ---------------------------------------------------------------------------- |
| 419 // JavaScriptScanner - base logic for JavaScript scanning. | 428 // JavaScriptScanner - base logic for JavaScript scanning. |
| 420 | 429 |
| 421 class JavaScriptScanner : public Scanner { | 430 class JavaScriptScanner : public Scanner { |
| 422 public: | 431 public: |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 617 // keyword with the current prefix). | 626 // keyword with the current prefix). |
| 618 const char* keyword_; | 627 const char* keyword_; |
| 619 int counter_; | 628 int counter_; |
| 620 Token::Value keyword_token_; | 629 Token::Value keyword_token_; |
| 621 }; | 630 }; |
| 622 | 631 |
| 623 | 632 |
| 624 } } // namespace v8::internal | 633 } } // namespace v8::internal |
| 625 | 634 |
| 626 #endif // V8_SCANNER_BASE_H_ | 635 #endif // V8_SCANNER_BASE_H_ |
| OLD | NEW |