OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Features shared by parsing and pre-parsing scanners. | 5 // Features shared by parsing and pre-parsing scanners. |
6 | 6 |
7 #include <stdint.h> | 7 #include <stdint.h> |
8 | 8 |
9 #include <cmath> | 9 #include <cmath> |
10 | 10 |
(...skipping 21 matching lines...) Expand all Loading... |
32 // ---------------------------------------------------------------------------- | 32 // ---------------------------------------------------------------------------- |
33 // Scanner | 33 // Scanner |
34 | 34 |
35 Scanner::Scanner(UnicodeCache* unicode_cache) | 35 Scanner::Scanner(UnicodeCache* unicode_cache) |
36 : unicode_cache_(unicode_cache), | 36 : unicode_cache_(unicode_cache), |
37 octal_pos_(Location::invalid()), | 37 octal_pos_(Location::invalid()), |
38 harmony_scoping_(false), | 38 harmony_scoping_(false), |
39 harmony_modules_(false), | 39 harmony_modules_(false), |
40 harmony_numeric_literals_(false), | 40 harmony_numeric_literals_(false), |
41 harmony_classes_(false), | 41 harmony_classes_(false), |
42 harmony_templates_(false) {} | 42 harmony_templates_(false), |
| 43 harmony_unicode_(false) {} |
43 | 44 |
44 | 45 |
45 void Scanner::Initialize(Utf16CharacterStream* source) { | 46 void Scanner::Initialize(Utf16CharacterStream* source) { |
46 source_ = source; | 47 source_ = source; |
47 // Need to capture identifiers in order to recognize "get" and "set" | 48 // Need to capture identifiers in order to recognize "get" and "set" |
48 // in object literals. | 49 // in object literals. |
49 Init(); | 50 Init(); |
50 // Skip initial whitespace allowing HTML comment ends just like | 51 // Skip initial whitespace allowing HTML comment ends just like |
51 // after a newline and scan first token. | 52 // after a newline and scan first token. |
52 has_line_terminator_before_next_ = true; | 53 has_line_terminator_before_next_ = true; |
(...skipping 12 matching lines...) Expand all Loading... |
65 return -1; | 66 return -1; |
66 } | 67 } |
67 x = x * 16 + d; | 68 x = x * 16 + d; |
68 Advance(); | 69 Advance(); |
69 } | 70 } |
70 | 71 |
71 return x; | 72 return x; |
72 } | 73 } |
73 | 74 |
74 | 75 |
| 76 uc32 Scanner::ScanUnlimitedLengthHexNumber(int max_value) { |
| 77 uc32 x = 0; |
| 78 int d = HexValue(c0_); |
| 79 if (d < 0) { |
| 80 return -1; |
| 81 } |
| 82 while (d >= 0) { |
| 83 x = x * 16 + d; |
| 84 if (x > max_value) return -1; |
| 85 Advance(); |
| 86 d = HexValue(c0_); |
| 87 } |
| 88 return x; |
| 89 } |
| 90 |
| 91 |
75 // Ensure that tokens can be stored in a byte. | 92 // Ensure that tokens can be stored in a byte. |
76 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); | 93 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); |
77 | 94 |
78 // Table of one-character tokens, by character (0x00..0x7f only). | 95 // Table of one-character tokens, by character (0x00..0x7f only). |
79 static const byte one_char_tokens[] = { | 96 static const byte one_char_tokens[] = { |
80 Token::ILLEGAL, | 97 Token::ILLEGAL, |
81 Token::ILLEGAL, | 98 Token::ILLEGAL, |
82 Token::ILLEGAL, | 99 Token::ILLEGAL, |
83 Token::ILLEGAL, | 100 Token::ILLEGAL, |
84 Token::ILLEGAL, | 101 Token::ILLEGAL, |
(...skipping 608 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
693 switch (c) { | 710 switch (c) { |
694 case '\'': // fall through | 711 case '\'': // fall through |
695 case '"' : // fall through | 712 case '"' : // fall through |
696 case '\\': break; | 713 case '\\': break; |
697 case 'b' : c = '\b'; break; | 714 case 'b' : c = '\b'; break; |
698 case 'f' : c = '\f'; break; | 715 case 'f' : c = '\f'; break; |
699 case 'n' : c = '\n'; break; | 716 case 'n' : c = '\n'; break; |
700 case 'r' : c = '\r'; break; | 717 case 'r' : c = '\r'; break; |
701 case 't' : c = '\t'; break; | 718 case 't' : c = '\t'; break; |
702 case 'u' : { | 719 case 'u' : { |
703 c = ScanHexNumber(4); | 720 c = ScanUnicodeEscape(); |
704 if (c < 0) return false; | 721 if (c < 0) return false; |
705 break; | 722 break; |
706 } | 723 } |
707 case 'v' : c = '\v'; break; | 724 case 'v' : c = '\v'; break; |
708 case 'x' : { | 725 case 'x' : { |
709 c = ScanHexNumber(2); | 726 c = ScanHexNumber(2); |
710 if (c < 0) return false; | 727 if (c < 0) return false; |
711 break; | 728 break; |
712 } | 729 } |
713 case '0' : // fall through | 730 case '0' : // fall through |
(...skipping 243 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
957 literal.Complete(); | 974 literal.Complete(); |
958 | 975 |
959 return Token::NUMBER; | 976 return Token::NUMBER; |
960 } | 977 } |
961 | 978 |
962 | 979 |
963 uc32 Scanner::ScanIdentifierUnicodeEscape() { | 980 uc32 Scanner::ScanIdentifierUnicodeEscape() { |
964 Advance(); | 981 Advance(); |
965 if (c0_ != 'u') return -1; | 982 if (c0_ != 'u') return -1; |
966 Advance(); | 983 Advance(); |
| 984 return ScanUnicodeEscape(); |
| 985 } |
| 986 |
| 987 |
| 988 uc32 Scanner::ScanUnicodeEscape() { |
| 989 // Accept both \uxxxx and \u{xxxxxx} (if harmony unicode escapes are |
| 990 // allowed). In the latter case, the number of hex digits between { } is |
| 991 // arbitrary. \ and u have already been read. |
| 992 if (c0_ == '{' && HarmonyUnicode()) { |
| 993 Advance(); |
| 994 uc32 cp = ScanUnlimitedLengthHexNumber(0x10ffff); |
| 995 if (cp < 0) { |
| 996 return -1; |
| 997 } |
| 998 if (c0_ != '}') { |
| 999 return -1; |
| 1000 } |
| 1001 Advance(); |
| 1002 return cp; |
| 1003 } |
967 return ScanHexNumber(4); | 1004 return ScanHexNumber(4); |
968 } | 1005 } |
969 | 1006 |
970 | 1007 |
971 // ---------------------------------------------------------------------------- | 1008 // ---------------------------------------------------------------------------- |
972 // Keyword Matcher | 1009 // Keyword Matcher |
973 | 1010 |
974 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ | 1011 #define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ |
975 KEYWORD_GROUP('b') \ | 1012 KEYWORD_GROUP('b') \ |
976 KEYWORD("break", Token::BREAK) \ | 1013 KEYWORD("break", Token::BREAK) \ |
(...skipping 426 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1403 } | 1440 } |
1404 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); | 1441 backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); |
1405 } | 1442 } |
1406 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); | 1443 backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); |
1407 | 1444 |
1408 backing_store_.AddBlock(bytes); | 1445 backing_store_.AddBlock(bytes); |
1409 return backing_store_.EndSequence().start(); | 1446 return backing_store_.EndSequence().start(); |
1410 } | 1447 } |
1411 | 1448 |
1412 } } // namespace v8::internal | 1449 } } // namespace v8::internal |
OLD | NEW |