OLD | NEW |
1 // Copyright 2011 the V8 project authors. All rights reserved. | 1 // Copyright 2011 the V8 project authors. All rights reserved. |
2 // Redistribution and use in source and binary forms, with or without | 2 // Redistribution and use in source and binary forms, with or without |
3 // modification, are permitted provided that the following conditions are | 3 // modification, are permitted provided that the following conditions are |
4 // met: | 4 // met: |
5 // | 5 // |
6 // * Redistributions of source code must retain the above copyright | 6 // * Redistributions of source code must retain the above copyright |
7 // notice, this list of conditions and the following disclaimer. | 7 // notice, this list of conditions and the following disclaimer. |
8 // * Redistributions in binary form must reproduce the above | 8 // * Redistributions in binary form must reproduce the above |
9 // copyright notice, this list of conditions and the following | 9 // copyright notice, this list of conditions and the following |
10 // disclaimer in the documentation and/or other materials provided | 10 // disclaimer in the documentation and/or other materials provided |
(...skipping 17 matching lines...) Expand all Loading... |
28 // Features shared by parsing and pre-parsing scanners. | 28 // Features shared by parsing and pre-parsing scanners. |
29 | 29 |
30 #include <cmath> | 30 #include <cmath> |
31 | 31 |
32 #include "scanner.h" | 32 #include "scanner.h" |
33 | 33 |
34 #include "../include/v8stdint.h" | 34 #include "../include/v8stdint.h" |
35 #include "char-predicates-inl.h" | 35 #include "char-predicates-inl.h" |
36 #include "conversions-inl.h" | 36 #include "conversions-inl.h" |
37 #include "list-inl.h" | 37 #include "list-inl.h" |
| 38 #include "v8.h" |
38 | 39 |
39 namespace v8 { | 40 namespace v8 { |
40 namespace internal { | 41 namespace internal { |
41 | 42 |
42 // ---------------------------------------------------------------------------- | 43 // ---------------------------------------------------------------------------- |
43 // Scanner | 44 // Scanner |
44 | 45 |
45 Scanner::Scanner(UnicodeCache* unicode_cache) | 46 Scanner::Scanner(UnicodeCache* unicode_cache) |
46 : unicode_cache_(unicode_cache), | 47 : unicode_cache_(unicode_cache), |
47 octal_pos_(Location::invalid()), | 48 octal_pos_(Location::invalid()), |
(...skipping 927 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
975 Advance(); | 976 Advance(); |
976 AddLiteralChar(next_char); | 977 AddLiteralChar(next_char); |
977 continue; | 978 continue; |
978 } | 979 } |
979 // Fallthrough if no longer able to complete keyword. | 980 // Fallthrough if no longer able to complete keyword. |
980 return ScanIdentifierSuffix(&literal); | 981 return ScanIdentifierSuffix(&literal); |
981 } | 982 } |
982 | 983 |
983 literal.Complete(); | 984 literal.Complete(); |
984 | 985 |
985 if (next_.literal_chars->is_ascii()) { | 986 if (next_.literal_chars->is_one_byte()) { |
986 Vector<const char> chars = next_.literal_chars->ascii_literal(); | 987 Vector<const char> chars = next_.literal_chars->one_byte_literal(); |
987 return KeywordOrIdentifierToken(chars.start(), | 988 return KeywordOrIdentifierToken(chars.start(), |
988 chars.length(), | 989 chars.length(), |
989 harmony_scoping_, | 990 harmony_scoping_, |
990 harmony_modules_); | 991 harmony_modules_); |
991 } | 992 } |
992 | 993 |
993 return Token::IDENTIFIER; | 994 return Token::IDENTIFIER; |
994 } | 995 } |
995 | 996 |
996 | 997 |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1107 Advance(); | 1108 Advance(); |
1108 } | 1109 } |
1109 } | 1110 } |
1110 literal.Complete(); | 1111 literal.Complete(); |
1111 | 1112 |
1112 next_.location.end_pos = source_pos() - 1; | 1113 next_.location.end_pos = source_pos() - 1; |
1113 return true; | 1114 return true; |
1114 } | 1115 } |
1115 | 1116 |
1116 | 1117 |
| 1118 Handle<String> Scanner::AllocateLiteralString(Isolate* isolate, |
| 1119 PretenureFlag tenured) { |
| 1120 if (is_literal_one_byte()) { |
| 1121 return isolate->factory()->NewStringFromOneByte( |
| 1122 Vector<const uint8_t>::cast(literal_one_byte_string()), tenured); |
| 1123 } else { |
| 1124 return isolate->factory()->NewStringFromTwoByte( |
| 1125 literal_utf16_string(), tenured); |
| 1126 } |
| 1127 } |
| 1128 |
| 1129 |
| 1130 Handle<String> Scanner::AllocateNextLiteralString(Isolate* isolate, |
| 1131 PretenureFlag tenured) { |
| 1132 if (is_next_literal_one_byte()) { |
| 1133 return isolate->factory()->NewStringFromOneByte( |
| 1134 Vector<const uint8_t>::cast(next_literal_one_byte_string()), tenured); |
| 1135 } else { |
| 1136 return isolate->factory()->NewStringFromTwoByte( |
| 1137 next_literal_utf16_string(), tenured); |
| 1138 } |
| 1139 } |
| 1140 |
| 1141 |
| 1142 Handle<String> Scanner::AllocateInternalizedString(Isolate* isolate) { |
| 1143 if (is_literal_one_byte()) { |
| 1144 return isolate->factory()->InternalizeOneByteString( |
| 1145 Vector<const uint8_t>::cast(literal_one_byte_string())); |
| 1146 } else { |
| 1147 return isolate->factory()->InternalizeTwoByteString( |
| 1148 literal_utf16_string()); |
| 1149 } |
| 1150 } |
| 1151 |
| 1152 |
| 1153 double Scanner::DoubleValue() { |
| 1154 ASSERT(is_literal_one_byte()); |
| 1155 return StringToDouble( |
| 1156 unicode_cache_, literal_one_byte_string(), |
| 1157 ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); |
| 1158 } |
| 1159 |
| 1160 |
1117 int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) { | 1161 int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) { |
1118 return AddSymbol(Vector<const byte>::cast(key), true, value); | 1162 return AddSymbol(Vector<const byte>::cast(key), true, value); |
1119 } | 1163 } |
1120 | 1164 |
1121 | 1165 |
1122 int DuplicateFinder::AddUtf16Symbol(Vector<const uint16_t> key, int value) { | 1166 int DuplicateFinder::AddUtf16Symbol(Vector<const uint16_t> key, int value) { |
1123 return AddSymbol(Vector<const byte>::cast(key), false, value); | 1167 return AddSymbol(Vector<const byte>::cast(key), false, value); |
1124 } | 1168 } |
1125 | 1169 |
1126 | 1170 |
1127 int DuplicateFinder::AddSymbol(Vector<const byte> key, | 1171 int DuplicateFinder::AddSymbol(Vector<const byte> key, |
1128 bool is_ascii, | 1172 bool is_one_byte, |
1129 int value) { | 1173 int value) { |
1130 uint32_t hash = Hash(key, is_ascii); | 1174 uint32_t hash = Hash(key, is_one_byte); |
1131 byte* encoding = BackupKey(key, is_ascii); | 1175 byte* encoding = BackupKey(key, is_one_byte); |
1132 HashMap::Entry* entry = map_.Lookup(encoding, hash, true); | 1176 HashMap::Entry* entry = map_.Lookup(encoding, hash, true); |
1133 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); | 1177 int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); |
1134 entry->value = | 1178 entry->value = |
1135 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); | 1179 reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); |
1136 return old_value; | 1180 return old_value; |
1137 } | 1181 } |
1138 | 1182 |
1139 | 1183 |
1140 int DuplicateFinder::AddNumber(Vector<const char> key, int value) { | 1184 int DuplicateFinder::AddNumber(Vector<const char> key, int value) { |
1141 ASSERT(key.length() > 0); | 1185 ASSERT(key.length() > 0); |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1182 while (pos < length) { | 1226 while (pos < length) { |
1183 byte digit = number[pos] - '0'; | 1227 byte digit = number[pos] - '0'; |
1184 if (digit > '9' - '0') return false; | 1228 if (digit > '9' - '0') return false; |
1185 invalid_last_digit = (digit == 0); | 1229 invalid_last_digit = (digit == 0); |
1186 pos++; | 1230 pos++; |
1187 } | 1231 } |
1188 return !invalid_last_digit; | 1232 return !invalid_last_digit; |
1189 } | 1233 } |
1190 | 1234 |
1191 | 1235 |
1192 uint32_t DuplicateFinder::Hash(Vector<const byte> key, bool is_ascii) { | 1236 uint32_t DuplicateFinder::Hash(Vector<const byte> key, bool is_one_byte) { |
1193 // Primitive hash function, almost identical to the one used | 1237 // Primitive hash function, almost identical to the one used |
1194 // for strings (except that it's seeded by the length and ASCII-ness). | 1238 // for strings (except that it's seeded by the length and ASCII-ness). |
1195 int length = key.length(); | 1239 int length = key.length(); |
1196 uint32_t hash = (length << 1) | (is_ascii ? 1 : 0) ; | 1240 uint32_t hash = (length << 1) | (is_one_byte ? 1 : 0) ; |
1197 for (int i = 0; i < length; i++) { | 1241 for (int i = 0; i < length; i++) { |
1198 uint32_t c = key[i]; | 1242 uint32_t c = key[i]; |
1199 hash = (hash + c) * 1025; | 1243 hash = (hash + c) * 1025; |
1200 hash ^= (hash >> 6); | 1244 hash ^= (hash >> 6); |
1201 } | 1245 } |
1202 return hash; | 1246 return hash; |
1203 } | 1247 } |
1204 | 1248 |
1205 | 1249 |
1206 bool DuplicateFinder::Match(void* first, void* second) { | 1250 bool DuplicateFinder::Match(void* first, void* second) { |
1207 // Decode lengths. | 1251 // Decode lengths. |
1208 // Length + ASCII-bit is encoded as base 128, most significant heptet first, | 1252 // Length + ASCII-bit is encoded as base 128, most significant heptet first, |
1209 // with a 8th bit being non-zero while there are more heptets. | 1253 // with a 8th bit being non-zero while there are more heptets. |
1210 // The value encodes the number of bytes following, and whether the original | 1254 // The value encodes the number of bytes following, and whether the original |
1211 // was ASCII. | 1255 // was ASCII. |
1212 byte* s1 = reinterpret_cast<byte*>(first); | 1256 byte* s1 = reinterpret_cast<byte*>(first); |
1213 byte* s2 = reinterpret_cast<byte*>(second); | 1257 byte* s2 = reinterpret_cast<byte*>(second); |
1214 uint32_t length_ascii_field = 0; | 1258 uint32_t length_one_byte_field = 0; |
1215 byte c1; | 1259 byte c1; |
1216 do { | 1260 do { |
1217 c1 = *s1; | 1261 c1 = *s1; |
1218 if (c1 != *s2) return false; | 1262 if (c1 != *s2) return false; |
1219 length_ascii_field = (length_ascii_field << 7) | (c1 & 0x7f); | 1263 length_one_byte_field = (length_one_byte_field << 7) | (c1 & 0x7f); |
1220 s1++; | 1264 s1++; |
1221 s2++; | 1265 s2++; |
1222 } while ((c1 & 0x80) != 0); | 1266 } while ((c1 & 0x80) != 0); |
1223 int length = static_cast<int>(length_ascii_field >> 1); | 1267 int length = static_cast<int>(length_one_byte_field >> 1); |
1224 return memcmp(s1, s2, length) == 0; | 1268 return memcmp(s1, s2, length) == 0; |
1225 } | 1269 } |
1226 | 1270 |
1227 | 1271 |
1228 byte* DuplicateFinder::BackupKey(Vector<const byte> bytes, | 1272 byte* DuplicateFinder::BackupKey(Vector<const byte> bytes, |
1229 bool is_ascii) { | 1273 bool is_one_byte) { |
1230 uint32_t ascii_length = (bytes.length() << 1) | (is_ascii ? 1 : 0); | 1274 uint32_t one_byte_length = (bytes.length() << 1) | (is_one_byte ? 1 : 0); |
1231 backing_store_.StartSequence(); | 1275 backing_store_.StartSequence(); |
1232 // Emit ascii_length as base-128 encoded number, with the 7th bit set | 1276 // Emit one_byte_length as base-128 encoded number, with the 7th bit set |
1233 // on the byte of every heptet except the last, least significant, one. | 1277 // on the byte of every heptet except the last, least significant, one. |
1234 if (ascii_length >= (1 << 7)) { | 1278 if (one_byte_length >= (1 << 7)) { |
1235 if (ascii_length >= (1 << 14)) { | 1279 if (one_byte_length >= (1 << 14)) { |
1236 if (ascii_length >= (1 << 21)) { | 1280 if (one_byte_length >= (1 << 21)) { |
1237 if (ascii_length >= (1 << 28)) { | 1281 if (one_byte_length >= (1 << 28)) { |
1238 backing_store_.Add(static_cast<byte>((ascii_length >> 28) | 0x80)); | 1282 backing_store_.Add(static_cast<byte>((one_byte_length >> 28) | 0x80)); |
1239 } | 1283 } |
1240 backing_store_.Add(static_cast<byte>((ascii_length >> 21) | 0x80u)); | 1284 backing_store_.Add(static_cast<byte>((one_byte_length >> 21) | 0x80u)); |
1241 } | 1285 } |
1242 backing_store_.Add(static_cast<byte>((ascii_length >> 14) | 0x80u)); | 1286 backing_store_.Add(static_cast<byte>((one_byte_length >> 14) | 0x80u)); |
1243 } | 1287 } |
1244 backing_store_.Add(static_cast<byte>((ascii_length >> 7) | 0x80u)); | 1288 backing_store_.Add(static_cast<byte>((one_byte_length >> 7) | 0x80u)); |
1245 } | 1289 } |
1246 backing_store_.Add(static_cast<byte>(ascii_length & 0x7f)); | 1290 backing_store_.Add(static_cast<byte>(one_byte_length & 0x7f)); |
1247 | 1291 |
1248 backing_store_.AddBlock(bytes); | 1292 backing_store_.AddBlock(bytes); |
1249 return backing_store_.EndSequence().start(); | 1293 return backing_store_.EndSequence().start(); |
1250 } | 1294 } |
1251 | 1295 |
1252 } } // namespace v8::internal | 1296 } } // namespace v8::internal |
OLD | NEW |