OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
6 | 6 |
7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
10 #include "vm/object.h" | 10 #include "vm/object.h" |
11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
13 #include "vm/thread.h" | 13 #include "vm/thread.h" |
14 #include "vm/token.h" | 14 #include "vm/token.h" |
15 #include "vm/unicode.h" | 15 #include "vm/unicode.h" |
16 | 16 |
17 namespace dart { | 17 namespace dart { |
18 | 18 |
19 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 19 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
20 | 20 |
21 | 21 |
22 Scanner::KeywordTable Scanner::keywords_[Token::numKeywords]; | 22 Scanner::KeywordTable Scanner::keywords_[Token::kNumKeywords]; |
| 23 int Scanner::keywords_char_offset_[Scanner::kNumLowercaseChars]; |
23 | 24 |
24 | 25 |
25 void Scanner::Reset() { | 26 void Scanner::Reset() { |
26 // Non-changing newline properties. | 27 // Non-changing newline properties. |
27 newline_token_.kind = Token::kNEWLINE; | 28 newline_token_.kind = Token::kNEWLINE; |
28 newline_token_.literal = NULL; | 29 newline_token_.literal = NULL; |
29 // We don't preserve the column information. | 30 // We don't preserve the column information. |
30 newline_token_.position.column = 0; | 31 newline_token_.position.column = 0; |
31 | 32 |
32 // Non-changing empty string token properties. | 33 // Non-changing empty string token properties. |
(...skipping 249 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
282 int ident_length = 0; | 283 int ident_length = 0; |
283 int ident_pos = lookahead_pos_; | 284 int ident_pos = lookahead_pos_; |
284 int32_t ident_char0 = source_.CharAt(ident_pos); | 285 int32_t ident_char0 = source_.CharAt(ident_pos); |
285 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { | 286 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { |
286 ReadChar(); | 287 ReadChar(); |
287 ident_length++; | 288 ident_length++; |
288 } | 289 } |
289 | 290 |
290 // Check whether the characters we read are a known keyword. | 291 // Check whether the characters we read are a known keyword. |
291 // Note, can't use strcmp since token_chars is not null-terminated. | 292 // Note, can't use strcmp since token_chars is not null-terminated. |
292 int i = 0; | 293 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { |
293 while (i < Token::numKeywords && | 294 int i = keywords_char_offset_[ident_char0 - 'a']; |
294 keywords_[i].keyword_chars[0] <= ident_char0) { | 295 while (i < Token::kNumKeywords && |
295 if (keywords_[i].keyword_len == ident_length) { | 296 keywords_[i].keyword_chars[0] <= ident_char0) { |
296 const char* keyword = keywords_[i].keyword_chars; | 297 if (keywords_[i].keyword_len == ident_length) { |
297 int char_pos = 0; | 298 const char* keyword = keywords_[i].keyword_chars; |
298 while ((char_pos < ident_length) && | 299 int char_pos = 1; |
299 (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) { | 300 while ((char_pos < ident_length) && |
300 char_pos++; | 301 (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) { |
| 302 char_pos++; |
| 303 } |
| 304 if (char_pos == ident_length) { |
| 305 current_token_.literal = keywords_[i].keyword_symbol; |
| 306 current_token_.kind = keywords_[i].kind; |
| 307 return; |
| 308 } |
301 } | 309 } |
302 if (char_pos == ident_length) { | 310 i++; |
303 current_token_.literal = keywords_[i].keyword_symbol; | |
304 current_token_.kind = keywords_[i].kind; | |
305 return; | |
306 } | |
307 } | 311 } |
308 i++; | |
309 } | 312 } |
310 | 313 |
311 // We did not read a keyword. | 314 // We did not read a keyword. |
312 current_token_.kind = Token::kIDENT; | 315 current_token_.kind = Token::kIDENT; |
313 String& literal = | 316 String& literal = |
314 String::ZoneHandle(Symbols::New(source_, ident_pos, ident_length)); | 317 String::ZoneHandle(Symbols::New(source_, ident_pos, ident_length)); |
315 if (ident_char0 == Library::kPrivateIdentifierStart) { | 318 if (ident_char0 == Library::kPrivateIdentifierStart) { |
316 // Private identifiers are mangled on a per library basis. | 319 // Private identifiers are mangled on a per library basis. |
317 literal = String::Concat(literal, private_key_); | 320 literal = String::Concat(literal, private_key_); |
318 literal = Symbols::New(literal); | 321 literal = Symbols::New(literal); |
(...skipping 611 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
930 OS::Print("\n%d (%d): ", currentLine, i); | 933 OS::Print("\n%d (%d): ", currentLine, i); |
931 } | 934 } |
932 OS::Print("%s ", Token::Name(td.kind)); | 935 OS::Print("%s ", Token::Name(td.kind)); |
933 } | 936 } |
934 OS::Print("\n"); | 937 OS::Print("\n"); |
935 } | 938 } |
936 | 939 |
937 | 940 |
938 void Scanner::InitOnce() { | 941 void Scanner::InitOnce() { |
939 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 942 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
940 for (int i = 0; i < Token::numKeywords; i++) { | 943 for (int i = 0; i < kNumLowercaseChars; i++) { |
| 944 keywords_char_offset_[i] = Token::kNumKeywords; |
| 945 } |
| 946 for (int i = 0; i < Token::kNumKeywords; i++) { |
941 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 947 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
942 keywords_[i].kind = token; | 948 keywords_[i].kind = token; |
943 keywords_[i].keyword_chars = Token::Str(token); | 949 keywords_[i].keyword_chars = Token::Str(token); |
944 keywords_[i].keyword_len = strlen(Token::Str(token)); | 950 keywords_[i].keyword_len = strlen(Token::Str(token)); |
945 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 951 keywords_[i].keyword_symbol = &Symbols::Keyword(token); |
| 952 |
| 953 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 954 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 955 keywords_char_offset_[ch] = i; |
| 956 } |
946 } | 957 } |
947 } | 958 } |
948 | 959 |
949 } // namespace dart | 960 } // namespace dart |
OLD | NEW |