| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
| 6 | 6 |
| 7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
| 8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/object.h" | 10 #include "vm/object.h" |
| 11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
| 12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
| 13 #include "vm/thread.h" | 13 #include "vm/thread.h" |
| 14 #include "vm/token.h" | 14 #include "vm/token.h" |
| 15 #include "vm/unicode.h" | 15 #include "vm/unicode.h" |
| 16 | 16 |
| 17 namespace dart { | 17 namespace dart { |
| 18 | 18 |
| 19 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 19 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
| 20 | 20 |
| 21 | 21 |
| 22 Scanner::KeywordTable Scanner::keywords_[Token::numKeywords]; | 22 Scanner::KeywordTable Scanner::keywords_[Token::kNumKeywords]; |
| 23 int Scanner::keywords_char_offset_[Scanner::kNumLowercaseChars]; |
| 23 | 24 |
| 24 | 25 |
| 25 void Scanner::Reset() { | 26 void Scanner::Reset() { |
| 26 // Non-changing newline properties. | 27 // Non-changing newline properties. |
| 27 newline_token_.kind = Token::kNEWLINE; | 28 newline_token_.kind = Token::kNEWLINE; |
| 28 newline_token_.literal = NULL; | 29 newline_token_.literal = NULL; |
| 29 // We don't preserve the column information. | 30 // We don't preserve the column information. |
| 30 newline_token_.position.column = 0; | 31 newline_token_.position.column = 0; |
| 31 | 32 |
| 32 // Non-changing empty string token properties. | 33 // Non-changing empty string token properties. |
| (...skipping 249 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 282 int ident_length = 0; | 283 int ident_length = 0; |
| 283 int ident_pos = lookahead_pos_; | 284 int ident_pos = lookahead_pos_; |
| 284 int32_t ident_char0 = source_.CharAt(ident_pos); | 285 int32_t ident_char0 = source_.CharAt(ident_pos); |
| 285 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { | 286 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { |
| 286 ReadChar(); | 287 ReadChar(); |
| 287 ident_length++; | 288 ident_length++; |
| 288 } | 289 } |
| 289 | 290 |
| 290 // Check whether the characters we read are a known keyword. | 291 // Check whether the characters we read are a known keyword. |
| 291 // Note, can't use strcmp since token_chars is not null-terminated. | 292 // Note, can't use strcmp since token_chars is not null-terminated. |
| 292 int i = 0; | 293 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { |
| 293 while (i < Token::numKeywords && | 294 int i = keywords_char_offset_[ident_char0 - 'a']; |
| 294 keywords_[i].keyword_chars[0] <= ident_char0) { | 295 while (i < Token::kNumKeywords && |
| 295 if (keywords_[i].keyword_len == ident_length) { | 296 keywords_[i].keyword_chars[0] <= ident_char0) { |
| 296 const char* keyword = keywords_[i].keyword_chars; | 297 if (keywords_[i].keyword_len == ident_length) { |
| 297 int char_pos = 0; | 298 const char* keyword = keywords_[i].keyword_chars; |
| 298 while ((char_pos < ident_length) && | 299 int char_pos = 1; |
| 299 (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) { | 300 while ((char_pos < ident_length) && |
| 300 char_pos++; | 301 (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) { |
| 302 char_pos++; |
| 303 } |
| 304 if (char_pos == ident_length) { |
| 305 current_token_.literal = keywords_[i].keyword_symbol; |
| 306 current_token_.kind = keywords_[i].kind; |
| 307 return; |
| 308 } |
| 301 } | 309 } |
| 302 if (char_pos == ident_length) { | 310 i++; |
| 303 current_token_.literal = keywords_[i].keyword_symbol; | |
| 304 current_token_.kind = keywords_[i].kind; | |
| 305 return; | |
| 306 } | |
| 307 } | 311 } |
| 308 i++; | |
| 309 } | 312 } |
| 310 | 313 |
| 311 // We did not read a keyword. | 314 // We did not read a keyword. |
| 312 current_token_.kind = Token::kIDENT; | 315 current_token_.kind = Token::kIDENT; |
| 313 String& literal = | 316 String& literal = |
| 314 String::ZoneHandle(Symbols::New(source_, ident_pos, ident_length)); | 317 String::ZoneHandle(Symbols::New(source_, ident_pos, ident_length)); |
| 315 if (ident_char0 == Library::kPrivateIdentifierStart) { | 318 if (ident_char0 == Library::kPrivateIdentifierStart) { |
| 316 // Private identifiers are mangled on a per library basis. | 319 // Private identifiers are mangled on a per library basis. |
| 317 literal = String::Concat(literal, private_key_); | 320 literal = String::Concat(literal, private_key_); |
| 318 literal = Symbols::New(literal); | 321 literal = Symbols::New(literal); |
| (...skipping 611 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 930 OS::Print("\n%d (%d): ", currentLine, i); | 933 OS::Print("\n%d (%d): ", currentLine, i); |
| 931 } | 934 } |
| 932 OS::Print("%s ", Token::Name(td.kind)); | 935 OS::Print("%s ", Token::Name(td.kind)); |
| 933 } | 936 } |
| 934 OS::Print("\n"); | 937 OS::Print("\n"); |
| 935 } | 938 } |
| 936 | 939 |
| 937 | 940 |
| 938 void Scanner::InitOnce() { | 941 void Scanner::InitOnce() { |
| 939 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 942 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
| 940 for (int i = 0; i < Token::numKeywords; i++) { | 943 for (int i = 0; i < kNumLowercaseChars; i++) { |
| 944 keywords_char_offset_[i] = Token::kNumKeywords; |
| 945 } |
| 946 for (int i = 0; i < Token::kNumKeywords; i++) { |
| 941 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 947 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
| 942 keywords_[i].kind = token; | 948 keywords_[i].kind = token; |
| 943 keywords_[i].keyword_chars = Token::Str(token); | 949 keywords_[i].keyword_chars = Token::Str(token); |
| 944 keywords_[i].keyword_len = strlen(Token::Str(token)); | 950 keywords_[i].keyword_len = strlen(Token::Str(token)); |
| 945 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 951 keywords_[i].keyword_symbol = &Symbols::Keyword(token); |
| 952 |
| 953 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 954 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 955 keywords_char_offset_[ch] = i; |
| 956 } |
| 946 } | 957 } |
| 947 } | 958 } |
| 948 | 959 |
| 949 } // namespace dart | 960 } // namespace dart |
| OLD | NEW |