| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
| 6 | 6 |
| 7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
| 8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/object.h" | 10 #include "vm/object.h" |
| 11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
| 12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
| 13 #include "vm/token.h" | 13 #include "vm/token.h" |
| 14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
| 15 | 15 |
| 16 namespace dart { | 16 namespace dart { |
| 17 | 17 |
| 18 // Quick access to the locally defined zone() and thread() methods. | 18 // Quick access to the locally defined zone() and thread() methods. |
| 19 #define Z (zone()) | 19 #define Z (zone()) |
| 20 #define T (thread()) | 20 #define T (thread()) |
| 21 | 21 |
| 22 | 22 |
| 23 class ScanContext : public ZoneAllocated { | 23 class ScanContext : public ZoneAllocated { |
| 24 public: | 24 public: |
| 25 explicit ScanContext(Scanner* scanner) | 25 explicit ScanContext(Scanner* scanner) |
| 26 : next_(scanner->saved_context_), | 26 : next_(scanner->saved_context_), |
| 27 string_delimiter_(scanner->string_delimiter_), | 27 string_delimiter_(scanner->string_delimiter_), |
| 28 string_is_multiline_(scanner->string_is_multiline_), | 28 string_is_multiline_(scanner->string_is_multiline_), |
| 29 brace_level_(scanner->brace_level_) {} | 29 brace_level_(scanner->brace_level_) {} |
| 30 | 30 |
| 31 void CopyTo(Scanner* scanner) { | 31 void CopyTo(Scanner* scanner) { |
| 32 scanner->string_delimiter_ = string_delimiter_; | 32 scanner->string_delimiter_ = string_delimiter_; |
| 33 scanner->string_is_multiline_ = string_is_multiline_; | 33 scanner->string_is_multiline_ = string_is_multiline_; |
| 34 scanner->brace_level_ = brace_level_; | 34 scanner->brace_level_ = brace_level_; |
| 35 } | 35 } |
| 36 | 36 |
| 37 ScanContext* next() const { return next_; } | 37 ScanContext* next() const { return next_; } |
| 38 | 38 |
| 39 private: | 39 private: |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 93 void Scanner::ErrorMsg(const char* msg) { | 93 void Scanner::ErrorMsg(const char* msg) { |
| 94 current_token_.kind = Token::kERROR; | 94 current_token_.kind = Token::kERROR; |
| 95 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(T, msg)); | 95 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(T, msg)); |
| 96 current_token_.position = c0_pos_; | 96 current_token_.position = c0_pos_; |
| 97 token_start_ = lookahead_pos_; | 97 token_start_ = lookahead_pos_; |
| 98 current_token_.offset = lookahead_pos_; | 98 current_token_.offset = lookahead_pos_; |
| 99 } | 99 } |
| 100 | 100 |
| 101 | 101 |
| 102 void Scanner::PushContext() { | 102 void Scanner::PushContext() { |
| 103 ScanContext* ctx = new(Z) ScanContext(this); | 103 ScanContext* ctx = new (Z) ScanContext(this); |
| 104 saved_context_ = ctx; | 104 saved_context_ = ctx; |
| 105 string_delimiter_ = '\0'; | 105 string_delimiter_ = '\0'; |
| 106 string_is_multiline_ = false; | 106 string_is_multiline_ = false; |
| 107 brace_level_ = 1; // Account for the opening ${ token. | 107 brace_level_ = 1; // Account for the opening ${ token. |
| 108 } | 108 } |
| 109 | 109 |
| 110 | 110 |
| 111 void Scanner::PopContext() { | 111 void Scanner::PopContext() { |
| 112 ASSERT(saved_context_ != NULL); | 112 ASSERT(saved_context_ != NULL); |
| 113 ASSERT(brace_level_ == 0); | 113 ASSERT(brace_level_ == 0); |
| (...skipping 25 matching lines...) Expand all Loading... |
| 139 return '0' <= c && c <= '9'; | 139 return '0' <= c && c <= '9'; |
| 140 } | 140 } |
| 141 | 141 |
| 142 | 142 |
| 143 bool Scanner::IsNumberStart(int32_t ch) { | 143 bool Scanner::IsNumberStart(int32_t ch) { |
| 144 return IsDecimalDigit(ch) || ch == '.'; | 144 return IsDecimalDigit(ch) || ch == '.'; |
| 145 } | 145 } |
| 146 | 146 |
| 147 | 147 |
| 148 bool Scanner::IsHexDigit(int32_t c) { | 148 bool Scanner::IsHexDigit(int32_t c) { |
| 149 return IsDecimalDigit(c) | 149 return IsDecimalDigit(c) || (('A' <= c) && (c <= 'F')) || |
| 150 || (('A' <= c) && (c <= 'F')) | 150 (('a' <= c) && (c <= 'f')); |
| 151 || (('a' <= c) && (c <= 'f')); | |
| 152 } | 151 } |
| 153 | 152 |
| 154 | 153 |
| 155 bool Scanner::IsIdentStartChar(int32_t c) { | 154 bool Scanner::IsIdentStartChar(int32_t c) { |
| 156 return IsLetter(c) || (c == '_') || (c == '$'); | 155 return IsLetter(c) || (c == '_') || (c == '$'); |
| 157 } | 156 } |
| 158 | 157 |
| 159 | 158 |
| 160 bool Scanner::IsIdentChar(int32_t c) { | 159 bool Scanner::IsIdentChar(int32_t c) { |
| 161 return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$'); | 160 return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$'); |
| 162 } | 161 } |
| 163 | 162 |
| 164 | 163 |
| 165 bool Scanner::IsIdent(const String& str) { | 164 bool Scanner::IsIdent(const String& str) { |
| 166 if (!str.IsOneByteString()) { | 165 if (!str.IsOneByteString()) { |
| 167 return false; | 166 return false; |
| 168 } | 167 } |
| 169 if (str.Length() == 0 || !IsIdentStartChar(CallCharAt()(str, 0))) { | 168 if (str.Length() == 0 || !IsIdentStartChar(CallCharAt()(str, 0))) { |
| 170 return false; | 169 return false; |
| 171 } | 170 } |
| 172 for (int i = 1; i < str.Length(); i++) { | 171 for (int i = 1; i < str.Length(); i++) { |
| 173 if (!IsIdentChar(CallCharAt()(str, i))) { | 172 if (!IsIdentChar(CallCharAt()(str, i))) { |
| 174 return false; | 173 return false; |
| 175 } | 174 } |
| 176 } | 175 } |
| 177 return true; | 176 return true; |
| 178 } | 177 } |
| 179 | 178 |
| 180 | 179 |
| 181 // This method is used when parsing integers in Dart code. We | 180 // This method is used when parsing integers in Dart code. We |
| 182 // are reusing the Scanner's handling of number literals in that situation. | 181 // are reusing the Scanner's handling of number literals in that situation. |
| 183 bool Scanner::IsValidInteger(const String& str, | 182 bool Scanner::IsValidInteger(const String& str, |
| 184 bool* is_positive, | 183 bool* is_positive, |
| 185 const String** value) { | 184 const String** value) { |
| 186 Scanner s(str, Symbols::Empty()); | 185 Scanner s(str, Symbols::Empty()); |
| 187 TokenDescriptor tokens[3]; | 186 TokenDescriptor tokens[3]; |
| 188 s.Scan(); | 187 s.Scan(); |
| 189 tokens[0] = s.current_token(); | 188 tokens[0] = s.current_token(); |
| 190 s.Scan(); | 189 s.Scan(); |
| 191 tokens[1] = s.current_token(); | 190 tokens[1] = s.current_token(); |
| 192 s.Scan(); | 191 s.Scan(); |
| 193 tokens[2] = s.current_token(); | 192 tokens[2] = s.current_token(); |
| 194 | 193 |
| 195 if ((tokens[0].kind == Token::kINTEGER) && | 194 if ((tokens[0].kind == Token::kINTEGER) && (tokens[1].kind == Token::kEOS)) { |
| 196 (tokens[1].kind == Token::kEOS)) { | |
| 197 *is_positive = true; | 195 *is_positive = true; |
| 198 *value = tokens[0].literal; | 196 *value = tokens[0].literal; |
| 199 return true; | 197 return true; |
| 200 } | 198 } |
| 201 if (((tokens[0].kind == Token::kADD) || | 199 if (((tokens[0].kind == Token::kADD) || (tokens[0].kind == Token::kSUB)) && |
| 202 (tokens[0].kind == Token::kSUB)) && | 200 (tokens[1].kind == Token::kINTEGER) && (tokens[2].kind == Token::kEOS)) { |
| 203 (tokens[1].kind == Token::kINTEGER) && | |
| 204 (tokens[2].kind == Token::kEOS)) { | |
| 205 // Check there is no space between "+/-" and number. | 201 // Check there is no space between "+/-" and number. |
| 206 if ((tokens[0].offset + 1) != tokens[1].offset) { | 202 if ((tokens[0].offset + 1) != tokens[1].offset) { |
| 207 return false; | 203 return false; |
| 208 } | 204 } |
| 209 *is_positive = tokens[0].kind == Token::kADD; | 205 *is_positive = tokens[0].kind == Token::kADD; |
| 210 *value = tokens[1].literal; | 206 *value = tokens[1].literal; |
| 211 return true; | 207 return true; |
| 212 } | 208 } |
| 213 return false; | 209 return false; |
| 214 } | 210 } |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 284 ReadChar(); // Consume asterisk. | 280 ReadChar(); // Consume asterisk. |
| 285 } else if (c == '*' && c0_ == '/') { | 281 } else if (c == '*' && c0_ == '/') { |
| 286 nesting_level--; | 282 nesting_level--; |
| 287 ReadChar(); // Consume slash. | 283 ReadChar(); // Consume slash. |
| 288 if (nesting_level == 0) { | 284 if (nesting_level == 0) { |
| 289 break; | 285 break; |
| 290 } | 286 } |
| 291 } | 287 } |
| 292 } | 288 } |
| 293 current_token_.kind = | 289 current_token_.kind = |
| 294 (nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL; | 290 (nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL; |
| 295 } | 291 } |
| 296 | 292 |
| 297 | 293 |
| 298 void Scanner::ScanIdentChars(bool allow_dollar) { | 294 void Scanner::ScanIdentChars(bool allow_dollar) { |
| 299 ASSERT(IsIdentStartChar(c0_)); | 295 ASSERT(IsIdentStartChar(c0_)); |
| 300 ASSERT(allow_dollar || (c0_ != '$')); | 296 ASSERT(allow_dollar || (c0_ != '$')); |
| 301 int ident_length = 0; | 297 int ident_length = 0; |
| 302 int ident_pos = lookahead_pos_; | 298 int ident_pos = lookahead_pos_; |
| 303 int32_t ident_char0 = CallCharAt()(source_, ident_pos); | 299 int32_t ident_char0 = CallCharAt()(source_, ident_pos); |
| 304 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { | 300 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { |
| 305 ReadChar(); | 301 ReadChar(); |
| 306 ident_length++; | 302 ident_length++; |
| 307 } | 303 } |
| 308 | 304 |
| 309 // Check whether the characters we read are a known keyword. | 305 // Check whether the characters we read are a known keyword. |
| 310 // Note, can't use strcmp since token_chars is not null-terminated. | 306 // Note, can't use strcmp since token_chars is not null-terminated. |
| 311 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { | 307 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { |
| 312 int i = keywords_char_offset_[ident_char0 - 'a']; | 308 int i = keywords_char_offset_[ident_char0 - 'a']; |
| 313 while ((i < Token::kNumKeywords) && | 309 while ((i < Token::kNumKeywords) && |
| 314 (keywords_[i].keyword_chars[0] <= ident_char0)) { | 310 (keywords_[i].keyword_chars[0] <= ident_char0)) { |
| 315 if (keywords_[i].keyword_len == ident_length) { | 311 if (keywords_[i].keyword_len == ident_length) { |
| 316 const char* keyword = keywords_[i].keyword_chars; | 312 const char* keyword = keywords_[i].keyword_chars; |
| 317 int char_pos = 1; | 313 int char_pos = 1; |
| 318 while ((char_pos < ident_length) && | 314 while ((char_pos < ident_length) && |
| 319 (keyword[char_pos] == | 315 (keyword[char_pos] == |
| 320 CallCharAt()(source_, ident_pos + char_pos))) { | 316 CallCharAt()(source_, ident_pos + char_pos))) { |
| 321 char_pos++; | 317 char_pos++; |
| 322 } | 318 } |
| 323 if (char_pos == ident_length) { | 319 if (char_pos == ident_length) { |
| 324 current_token_.literal = keywords_[i].keyword_symbol; | 320 current_token_.literal = keywords_[i].keyword_symbol; |
| 325 current_token_.kind = keywords_[i].kind; | 321 current_token_.kind = keywords_[i].kind; |
| 326 return; | 322 return; |
| 327 } | 323 } |
| 328 } | 324 } |
| 329 i++; | 325 i++; |
| 330 } | 326 } |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 367 while (IsDecimalDigit(c0_)) { | 363 while (IsDecimalDigit(c0_)) { |
| 368 ReadChar(); | 364 ReadChar(); |
| 369 } | 365 } |
| 370 if (c0_ == '.' && !dec_point_seen && IsDecimalDigit(LookaheadChar(1))) { | 366 if (c0_ == '.' && !dec_point_seen && IsDecimalDigit(LookaheadChar(1))) { |
| 371 Recognize(Token::kDOUBLE); | 367 Recognize(Token::kDOUBLE); |
| 372 while (IsDecimalDigit(c0_)) { | 368 while (IsDecimalDigit(c0_)) { |
| 373 ReadChar(); | 369 ReadChar(); |
| 374 } | 370 } |
| 375 } | 371 } |
| 376 if (((c0_ == 'e') || (c0_ == 'E')) && | 372 if (((c0_ == 'e') || (c0_ == 'E')) && |
| 377 (IsDecimalDigit(LookaheadChar(1)) || | 373 (IsDecimalDigit(LookaheadChar(1)) || (LookaheadChar(1) == '-') || |
| 378 (LookaheadChar(1) == '-') || | |
| 379 (LookaheadChar(1) == '+'))) { | 374 (LookaheadChar(1) == '+'))) { |
| 380 Recognize(Token::kDOUBLE); | 375 Recognize(Token::kDOUBLE); |
| 381 if ((c0_ == '-') || (c0_ == '+')) { | 376 if ((c0_ == '-') || (c0_ == '+')) { |
| 382 ReadChar(); | 377 ReadChar(); |
| 383 } | 378 } |
| 384 if (!IsDecimalDigit(c0_)) { | 379 if (!IsDecimalDigit(c0_)) { |
| 385 ErrorMsg("missing exponent digits"); | 380 ErrorMsg("missing exponent digits"); |
| 386 return; | 381 return; |
| 387 } | 382 } |
| 388 while (IsDecimalDigit(c0_)) { | 383 while (IsDecimalDigit(c0_)) { |
| (...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 537 return; | 532 return; |
| 538 } | 533 } |
| 539 escape_char = c0_; | 534 escape_char = c0_; |
| 540 break; | 535 break; |
| 541 } | 536 } |
| 542 string_chars.Add(escape_char); | 537 string_chars.Add(escape_char); |
| 543 } else if (c0_ == '$' && !is_raw) { | 538 } else if (c0_ == '$' && !is_raw) { |
| 544 // Scanned a string piece. | 539 // Scanned a string piece. |
| 545 ASSERT(string_chars.data() != NULL); | 540 ASSERT(string_chars.data() != NULL); |
| 546 // Strings are canonicalized: Allocate a symbol. | 541 // Strings are canonicalized: Allocate a symbol. |
| 547 current_token_.literal = &String::ZoneHandle(Z, | 542 current_token_.literal = &String::ZoneHandle( |
| 548 Symbols::FromUTF32(T, string_chars.data(), string_chars.length())); | 543 Z, Symbols::FromUTF32(T, string_chars.data(), string_chars.length())); |
| 549 // Preserve error tokens. | 544 // Preserve error tokens. |
| 550 if (current_token_.kind != Token::kERROR) { | 545 if (current_token_.kind != Token::kERROR) { |
| 551 current_token_.kind = Token::kSTRING; | 546 current_token_.kind = Token::kSTRING; |
| 552 } | 547 } |
| 553 return; | 548 return; |
| 554 } else if (c0_ == string_delimiter_) { | 549 } else if (c0_ == string_delimiter_) { |
| 555 // Check if we are at the end of the string literal. | 550 // Check if we are at the end of the string literal. |
| 556 if (!string_is_multiline_ || | 551 if (!string_is_multiline_ || ((LookaheadChar(1) == string_delimiter_) && |
| 557 ((LookaheadChar(1) == string_delimiter_) && | 552 (LookaheadChar(2) == string_delimiter_))) { |
| 558 (LookaheadChar(2) == string_delimiter_))) { | |
| 559 if (string_is_multiline_) { | 553 if (string_is_multiline_) { |
| 560 ReadChar(); // Skip two string delimiters. | 554 ReadChar(); // Skip two string delimiters. |
| 561 ReadChar(); | 555 ReadChar(); |
| 562 } | 556 } |
| 563 // Preserve error tokens. | 557 // Preserve error tokens. |
| 564 if (current_token_.kind == Token::kERROR) { | 558 if (current_token_.kind == Token::kERROR) { |
| 565 ReadChar(); | 559 ReadChar(); |
| 566 } else { | 560 } else { |
| 567 Recognize(Token::kSTRING); | 561 Recognize(Token::kSTRING); |
| 568 ASSERT(string_chars.data() != NULL); | 562 ASSERT(string_chars.data() != NULL); |
| 569 // Strings are canonicalized: Allocate a symbol. | 563 // Strings are canonicalized: Allocate a symbol. |
| 570 current_token_.literal = &String::ZoneHandle(Z, | 564 current_token_.literal = |
| 571 Symbols::FromUTF32(T, | 565 &String::ZoneHandle(Z, Symbols::FromUTF32(T, string_chars.data(), |
| 572 string_chars.data(), string_chars.length())); | 566 string_chars.length())); |
| 573 } | 567 } |
| 574 EndStringLiteral(); | 568 EndStringLiteral(); |
| 575 return; | 569 return; |
| 576 } else { | 570 } else { |
| 577 string_chars.Add(string_delimiter_); | 571 string_chars.Add(string_delimiter_); |
| 578 } | 572 } |
| 579 } else { | 573 } else { |
| 580 // Test for a two part utf16 sequence, and decode to a code point | 574 // Test for a two part utf16 sequence, and decode to a code point |
| 581 // if we find one. | 575 // if we find one. |
| 582 int32_t ch1 = c0_; | 576 int32_t ch1 = c0_; |
| (...skipping 287 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 870 default: | 864 default: |
| 871 if (IsIdentStartChar(c0_)) { | 865 if (IsIdentStartChar(c0_)) { |
| 872 ScanIdent(); | 866 ScanIdent(); |
| 873 } else if (IsDecimalDigit(c0_)) { | 867 } else if (IsDecimalDigit(c0_)) { |
| 874 ScanNumber(false); | 868 ScanNumber(false); |
| 875 } else { | 869 } else { |
| 876 char msg[128]; | 870 char msg[128]; |
| 877 char utf8_char[5]; | 871 char utf8_char[5]; |
| 878 int len = Utf8::Encode(c0_, utf8_char); | 872 int len = Utf8::Encode(c0_, utf8_char); |
| 879 utf8_char[len] = '\0'; | 873 utf8_char[len] = '\0'; |
| 880 OS::SNPrint(msg, sizeof(msg), | 874 OS::SNPrint(msg, sizeof(msg), "unexpected character: '%s' (U+%04X)\n", |
| 881 "unexpected character: '%s' (U+%04X)\n", utf8_char, c0_); | 875 utf8_char, c0_); |
| 882 ErrorMsg(msg); | 876 ErrorMsg(msg); |
| 883 ReadChar(); | 877 ReadChar(); |
| 884 } | 878 } |
| 885 } | 879 } |
| 886 } while (current_token_.kind == Token::kWHITESP); | 880 } while (current_token_.kind == Token::kWHITESP); |
| 887 } | 881 } |
| 888 | 882 |
| 889 | 883 |
| 890 void Scanner::ScanAll(TokenCollector* collector) { | 884 void Scanner::ScanAll(TokenCollector* collector) { |
| 891 Reset(); | 885 Reset(); |
| 892 do { | 886 do { |
| 893 Scan(); | 887 Scan(); |
| 894 bool inserted_new_lines = false; | 888 bool inserted_new_lines = false; |
| 895 for (intptr_t diff = current_token_.position.line - prev_token_line_; | 889 for (intptr_t diff = current_token_.position.line - prev_token_line_; |
| 896 diff > 0; | 890 diff > 0; diff--) { |
| 897 diff--) { | |
| 898 newline_token_.position.line = current_token_.position.line - diff; | 891 newline_token_.position.line = current_token_.position.line - diff; |
| 899 collector->AddToken(newline_token_); | 892 collector->AddToken(newline_token_); |
| 900 inserted_new_lines = true; | 893 inserted_new_lines = true; |
| 901 } | 894 } |
| 902 if (inserted_new_lines && | 895 if (inserted_new_lines && |
| 903 ((current_token_.kind == Token::kINTERPOL_VAR) || | 896 ((current_token_.kind == Token::kINTERPOL_VAR) || |
| 904 (current_token_.kind == Token::kINTERPOL_START))) { | 897 (current_token_.kind == Token::kINTERPOL_START))) { |
| 905 // NOTE: If this changes, be sure to update | 898 // NOTE: If this changes, be sure to update |
| 906 // Script::GenerateLineNumberArray to stay in sync. | 899 // Script::GenerateLineNumberArray to stay in sync. |
| 907 empty_string_token_.position.line = current_token_.position.line; | 900 empty_string_token_.position.line = current_token_.position.line; |
| 908 collector->AddToken(empty_string_token_); | 901 collector->AddToken(empty_string_token_); |
| 909 } | 902 } |
| 910 collector->AddToken(current_token_); | 903 collector->AddToken(current_token_); |
| 911 prev_token_line_ = current_token_.position.line; | 904 prev_token_line_ = current_token_.position.line; |
| 912 } while (current_token_.kind != Token::kEOS); | 905 } while (current_token_.kind != Token::kEOS); |
| 913 } | 906 } |
| 914 | 907 |
| 915 | 908 |
| 916 void Scanner::ScanTo(intptr_t token_index) { | 909 void Scanner::ScanTo(intptr_t token_index) { |
| 917 ASSERT(token_index >= 0); | 910 ASSERT(token_index >= 0); |
| 918 intptr_t index = 0; | 911 intptr_t index = 0; |
| 919 Reset(); | 912 Reset(); |
| 920 do { | 913 do { |
| 921 Scan(); | 914 Scan(); |
| 922 bool inserted_new_lines = false; | 915 bool inserted_new_lines = false; |
| 923 for (intptr_t diff = current_token_.position.line - prev_token_line_; | 916 for (intptr_t diff = current_token_.position.line - prev_token_line_; |
| 924 diff > 0; | 917 diff > 0; diff--) { |
| 925 diff--) { | |
| 926 // Advance the index to account for tokens added in ScanAll. | 918 // Advance the index to account for tokens added in ScanAll. |
| 927 index++; | 919 index++; |
| 928 inserted_new_lines = true; | 920 inserted_new_lines = true; |
| 929 } | 921 } |
| 930 if (inserted_new_lines && | 922 if (inserted_new_lines && |
| 931 ((current_token_.kind == Token::kINTERPOL_VAR) || | 923 ((current_token_.kind == Token::kINTERPOL_VAR) || |
| 932 (current_token_.kind == Token::kINTERPOL_START))) { | 924 (current_token_.kind == Token::kINTERPOL_START))) { |
| 933 // Advance the index to account for tokens added in ScanAll. | 925 // Advance the index to account for tokens added in ScanAll. |
| 934 index++; | 926 index++; |
| 935 } | 927 } |
| 936 index++; | 928 index++; |
| 937 prev_token_line_ = current_token_.position.line; | 929 prev_token_line_ = current_token_.position.line; |
| 938 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); | 930 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); |
| 939 } | 931 } |
| 940 | 932 |
| 941 | 933 |
| 942 void Scanner::InitOnce() { | 934 void Scanner::InitOnce() { |
| 943 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 935 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
| 944 for (int i = 0; i < kNumLowercaseChars; i++) { | 936 for (int i = 0; i < kNumLowercaseChars; i++) { |
| 945 keywords_char_offset_[i] = Token::kNumKeywords; | 937 keywords_char_offset_[i] = Token::kNumKeywords; |
| 946 } | 938 } |
| 947 for (int i = 0; i < Token::kNumKeywords; i++) { | 939 for (int i = 0; i < Token::kNumKeywords; i++) { |
| 948 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 940 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
| 949 keywords_[i].kind = token; | 941 keywords_[i].kind = token; |
| 950 keywords_[i].keyword_chars = Token::Str(token); | 942 keywords_[i].keyword_chars = Token::Str(token); |
| 951 keywords_[i].keyword_len = strlen(Token::Str(token)); | 943 keywords_[i].keyword_len = strlen(Token::Str(token)); |
| 952 keywords_[i].keyword_symbol = &Symbols::Token(token); | 944 keywords_[i].keyword_symbol = &Symbols::Token(token); |
| 953 | 945 |
| 954 int ch = keywords_[i].keyword_chars[0] - 'a'; | 946 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 955 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 947 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 956 keywords_char_offset_[ch] = i; | 948 keywords_char_offset_[ch] = i; |
| 957 } | 949 } |
| 958 } | 950 } |
| 959 } | 951 } |
| 960 | 952 |
| 961 } // namespace dart | 953 } // namespace dart |
| OLD | NEW |