| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
| 6 | 6 |
| 7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
| 8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/object.h" | 10 #include "vm/object.h" |
| 11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
| 12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
| 13 #include "vm/token.h" | 13 #include "vm/token.h" |
| 14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
| 15 | 15 |
| 16 namespace dart { | 16 namespace dart { |
| 17 | 17 |
| 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
| 19 | 19 |
| 20 | 20 |
| 21 // Quick access to the locally defined zone() method. | 21 // Quick access to the locally defined zone() and thread() methods. |
| 22 #define Z (zone()) | 22 #define Z (zone()) |
| 23 #define T (thread()) |
| 23 | 24 |
| 24 | 25 |
| 25 class ScanContext : public ZoneAllocated { | 26 class ScanContext : public ZoneAllocated { |
| 26 public: | 27 public: |
| 27 explicit ScanContext(Scanner* scanner) | 28 explicit ScanContext(Scanner* scanner) |
| 28 : next_(scanner->saved_context_), | 29 : next_(scanner->saved_context_), |
| 29 string_delimiter_(scanner->string_delimiter_), | 30 string_delimiter_(scanner->string_delimiter_), |
| 30 string_is_multiline_(scanner->string_is_multiline_), | 31 string_is_multiline_(scanner->string_is_multiline_), |
| 31 brace_level_(scanner->brace_level_) {} | 32 brace_level_(scanner->brace_level_) {} |
| 32 | 33 |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 76 ReadChar(); | 77 ReadChar(); |
| 77 } | 78 } |
| 78 | 79 |
| 79 | 80 |
| 80 Scanner::Scanner(const String& src, const String& private_key) | 81 Scanner::Scanner(const String& src, const String& private_key) |
| 81 : source_(src), | 82 : source_(src), |
| 82 source_length_(src.Length()), | 83 source_length_(src.Length()), |
| 83 saved_context_(NULL), | 84 saved_context_(NULL), |
| 84 private_key_(String::ZoneHandle(private_key.raw())), | 85 private_key_(String::ZoneHandle(private_key.raw())), |
| 85 char_at_func_(src.CharAtFunc()), | 86 char_at_func_(src.CharAtFunc()), |
| 86 zone_(Thread::Current()->zone()) { | 87 thread_(Thread::Current()), |
| 88 zone_(thread_->zone()) { |
| 87 Reset(); | 89 Reset(); |
| 88 } | 90 } |
| 89 | 91 |
| 90 | 92 |
| 91 Scanner::~Scanner() {} | 93 Scanner::~Scanner() {} |
| 92 | 94 |
| 93 | 95 |
| 94 void Scanner::ErrorMsg(const char* msg) { | 96 void Scanner::ErrorMsg(const char* msg) { |
| 95 current_token_.kind = Token::kERROR; | 97 current_token_.kind = Token::kERROR; |
| 96 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(msg)); | 98 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(T, msg)); |
| 97 current_token_.position = c0_pos_; | 99 current_token_.position = c0_pos_; |
| 98 token_start_ = lookahead_pos_; | 100 token_start_ = lookahead_pos_; |
| 99 current_token_.offset = lookahead_pos_; | 101 current_token_.offset = lookahead_pos_; |
| 100 } | 102 } |
| 101 | 103 |
| 102 | 104 |
| 103 void Scanner::PushContext() { | 105 void Scanner::PushContext() { |
| 104 ScanContext* ctx = new(Z) ScanContext(this); | 106 ScanContext* ctx = new(Z) ScanContext(this); |
| 105 saved_context_ = ctx; | 107 saved_context_ = ctx; |
| 106 string_delimiter_ = '\0'; | 108 string_delimiter_ = '\0'; |
| (...skipping 214 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 321 return; | 323 return; |
| 322 } | 324 } |
| 323 } | 325 } |
| 324 i++; | 326 i++; |
| 325 } | 327 } |
| 326 } | 328 } |
| 327 | 329 |
| 328 // We did not read a keyword. | 330 // We did not read a keyword. |
| 329 current_token_.kind = Token::kIDENT; | 331 current_token_.kind = Token::kIDENT; |
| 330 String& literal = | 332 String& literal = |
| 331 String::ZoneHandle(Z, Symbols::New(source_, ident_pos, ident_length)); | 333 String::ZoneHandle(Z, Symbols::New(T, source_, ident_pos, ident_length)); |
| 332 if (ident_char0 == Library::kPrivateIdentifierStart) { | 334 if (ident_char0 == Library::kPrivateIdentifierStart) { |
| 333 // Private identifiers are mangled on a per library basis. | 335 // Private identifiers are mangled on a per library basis. |
| 334 literal = Symbols::FromConcat(literal, private_key_); | 336 literal = Symbols::FromConcat(T, literal, private_key_); |
| 335 } | 337 } |
| 336 current_token_.literal = &literal; | 338 current_token_.literal = &literal; |
| 337 } | 339 } |
| 338 | 340 |
| 339 | 341 |
| 340 // Parse integer or double number literal of format: | 342 // Parse integer or double number literal of format: |
| 341 // NUMBER = INTEGER | DOUBLE | 343 // NUMBER = INTEGER | DOUBLE |
| 342 // INTEGER = D+ | (("0x" | "0X") H+) | 344 // INTEGER = D+ | (("0x" | "0X") H+) |
| 343 // DOUBLE = ((D+ ["." D*]) | ("." D+)) [ EXPONENT ] | 345 // DOUBLE = ((D+ ["." D*]) | ("." D+)) [ EXPONENT ] |
| 344 // EXPONENT = ("e" | "E") ["+" | "-"] D+ | 346 // EXPONENT = ("e" | "E") ["+" | "-"] D+ |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 379 return; | 381 return; |
| 380 } | 382 } |
| 381 while (IsDecimalDigit(c0_)) { | 383 while (IsDecimalDigit(c0_)) { |
| 382 ReadChar(); | 384 ReadChar(); |
| 383 } | 385 } |
| 384 } | 386 } |
| 385 } | 387 } |
| 386 if (current_token_.kind != Token::kILLEGAL) { | 388 if (current_token_.kind != Token::kILLEGAL) { |
| 387 intptr_t len = lookahead_pos_ - token_start_; | 389 intptr_t len = lookahead_pos_ - token_start_; |
| 388 const String& str = | 390 const String& str = |
| 389 String::ZoneHandle(Z, Symbols::New(source_, token_start_, len)); | 391 String::ZoneHandle(Z, Symbols::New(T, source_, token_start_, len)); |
| 390 current_token_.literal = &str; | 392 current_token_.literal = &str; |
| 391 } | 393 } |
| 392 } | 394 } |
| 393 | 395 |
| 394 | 396 |
| 395 void Scanner::SkipLine() { | 397 void Scanner::SkipLine() { |
| 396 while (c0_ != '\n' && c0_ != '\0') { | 398 while (c0_ != '\n' && c0_ != '\0') { |
| 397 ReadChar(); | 399 ReadChar(); |
| 398 } | 400 } |
| 399 } | 401 } |
| (...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 533 } | 535 } |
| 534 escape_char = c0_; | 536 escape_char = c0_; |
| 535 break; | 537 break; |
| 536 } | 538 } |
| 537 string_chars.Add(escape_char); | 539 string_chars.Add(escape_char); |
| 538 } else if (c0_ == '$' && !is_raw) { | 540 } else if (c0_ == '$' && !is_raw) { |
| 539 // Scanned a string piece. | 541 // Scanned a string piece. |
| 540 ASSERT(string_chars.data() != NULL); | 542 ASSERT(string_chars.data() != NULL); |
| 541 // Strings are canonicalized: Allocate a symbol. | 543 // Strings are canonicalized: Allocate a symbol. |
| 542 current_token_.literal = &String::ZoneHandle(Z, | 544 current_token_.literal = &String::ZoneHandle(Z, |
| 543 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 545 Symbols::FromUTF32(T, string_chars.data(), string_chars.length())); |
| 544 // Preserve error tokens. | 546 // Preserve error tokens. |
| 545 if (current_token_.kind != Token::kERROR) { | 547 if (current_token_.kind != Token::kERROR) { |
| 546 current_token_.kind = Token::kSTRING; | 548 current_token_.kind = Token::kSTRING; |
| 547 } | 549 } |
| 548 return; | 550 return; |
| 549 } else if (c0_ == string_delimiter_) { | 551 } else if (c0_ == string_delimiter_) { |
| 550 // Check if we are at the end of the string literal. | 552 // Check if we are at the end of the string literal. |
| 551 if (!string_is_multiline_ || | 553 if (!string_is_multiline_ || |
| 552 ((LookaheadChar(1) == string_delimiter_) && | 554 ((LookaheadChar(1) == string_delimiter_) && |
| 553 (LookaheadChar(2) == string_delimiter_))) { | 555 (LookaheadChar(2) == string_delimiter_))) { |
| 554 if (string_is_multiline_) { | 556 if (string_is_multiline_) { |
| 555 ReadChar(); // Skip two string delimiters. | 557 ReadChar(); // Skip two string delimiters. |
| 556 ReadChar(); | 558 ReadChar(); |
| 557 } | 559 } |
| 558 // Preserve error tokens. | 560 // Preserve error tokens. |
| 559 if (current_token_.kind == Token::kERROR) { | 561 if (current_token_.kind == Token::kERROR) { |
| 560 ReadChar(); | 562 ReadChar(); |
| 561 } else { | 563 } else { |
| 562 Recognize(Token::kSTRING); | 564 Recognize(Token::kSTRING); |
| 563 ASSERT(string_chars.data() != NULL); | 565 ASSERT(string_chars.data() != NULL); |
| 564 // Strings are canonicalized: Allocate a symbol. | 566 // Strings are canonicalized: Allocate a symbol. |
| 565 current_token_.literal = &String::ZoneHandle(Z, | 567 current_token_.literal = &String::ZoneHandle(Z, |
| 566 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 568 Symbols::FromUTF32(T, |
| 569 string_chars.data(), string_chars.length())); |
| 567 } | 570 } |
| 568 EndStringLiteral(); | 571 EndStringLiteral(); |
| 569 return; | 572 return; |
| 570 } else { | 573 } else { |
| 571 string_chars.Add(string_delimiter_); | 574 string_chars.Add(string_delimiter_); |
| 572 } | 575 } |
| 573 } else { | 576 } else { |
| 574 // Test for a two part utf16 sequence, and decode to a code point | 577 // Test for a two part utf16 sequence, and decode to a code point |
| 575 // if we find one. | 578 // if we find one. |
| 576 int32_t ch1 = c0_; | 579 int32_t ch1 = c0_; |
| (...skipping 386 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 963 void Scanner::InitOnce() { | 966 void Scanner::InitOnce() { |
| 964 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 967 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
| 965 for (int i = 0; i < kNumLowercaseChars; i++) { | 968 for (int i = 0; i < kNumLowercaseChars; i++) { |
| 966 keywords_char_offset_[i] = Token::kNumKeywords; | 969 keywords_char_offset_[i] = Token::kNumKeywords; |
| 967 } | 970 } |
| 968 for (int i = 0; i < Token::kNumKeywords; i++) { | 971 for (int i = 0; i < Token::kNumKeywords; i++) { |
| 969 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 972 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
| 970 keywords_[i].kind = token; | 973 keywords_[i].kind = token; |
| 971 keywords_[i].keyword_chars = Token::Str(token); | 974 keywords_[i].keyword_chars = Token::Str(token); |
| 972 keywords_[i].keyword_len = strlen(Token::Str(token)); | 975 keywords_[i].keyword_len = strlen(Token::Str(token)); |
| 973 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 976 keywords_[i].keyword_symbol = &Symbols::Token(token); |
| 974 | 977 |
| 975 int ch = keywords_[i].keyword_chars[0] - 'a'; | 978 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 976 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 979 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 977 keywords_char_offset_[ch] = i; | 980 keywords_char_offset_[ch] = i; |
| 978 } | 981 } |
| 979 } | 982 } |
| 980 } | 983 } |
| 981 | 984 |
| 982 } // namespace dart | 985 } // namespace dart |
| OLD | NEW |