| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
| 6 | 6 |
| 7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
| 8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/object.h" | 10 #include "vm/object.h" |
| 11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
| 12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
| 13 #include "vm/token.h" | 13 #include "vm/token.h" |
| 14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
| 15 | 15 |
| 16 namespace dart { | 16 namespace dart { |
| 17 | 17 |
| 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
| 19 | 19 |
| 20 | 20 |
| 21 // Quick access to the locally defined isolate() method. | 21 // Quick access to the locally defined zone() method. |
| 22 #define I (isolate()) | 22 #define Z (zone()) |
| 23 | 23 |
| 24 | 24 |
| 25 class ScanContext : public ZoneAllocated { | 25 class ScanContext : public ZoneAllocated { |
| 26 public: | 26 public: |
| 27 explicit ScanContext(Scanner* scanner) | 27 explicit ScanContext(Scanner* scanner) |
| 28 : next_(scanner->saved_context_), | 28 : next_(scanner->saved_context_), |
| 29 string_delimiter_(scanner->string_delimiter_), | 29 string_delimiter_(scanner->string_delimiter_), |
| 30 string_is_multiline_(scanner->string_is_multiline_), | 30 string_is_multiline_(scanner->string_is_multiline_), |
| 31 brace_level_(scanner->brace_level_) {} | 31 brace_level_(scanner->brace_level_) {} |
| 32 | 32 |
| (...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 76 ReadChar(); | 76 ReadChar(); |
| 77 } | 77 } |
| 78 | 78 |
| 79 | 79 |
| 80 Scanner::Scanner(const String& src, const String& private_key) | 80 Scanner::Scanner(const String& src, const String& private_key) |
| 81 : source_(src), | 81 : source_(src), |
| 82 source_length_(src.Length()), | 82 source_length_(src.Length()), |
| 83 saved_context_(NULL), | 83 saved_context_(NULL), |
| 84 private_key_(String::ZoneHandle(private_key.raw())), | 84 private_key_(String::ZoneHandle(private_key.raw())), |
| 85 char_at_func_(src.CharAtFunc()), | 85 char_at_func_(src.CharAtFunc()), |
| 86 isolate_(Isolate::Current()) { | 86 zone_(Thread::Current()->zone()) { |
| 87 Reset(); | 87 Reset(); |
| 88 } | 88 } |
| 89 | 89 |
| 90 | 90 |
| 91 Scanner::~Scanner() {} | 91 Scanner::~Scanner() {} |
| 92 | 92 |
| 93 | 93 |
| 94 void Scanner::ErrorMsg(const char* msg) { | 94 void Scanner::ErrorMsg(const char* msg) { |
| 95 current_token_.kind = Token::kERROR; | 95 current_token_.kind = Token::kERROR; |
| 96 current_token_.literal = &String::ZoneHandle(I, Symbols::New(msg)); | 96 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(msg)); |
| 97 current_token_.position = c0_pos_; | 97 current_token_.position = c0_pos_; |
| 98 token_start_ = lookahead_pos_; | 98 token_start_ = lookahead_pos_; |
| 99 current_token_.offset = lookahead_pos_; | 99 current_token_.offset = lookahead_pos_; |
| 100 } | 100 } |
| 101 | 101 |
| 102 | 102 |
| 103 void Scanner::PushContext() { | 103 void Scanner::PushContext() { |
| 104 ScanContext* ctx = new(I) ScanContext(this); | 104 ScanContext* ctx = new(Z) ScanContext(this); |
| 105 saved_context_ = ctx; | 105 saved_context_ = ctx; |
| 106 string_delimiter_ = '\0'; | 106 string_delimiter_ = '\0'; |
| 107 string_is_multiline_ = false; | 107 string_is_multiline_ = false; |
| 108 brace_level_ = 1; // Account for the opening ${ token. | 108 brace_level_ = 1; // Account for the opening ${ token. |
| 109 } | 109 } |
| 110 | 110 |
| 111 | 111 |
| 112 void Scanner::PopContext() { | 112 void Scanner::PopContext() { |
| 113 ASSERT(saved_context_ != NULL); | 113 ASSERT(saved_context_ != NULL); |
| 114 ASSERT(brace_level_ == 0); | 114 ASSERT(brace_level_ == 0); |
| (...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 321 return; | 321 return; |
| 322 } | 322 } |
| 323 } | 323 } |
| 324 i++; | 324 i++; |
| 325 } | 325 } |
| 326 } | 326 } |
| 327 | 327 |
| 328 // We did not read a keyword. | 328 // We did not read a keyword. |
| 329 current_token_.kind = Token::kIDENT; | 329 current_token_.kind = Token::kIDENT; |
| 330 String& literal = | 330 String& literal = |
| 331 String::ZoneHandle(I, Symbols::New(source_, ident_pos, ident_length)); | 331 String::ZoneHandle(Z, Symbols::New(source_, ident_pos, ident_length)); |
| 332 if (ident_char0 == Library::kPrivateIdentifierStart) { | 332 if (ident_char0 == Library::kPrivateIdentifierStart) { |
| 333 // Private identifiers are mangled on a per library basis. | 333 // Private identifiers are mangled on a per library basis. |
| 334 literal = String::Concat(literal, private_key_); | 334 literal = String::Concat(literal, private_key_); |
| 335 literal = Symbols::New(literal); | 335 literal = Symbols::New(literal); |
| 336 } | 336 } |
| 337 current_token_.literal = &literal; | 337 current_token_.literal = &literal; |
| 338 } | 338 } |
| 339 | 339 |
| 340 | 340 |
| 341 // Parse integer or double number literal of format: | 341 // Parse integer or double number literal of format: |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 379 ErrorMsg("missing exponent digits"); | 379 ErrorMsg("missing exponent digits"); |
| 380 return; | 380 return; |
| 381 } | 381 } |
| 382 while (IsDecimalDigit(c0_)) { | 382 while (IsDecimalDigit(c0_)) { |
| 383 ReadChar(); | 383 ReadChar(); |
| 384 } | 384 } |
| 385 } | 385 } |
| 386 } | 386 } |
| 387 if (current_token_.kind != Token::kILLEGAL) { | 387 if (current_token_.kind != Token::kILLEGAL) { |
| 388 intptr_t len = lookahead_pos_ - token_start_; | 388 intptr_t len = lookahead_pos_ - token_start_; |
| 389 String& str = String::ZoneHandle(I, | 389 String& str = String::ZoneHandle(Z, |
| 390 String::SubString(source_, token_start_, len, Heap::kOld)); | 390 String::SubString(source_, token_start_, len, Heap::kOld)); |
| 391 str = Symbols::New(str); | 391 str = Symbols::New(str); |
| 392 current_token_.literal = &str; | 392 current_token_.literal = &str; |
| 393 } | 393 } |
| 394 } | 394 } |
| 395 | 395 |
| 396 | 396 |
| 397 void Scanner::SkipLine() { | 397 void Scanner::SkipLine() { |
| 398 while (c0_ != '\n' && c0_ != '\0') { | 398 while (c0_ != '\n' && c0_ != '\0') { |
| 399 ReadChar(); | 399 ReadChar(); |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 534 return; | 534 return; |
| 535 } | 535 } |
| 536 escape_char = c0_; | 536 escape_char = c0_; |
| 537 break; | 537 break; |
| 538 } | 538 } |
| 539 string_chars.Add(escape_char); | 539 string_chars.Add(escape_char); |
| 540 } else if (c0_ == '$' && !is_raw) { | 540 } else if (c0_ == '$' && !is_raw) { |
| 541 // Scanned a string piece. | 541 // Scanned a string piece. |
| 542 ASSERT(string_chars.data() != NULL); | 542 ASSERT(string_chars.data() != NULL); |
| 543 // Strings are canonicalized: Allocate a symbol. | 543 // Strings are canonicalized: Allocate a symbol. |
| 544 current_token_.literal = &String::ZoneHandle(I, | 544 current_token_.literal = &String::ZoneHandle(Z, |
| 545 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 545 Symbols::FromUTF32(string_chars.data(), string_chars.length())); |
| 546 // Preserve error tokens. | 546 // Preserve error tokens. |
| 547 if (current_token_.kind != Token::kERROR) { | 547 if (current_token_.kind != Token::kERROR) { |
| 548 current_token_.kind = Token::kSTRING; | 548 current_token_.kind = Token::kSTRING; |
| 549 } | 549 } |
| 550 return; | 550 return; |
| 551 } else if (c0_ == string_delimiter_) { | 551 } else if (c0_ == string_delimiter_) { |
| 552 // Check if we are at the end of the string literal. | 552 // Check if we are at the end of the string literal. |
| 553 if (!string_is_multiline_ || | 553 if (!string_is_multiline_ || |
| 554 ((LookaheadChar(1) == string_delimiter_) && | 554 ((LookaheadChar(1) == string_delimiter_) && |
| 555 (LookaheadChar(2) == string_delimiter_))) { | 555 (LookaheadChar(2) == string_delimiter_))) { |
| 556 if (string_is_multiline_) { | 556 if (string_is_multiline_) { |
| 557 ReadChar(); // Skip two string delimiters. | 557 ReadChar(); // Skip two string delimiters. |
| 558 ReadChar(); | 558 ReadChar(); |
| 559 } | 559 } |
| 560 // Preserve error tokens. | 560 // Preserve error tokens. |
| 561 if (current_token_.kind == Token::kERROR) { | 561 if (current_token_.kind == Token::kERROR) { |
| 562 ReadChar(); | 562 ReadChar(); |
| 563 } else { | 563 } else { |
| 564 Recognize(Token::kSTRING); | 564 Recognize(Token::kSTRING); |
| 565 ASSERT(string_chars.data() != NULL); | 565 ASSERT(string_chars.data() != NULL); |
| 566 // Strings are canonicalized: Allocate a symbol. | 566 // Strings are canonicalized: Allocate a symbol. |
| 567 current_token_.literal = &String::ZoneHandle(I, | 567 current_token_.literal = &String::ZoneHandle(Z, |
| 568 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 568 Symbols::FromUTF32(string_chars.data(), string_chars.length())); |
| 569 } | 569 } |
| 570 EndStringLiteral(); | 570 EndStringLiteral(); |
| 571 return; | 571 return; |
| 572 } else { | 572 } else { |
| 573 string_chars.Add(string_delimiter_); | 573 string_chars.Add(string_delimiter_); |
| 574 } | 574 } |
| 575 } else { | 575 } else { |
| 576 // Test for a two part utf16 sequence, and decode to a code point | 576 // Test for a two part utf16 sequence, and decode to a code point |
| 577 // if we find one. | 577 // if we find one. |
| (...skipping 344 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 922 (current_token_.kind == Token::kINTERPOL_START))) { | 922 (current_token_.kind == Token::kINTERPOL_START))) { |
| 923 index++; // Advance the index to account for tokens added in ScanAll. | 923 index++; // Advance the index to account for tokens added in ScanAll. |
| 924 } | 924 } |
| 925 index++; | 925 index++; |
| 926 prev_token_line_ = current_token_.position.line; | 926 prev_token_line_ = current_token_.position.line; |
| 927 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); | 927 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); |
| 928 } | 928 } |
| 929 | 929 |
| 930 | 930 |
| 931 const Scanner::GrowableTokenStream& Scanner::GetStream() { | 931 const Scanner::GrowableTokenStream& Scanner::GetStream() { |
| 932 GrowableTokenStream* ts = new(I) GrowableTokenStream(128); | 932 GrowableTokenStream* ts = new(Z) GrowableTokenStream(128); |
| 933 ScanAll(ts); | 933 ScanAll(ts); |
| 934 if (FLAG_print_tokens) { | 934 if (FLAG_print_tokens) { |
| 935 Scanner::PrintTokens(*ts); | 935 Scanner::PrintTokens(*ts); |
| 936 } | 936 } |
| 937 return *ts; | 937 return *ts; |
| 938 } | 938 } |
| 939 | 939 |
| 940 | 940 |
| 941 void Scanner::PrintTokens(const GrowableTokenStream& ts) { | 941 void Scanner::PrintTokens(const GrowableTokenStream& ts) { |
| 942 int currentLine = -1; | 942 int currentLine = -1; |
| (...skipping 22 matching lines...) Expand all Loading... |
| 965 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 965 keywords_[i].keyword_symbol = &Symbols::Keyword(token); |
| 966 | 966 |
| 967 int ch = keywords_[i].keyword_chars[0] - 'a'; | 967 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 968 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 968 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 969 keywords_char_offset_[ch] = i; | 969 keywords_char_offset_[ch] = i; |
| 970 } | 970 } |
| 971 } | 971 } |
| 972 } | 972 } |
| 973 | 973 |
| 974 } // namespace dart | 974 } // namespace dart |
| OLD | NEW |