OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
6 | 6 |
7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
10 #include "vm/object.h" | 10 #include "vm/object.h" |
11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
13 #include "vm/token.h" | 13 #include "vm/token.h" |
14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
15 | 15 |
16 namespace dart { | 16 namespace dart { |
17 | 17 |
18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
19 | 19 |
20 | 20 |
21 // Quick access to the locally defined zone() method. | 21 // Quick access to the locally defined zone() and thread() methods. |
22 #define Z (zone()) | 22 #define Z (zone()) |
| 23 #define T (thread()) |
23 | 24 |
24 | 25 |
25 class ScanContext : public ZoneAllocated { | 26 class ScanContext : public ZoneAllocated { |
26 public: | 27 public: |
27 explicit ScanContext(Scanner* scanner) | 28 explicit ScanContext(Scanner* scanner) |
28 : next_(scanner->saved_context_), | 29 : next_(scanner->saved_context_), |
29 string_delimiter_(scanner->string_delimiter_), | 30 string_delimiter_(scanner->string_delimiter_), |
30 string_is_multiline_(scanner->string_is_multiline_), | 31 string_is_multiline_(scanner->string_is_multiline_), |
31 brace_level_(scanner->brace_level_) {} | 32 brace_level_(scanner->brace_level_) {} |
32 | 33 |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
76 ReadChar(); | 77 ReadChar(); |
77 } | 78 } |
78 | 79 |
79 | 80 |
80 Scanner::Scanner(const String& src, const String& private_key) | 81 Scanner::Scanner(const String& src, const String& private_key) |
81 : source_(src), | 82 : source_(src), |
82 source_length_(src.Length()), | 83 source_length_(src.Length()), |
83 saved_context_(NULL), | 84 saved_context_(NULL), |
84 private_key_(String::ZoneHandle(private_key.raw())), | 85 private_key_(String::ZoneHandle(private_key.raw())), |
85 char_at_func_(src.CharAtFunc()), | 86 char_at_func_(src.CharAtFunc()), |
86 zone_(Thread::Current()->zone()) { | 87 thread_(Thread::Current()), |
| 88 zone_(thread_->zone()) { |
87 Reset(); | 89 Reset(); |
88 } | 90 } |
89 | 91 |
90 | 92 |
91 Scanner::~Scanner() {} | 93 Scanner::~Scanner() {} |
92 | 94 |
93 | 95 |
94 void Scanner::ErrorMsg(const char* msg) { | 96 void Scanner::ErrorMsg(const char* msg) { |
95 current_token_.kind = Token::kERROR; | 97 current_token_.kind = Token::kERROR; |
96 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(msg)); | 98 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(T, msg)); |
97 current_token_.position = c0_pos_; | 99 current_token_.position = c0_pos_; |
98 token_start_ = lookahead_pos_; | 100 token_start_ = lookahead_pos_; |
99 current_token_.offset = lookahead_pos_; | 101 current_token_.offset = lookahead_pos_; |
100 } | 102 } |
101 | 103 |
102 | 104 |
103 void Scanner::PushContext() { | 105 void Scanner::PushContext() { |
104 ScanContext* ctx = new(Z) ScanContext(this); | 106 ScanContext* ctx = new(Z) ScanContext(this); |
105 saved_context_ = ctx; | 107 saved_context_ = ctx; |
106 string_delimiter_ = '\0'; | 108 string_delimiter_ = '\0'; |
(...skipping 214 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
321 return; | 323 return; |
322 } | 324 } |
323 } | 325 } |
324 i++; | 326 i++; |
325 } | 327 } |
326 } | 328 } |
327 | 329 |
328 // We did not read a keyword. | 330 // We did not read a keyword. |
329 current_token_.kind = Token::kIDENT; | 331 current_token_.kind = Token::kIDENT; |
330 String& literal = | 332 String& literal = |
331 String::ZoneHandle(Z, Symbols::New(source_, ident_pos, ident_length)); | 333 String::ZoneHandle(Z, Symbols::New(T, source_, ident_pos, ident_length)); |
332 if (ident_char0 == Library::kPrivateIdentifierStart) { | 334 if (ident_char0 == Library::kPrivateIdentifierStart) { |
333 // Private identifiers are mangled on a per library basis. | 335 // Private identifiers are mangled on a per library basis. |
334 literal = Symbols::FromConcat(literal, private_key_); | 336 literal = Symbols::FromConcat(T, literal, private_key_); |
335 } | 337 } |
336 current_token_.literal = &literal; | 338 current_token_.literal = &literal; |
337 } | 339 } |
338 | 340 |
339 | 341 |
340 // Parse integer or double number literal of format: | 342 // Parse integer or double number literal of format: |
341 // NUMBER = INTEGER | DOUBLE | 343 // NUMBER = INTEGER | DOUBLE |
342 // INTEGER = D+ | (("0x" | "0X") H+) | 344 // INTEGER = D+ | (("0x" | "0X") H+) |
343 // DOUBLE = ((D+ ["." D*]) | ("." D+)) [ EXPONENT ] | 345 // DOUBLE = ((D+ ["." D*]) | ("." D+)) [ EXPONENT ] |
344 // EXPONENT = ("e" | "E") ["+" | "-"] D+ | 346 // EXPONENT = ("e" | "E") ["+" | "-"] D+ |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
379 return; | 381 return; |
380 } | 382 } |
381 while (IsDecimalDigit(c0_)) { | 383 while (IsDecimalDigit(c0_)) { |
382 ReadChar(); | 384 ReadChar(); |
383 } | 385 } |
384 } | 386 } |
385 } | 387 } |
386 if (current_token_.kind != Token::kILLEGAL) { | 388 if (current_token_.kind != Token::kILLEGAL) { |
387 intptr_t len = lookahead_pos_ - token_start_; | 389 intptr_t len = lookahead_pos_ - token_start_; |
388 const String& str = | 390 const String& str = |
389 String::ZoneHandle(Z, Symbols::New(source_, token_start_, len)); | 391 String::ZoneHandle(Z, Symbols::New(T, source_, token_start_, len)); |
390 current_token_.literal = &str; | 392 current_token_.literal = &str; |
391 } | 393 } |
392 } | 394 } |
393 | 395 |
394 | 396 |
395 void Scanner::SkipLine() { | 397 void Scanner::SkipLine() { |
396 while (c0_ != '\n' && c0_ != '\0') { | 398 while (c0_ != '\n' && c0_ != '\0') { |
397 ReadChar(); | 399 ReadChar(); |
398 } | 400 } |
399 } | 401 } |
(...skipping 133 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
533 } | 535 } |
534 escape_char = c0_; | 536 escape_char = c0_; |
535 break; | 537 break; |
536 } | 538 } |
537 string_chars.Add(escape_char); | 539 string_chars.Add(escape_char); |
538 } else if (c0_ == '$' && !is_raw) { | 540 } else if (c0_ == '$' && !is_raw) { |
539 // Scanned a string piece. | 541 // Scanned a string piece. |
540 ASSERT(string_chars.data() != NULL); | 542 ASSERT(string_chars.data() != NULL); |
541 // Strings are canonicalized: Allocate a symbol. | 543 // Strings are canonicalized: Allocate a symbol. |
542 current_token_.literal = &String::ZoneHandle(Z, | 544 current_token_.literal = &String::ZoneHandle(Z, |
543 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 545 Symbols::FromUTF32(T, string_chars.data(), string_chars.length())); |
544 // Preserve error tokens. | 546 // Preserve error tokens. |
545 if (current_token_.kind != Token::kERROR) { | 547 if (current_token_.kind != Token::kERROR) { |
546 current_token_.kind = Token::kSTRING; | 548 current_token_.kind = Token::kSTRING; |
547 } | 549 } |
548 return; | 550 return; |
549 } else if (c0_ == string_delimiter_) { | 551 } else if (c0_ == string_delimiter_) { |
550 // Check if we are at the end of the string literal. | 552 // Check if we are at the end of the string literal. |
551 if (!string_is_multiline_ || | 553 if (!string_is_multiline_ || |
552 ((LookaheadChar(1) == string_delimiter_) && | 554 ((LookaheadChar(1) == string_delimiter_) && |
553 (LookaheadChar(2) == string_delimiter_))) { | 555 (LookaheadChar(2) == string_delimiter_))) { |
554 if (string_is_multiline_) { | 556 if (string_is_multiline_) { |
555 ReadChar(); // Skip two string delimiters. | 557 ReadChar(); // Skip two string delimiters. |
556 ReadChar(); | 558 ReadChar(); |
557 } | 559 } |
558 // Preserve error tokens. | 560 // Preserve error tokens. |
559 if (current_token_.kind == Token::kERROR) { | 561 if (current_token_.kind == Token::kERROR) { |
560 ReadChar(); | 562 ReadChar(); |
561 } else { | 563 } else { |
562 Recognize(Token::kSTRING); | 564 Recognize(Token::kSTRING); |
563 ASSERT(string_chars.data() != NULL); | 565 ASSERT(string_chars.data() != NULL); |
564 // Strings are canonicalized: Allocate a symbol. | 566 // Strings are canonicalized: Allocate a symbol. |
565 current_token_.literal = &String::ZoneHandle(Z, | 567 current_token_.literal = &String::ZoneHandle(Z, |
566 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 568 Symbols::FromUTF32(T, |
| 569 string_chars.data(), string_chars.length())); |
567 } | 570 } |
568 EndStringLiteral(); | 571 EndStringLiteral(); |
569 return; | 572 return; |
570 } else { | 573 } else { |
571 string_chars.Add(string_delimiter_); | 574 string_chars.Add(string_delimiter_); |
572 } | 575 } |
573 } else { | 576 } else { |
574 // Test for a two part utf16 sequence, and decode to a code point | 577 // Test for a two part utf16 sequence, and decode to a code point |
575 // if we find one. | 578 // if we find one. |
576 int32_t ch1 = c0_; | 579 int32_t ch1 = c0_; |
(...skipping 386 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
963 void Scanner::InitOnce() { | 966 void Scanner::InitOnce() { |
964 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 967 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
965 for (int i = 0; i < kNumLowercaseChars; i++) { | 968 for (int i = 0; i < kNumLowercaseChars; i++) { |
966 keywords_char_offset_[i] = Token::kNumKeywords; | 969 keywords_char_offset_[i] = Token::kNumKeywords; |
967 } | 970 } |
968 for (int i = 0; i < Token::kNumKeywords; i++) { | 971 for (int i = 0; i < Token::kNumKeywords; i++) { |
969 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 972 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
970 keywords_[i].kind = token; | 973 keywords_[i].kind = token; |
971 keywords_[i].keyword_chars = Token::Str(token); | 974 keywords_[i].keyword_chars = Token::Str(token); |
972 keywords_[i].keyword_len = strlen(Token::Str(token)); | 975 keywords_[i].keyword_len = strlen(Token::Str(token)); |
973 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 976 keywords_[i].keyword_symbol = &Symbols::Token(token); |
974 | 977 |
975 int ch = keywords_[i].keyword_chars[0] - 'a'; | 978 int ch = keywords_[i].keyword_chars[0] - 'a'; |
976 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 979 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
977 keywords_char_offset_[ch] = i; | 980 keywords_char_offset_[ch] = i; |
978 } | 981 } |
979 } | 982 } |
980 } | 983 } |
981 | 984 |
982 } // namespace dart | 985 } // namespace dart |
OLD | NEW |