OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
6 | 6 |
7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
10 #include "vm/object.h" | 10 #include "vm/object.h" |
11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
13 #include "vm/token.h" | 13 #include "vm/token.h" |
14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
15 | 15 |
16 namespace dart { | 16 namespace dart { |
17 | 17 |
18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | 18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); |
19 | 19 |
20 | 20 |
21 // Quick access to the locally defined isolate() method. | 21 // Quick access to the locally defined zone() method. |
22 #define I (isolate()) | 22 #define Z (zone()) |
23 | 23 |
24 | 24 |
25 class ScanContext : public ZoneAllocated { | 25 class ScanContext : public ZoneAllocated { |
26 public: | 26 public: |
27 explicit ScanContext(Scanner* scanner) | 27 explicit ScanContext(Scanner* scanner) |
28 : next_(scanner->saved_context_), | 28 : next_(scanner->saved_context_), |
29 string_delimiter_(scanner->string_delimiter_), | 29 string_delimiter_(scanner->string_delimiter_), |
30 string_is_multiline_(scanner->string_is_multiline_), | 30 string_is_multiline_(scanner->string_is_multiline_), |
31 brace_level_(scanner->brace_level_) {} | 31 brace_level_(scanner->brace_level_) {} |
32 | 32 |
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
76 ReadChar(); | 76 ReadChar(); |
77 } | 77 } |
78 | 78 |
79 | 79 |
80 Scanner::Scanner(const String& src, const String& private_key) | 80 Scanner::Scanner(const String& src, const String& private_key) |
81 : source_(src), | 81 : source_(src), |
82 source_length_(src.Length()), | 82 source_length_(src.Length()), |
83 saved_context_(NULL), | 83 saved_context_(NULL), |
84 private_key_(String::ZoneHandle(private_key.raw())), | 84 private_key_(String::ZoneHandle(private_key.raw())), |
85 char_at_func_(src.CharAtFunc()), | 85 char_at_func_(src.CharAtFunc()), |
86 isolate_(Isolate::Current()) { | 86 zone_(Thread::Current()->zone()) { |
87 Reset(); | 87 Reset(); |
88 } | 88 } |
89 | 89 |
90 | 90 |
91 Scanner::~Scanner() {} | 91 Scanner::~Scanner() {} |
92 | 92 |
93 | 93 |
94 void Scanner::ErrorMsg(const char* msg) { | 94 void Scanner::ErrorMsg(const char* msg) { |
95 current_token_.kind = Token::kERROR; | 95 current_token_.kind = Token::kERROR; |
96 current_token_.literal = &String::ZoneHandle(I, Symbols::New(msg)); | 96 current_token_.literal = &String::ZoneHandle(Z, Symbols::New(msg)); |
97 current_token_.position = c0_pos_; | 97 current_token_.position = c0_pos_; |
98 token_start_ = lookahead_pos_; | 98 token_start_ = lookahead_pos_; |
99 current_token_.offset = lookahead_pos_; | 99 current_token_.offset = lookahead_pos_; |
100 } | 100 } |
101 | 101 |
102 | 102 |
103 void Scanner::PushContext() { | 103 void Scanner::PushContext() { |
104 ScanContext* ctx = new(I) ScanContext(this); | 104 ScanContext* ctx = new(Z) ScanContext(this); |
105 saved_context_ = ctx; | 105 saved_context_ = ctx; |
106 string_delimiter_ = '\0'; | 106 string_delimiter_ = '\0'; |
107 string_is_multiline_ = false; | 107 string_is_multiline_ = false; |
108 brace_level_ = 1; // Account for the opening ${ token. | 108 brace_level_ = 1; // Account for the opening ${ token. |
109 } | 109 } |
110 | 110 |
111 | 111 |
112 void Scanner::PopContext() { | 112 void Scanner::PopContext() { |
113 ASSERT(saved_context_ != NULL); | 113 ASSERT(saved_context_ != NULL); |
114 ASSERT(brace_level_ == 0); | 114 ASSERT(brace_level_ == 0); |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
321 return; | 321 return; |
322 } | 322 } |
323 } | 323 } |
324 i++; | 324 i++; |
325 } | 325 } |
326 } | 326 } |
327 | 327 |
328 // We did not read a keyword. | 328 // We did not read a keyword. |
329 current_token_.kind = Token::kIDENT; | 329 current_token_.kind = Token::kIDENT; |
330 String& literal = | 330 String& literal = |
331 String::ZoneHandle(I, Symbols::New(source_, ident_pos, ident_length)); | 331 String::ZoneHandle(Z, Symbols::New(source_, ident_pos, ident_length)); |
332 if (ident_char0 == Library::kPrivateIdentifierStart) { | 332 if (ident_char0 == Library::kPrivateIdentifierStart) { |
333 // Private identifiers are mangled on a per library basis. | 333 // Private identifiers are mangled on a per library basis. |
334 literal = String::Concat(literal, private_key_); | 334 literal = String::Concat(literal, private_key_); |
335 literal = Symbols::New(literal); | 335 literal = Symbols::New(literal); |
336 } | 336 } |
337 current_token_.literal = &literal; | 337 current_token_.literal = &literal; |
338 } | 338 } |
339 | 339 |
340 | 340 |
341 // Parse integer or double number literal of format: | 341 // Parse integer or double number literal of format: |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
379 ErrorMsg("missing exponent digits"); | 379 ErrorMsg("missing exponent digits"); |
380 return; | 380 return; |
381 } | 381 } |
382 while (IsDecimalDigit(c0_)) { | 382 while (IsDecimalDigit(c0_)) { |
383 ReadChar(); | 383 ReadChar(); |
384 } | 384 } |
385 } | 385 } |
386 } | 386 } |
387 if (current_token_.kind != Token::kILLEGAL) { | 387 if (current_token_.kind != Token::kILLEGAL) { |
388 intptr_t len = lookahead_pos_ - token_start_; | 388 intptr_t len = lookahead_pos_ - token_start_; |
389 String& str = String::ZoneHandle(I, | 389 String& str = String::ZoneHandle(Z, |
390 String::SubString(source_, token_start_, len, Heap::kOld)); | 390 String::SubString(source_, token_start_, len, Heap::kOld)); |
391 str = Symbols::New(str); | 391 str = Symbols::New(str); |
392 current_token_.literal = &str; | 392 current_token_.literal = &str; |
393 } | 393 } |
394 } | 394 } |
395 | 395 |
396 | 396 |
397 void Scanner::SkipLine() { | 397 void Scanner::SkipLine() { |
398 while (c0_ != '\n' && c0_ != '\0') { | 398 while (c0_ != '\n' && c0_ != '\0') { |
399 ReadChar(); | 399 ReadChar(); |
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
534 return; | 534 return; |
535 } | 535 } |
536 escape_char = c0_; | 536 escape_char = c0_; |
537 break; | 537 break; |
538 } | 538 } |
539 string_chars.Add(escape_char); | 539 string_chars.Add(escape_char); |
540 } else if (c0_ == '$' && !is_raw) { | 540 } else if (c0_ == '$' && !is_raw) { |
541 // Scanned a string piece. | 541 // Scanned a string piece. |
542 ASSERT(string_chars.data() != NULL); | 542 ASSERT(string_chars.data() != NULL); |
543 // Strings are canonicalized: Allocate a symbol. | 543 // Strings are canonicalized: Allocate a symbol. |
544 current_token_.literal = &String::ZoneHandle(I, | 544 current_token_.literal = &String::ZoneHandle(Z, |
545 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 545 Symbols::FromUTF32(string_chars.data(), string_chars.length())); |
546 // Preserve error tokens. | 546 // Preserve error tokens. |
547 if (current_token_.kind != Token::kERROR) { | 547 if (current_token_.kind != Token::kERROR) { |
548 current_token_.kind = Token::kSTRING; | 548 current_token_.kind = Token::kSTRING; |
549 } | 549 } |
550 return; | 550 return; |
551 } else if (c0_ == string_delimiter_) { | 551 } else if (c0_ == string_delimiter_) { |
552 // Check if we are at the end of the string literal. | 552 // Check if we are at the end of the string literal. |
553 if (!string_is_multiline_ || | 553 if (!string_is_multiline_ || |
554 ((LookaheadChar(1) == string_delimiter_) && | 554 ((LookaheadChar(1) == string_delimiter_) && |
555 (LookaheadChar(2) == string_delimiter_))) { | 555 (LookaheadChar(2) == string_delimiter_))) { |
556 if (string_is_multiline_) { | 556 if (string_is_multiline_) { |
557 ReadChar(); // Skip two string delimiters. | 557 ReadChar(); // Skip two string delimiters. |
558 ReadChar(); | 558 ReadChar(); |
559 } | 559 } |
560 // Preserve error tokens. | 560 // Preserve error tokens. |
561 if (current_token_.kind == Token::kERROR) { | 561 if (current_token_.kind == Token::kERROR) { |
562 ReadChar(); | 562 ReadChar(); |
563 } else { | 563 } else { |
564 Recognize(Token::kSTRING); | 564 Recognize(Token::kSTRING); |
565 ASSERT(string_chars.data() != NULL); | 565 ASSERT(string_chars.data() != NULL); |
566 // Strings are canonicalized: Allocate a symbol. | 566 // Strings are canonicalized: Allocate a symbol. |
567 current_token_.literal = &String::ZoneHandle(I, | 567 current_token_.literal = &String::ZoneHandle(Z, |
568 Symbols::FromUTF32(string_chars.data(), string_chars.length())); | 568 Symbols::FromUTF32(string_chars.data(), string_chars.length())); |
569 } | 569 } |
570 EndStringLiteral(); | 570 EndStringLiteral(); |
571 return; | 571 return; |
572 } else { | 572 } else { |
573 string_chars.Add(string_delimiter_); | 573 string_chars.Add(string_delimiter_); |
574 } | 574 } |
575 } else { | 575 } else { |
576 // Test for a two part utf16 sequence, and decode to a code point | 576 // Test for a two part utf16 sequence, and decode to a code point |
577 // if we find one. | 577 // if we find one. |
(...skipping 344 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
922 (current_token_.kind == Token::kINTERPOL_START))) { | 922 (current_token_.kind == Token::kINTERPOL_START))) { |
923 index++; // Advance the index to account for tokens added in ScanAll. | 923 index++; // Advance the index to account for tokens added in ScanAll. |
924 } | 924 } |
925 index++; | 925 index++; |
926 prev_token_line_ = current_token_.position.line; | 926 prev_token_line_ = current_token_.position.line; |
927 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); | 927 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); |
928 } | 928 } |
929 | 929 |
930 | 930 |
931 const Scanner::GrowableTokenStream& Scanner::GetStream() { | 931 const Scanner::GrowableTokenStream& Scanner::GetStream() { |
932 GrowableTokenStream* ts = new(I) GrowableTokenStream(128); | 932 GrowableTokenStream* ts = new(Z) GrowableTokenStream(128); |
933 ScanAll(ts); | 933 ScanAll(ts); |
934 if (FLAG_print_tokens) { | 934 if (FLAG_print_tokens) { |
935 Scanner::PrintTokens(*ts); | 935 Scanner::PrintTokens(*ts); |
936 } | 936 } |
937 return *ts; | 937 return *ts; |
938 } | 938 } |
939 | 939 |
940 | 940 |
941 void Scanner::PrintTokens(const GrowableTokenStream& ts) { | 941 void Scanner::PrintTokens(const GrowableTokenStream& ts) { |
942 int currentLine = -1; | 942 int currentLine = -1; |
(...skipping 22 matching lines...) Expand all Loading... |
965 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 965 keywords_[i].keyword_symbol = &Symbols::Keyword(token); |
966 | 966 |
967 int ch = keywords_[i].keyword_chars[0] - 'a'; | 967 int ch = keywords_[i].keyword_chars[0] - 'a'; |
968 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 968 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
969 keywords_char_offset_[ch] = i; | 969 keywords_char_offset_[ch] = i; |
970 } | 970 } |
971 } | 971 } |
972 } | 972 } |
973 | 973 |
974 } // namespace dart | 974 } // namespace dart |
OLD | NEW |