OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
6 | 6 |
7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
10 #include "vm/object.h" | 10 #include "vm/object.h" |
11 #include "vm/object_store.h" | 11 #include "vm/object_store.h" |
12 #include "vm/symbols.h" | 12 #include "vm/symbols.h" |
13 #include "vm/token.h" | 13 #include "vm/token.h" |
14 #include "vm/unicode.h" | 14 #include "vm/unicode.h" |
15 | 15 |
16 namespace dart { | 16 namespace dart { |
17 | 17 |
18 DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens."); | |
19 | |
20 | |
21 // Quick access to the locally defined zone() and thread() methods. | 18 // Quick access to the locally defined zone() and thread() methods. |
22 #define Z (zone()) | 19 #define Z (zone()) |
23 #define T (thread()) | 20 #define T (thread()) |
24 | 21 |
25 | 22 |
26 class ScanContext : public ZoneAllocated { | 23 class ScanContext : public ZoneAllocated { |
27 public: | 24 public: |
28 explicit ScanContext(Scanner* scanner) | 25 explicit ScanContext(Scanner* scanner) |
29 : next_(scanner->saved_context_), | 26 : next_(scanner->saved_context_), |
30 string_delimiter_(scanner->string_delimiter_), | 27 string_delimiter_(scanner->string_delimiter_), |
(...skipping 143 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
174 } | 171 } |
175 for (int i = 1; i < str.Length(); i++) { | 172 for (int i = 1; i < str.Length(); i++) { |
176 if (!IsIdentChar(CallCharAt()(str, i))) { | 173 if (!IsIdentChar(CallCharAt()(str, i))) { |
177 return false; | 174 return false; |
178 } | 175 } |
179 } | 176 } |
180 return true; | 177 return true; |
181 } | 178 } |
182 | 179 |
183 | 180 |
184 // This method is used when parsing integers and doubles in Dart code. We | 181 // This method is used when parsing integers in Dart code. We |
185 // are reusing the Scanner's handling of number literals in that situation. | 182 // are reusing the Scanner's handling of number literals in that situation. |
186 bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens, | 183 bool Scanner::IsValidInteger(const String& str, |
187 Token::Kind literal_kind, | |
188 bool* is_positive, | 184 bool* is_positive, |
189 const String** value) { | 185 const String** value) { |
190 if ((tokens.length() == 2) && | 186 Scanner s(str, Symbols::Empty()); |
191 (tokens[0].kind == literal_kind) && | 187 TokenDescriptor tokens[3]; |
| 188 s.Scan(); |
| 189 tokens[0] = s.current_token(); |
| 190 s.Scan(); |
| 191 tokens[1] = s.current_token(); |
| 192 s.Scan(); |
| 193 tokens[2] = s.current_token(); |
| 194 |
| 195 if ((tokens[0].kind == Token::kINTEGER) && |
192 (tokens[1].kind == Token::kEOS)) { | 196 (tokens[1].kind == Token::kEOS)) { |
193 *is_positive = true; | 197 *is_positive = true; |
194 *value = tokens[0].literal; | 198 *value = tokens[0].literal; |
195 return true; | 199 return true; |
196 } | 200 } |
197 if ((tokens.length() == 3) && | 201 if (((tokens[0].kind == Token::kADD) || |
198 ((tokens[0].kind == Token::kADD) || | |
199 (tokens[0].kind == Token::kSUB)) && | 202 (tokens[0].kind == Token::kSUB)) && |
200 (tokens[1].kind == literal_kind) && | 203 (tokens[1].kind == Token::kINTEGER) && |
201 (tokens[2].kind == Token::kEOS)) { | 204 (tokens[2].kind == Token::kEOS)) { |
202 // Check there is no space between "+/-" and number. | 205 // Check there is no space between "+/-" and number. |
203 if ((tokens[0].offset + 1) != tokens[1].offset) { | 206 if ((tokens[0].offset + 1) != tokens[1].offset) { |
204 return false; | 207 return false; |
205 } | 208 } |
206 *is_positive = tokens[0].kind == Token::kADD; | 209 *is_positive = tokens[0].kind == Token::kADD; |
207 *value = tokens[1].literal; | 210 *value = tokens[1].literal; |
208 return true; | 211 return true; |
209 } | 212 } |
210 return false; | 213 return false; |
(...skipping 668 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
879 OS::SNPrint(msg, sizeof(msg), | 882 OS::SNPrint(msg, sizeof(msg), |
880 "unexpected character: '%s' (U+%04X)\n", utf8_char, c0_); | 883 "unexpected character: '%s' (U+%04X)\n", utf8_char, c0_); |
881 ErrorMsg(msg); | 884 ErrorMsg(msg); |
882 ReadChar(); | 885 ReadChar(); |
883 } | 886 } |
884 } | 887 } |
885 } while (current_token_.kind == Token::kWHITESP); | 888 } while (current_token_.kind == Token::kWHITESP); |
886 } | 889 } |
887 | 890 |
888 | 891 |
889 void Scanner::ScanAll(GrowableTokenStream* token_stream) { | 892 void Scanner::ScanAll(TokenCollector* collector) { |
890 Reset(); | 893 Reset(); |
891 do { | 894 do { |
892 Scan(); | 895 Scan(); |
893 | |
894 bool inserted_new_lines = false; | 896 bool inserted_new_lines = false; |
895 for (intptr_t diff = current_token_.position.line - prev_token_line_; | 897 for (intptr_t diff = current_token_.position.line - prev_token_line_; |
896 diff > 0; | 898 diff > 0; |
897 diff--) { | 899 diff--) { |
898 newline_token_.position.line = current_token_.position.line - diff; | 900 newline_token_.position.line = current_token_.position.line - diff; |
899 token_stream->Add(newline_token_); | 901 collector->AddToken(newline_token_); |
900 inserted_new_lines = true; | 902 inserted_new_lines = true; |
901 } | 903 } |
902 | |
903 if (inserted_new_lines && | 904 if (inserted_new_lines && |
904 ((current_token_.kind == Token::kINTERPOL_VAR) || | 905 ((current_token_.kind == Token::kINTERPOL_VAR) || |
905 (current_token_.kind == Token::kINTERPOL_START))) { | 906 (current_token_.kind == Token::kINTERPOL_START))) { |
906 // NOTE: If this changes, be sure to update | 907 // NOTE: If this changes, be sure to update |
907 // Script::GenerateLineNumberArray to stay in sync. | 908 // Script::GenerateLineNumberArray to stay in sync. |
908 empty_string_token_.position.line = current_token_.position.line; | 909 empty_string_token_.position.line = current_token_.position.line; |
909 token_stream->Add(empty_string_token_); | 910 collector->AddToken(empty_string_token_); |
910 } | 911 } |
911 token_stream->Add(current_token_); | 912 collector->AddToken(current_token_); |
912 prev_token_line_ = current_token_.position.line; | 913 prev_token_line_ = current_token_.position.line; |
913 } while (current_token_.kind != Token::kEOS); | 914 } while (current_token_.kind != Token::kEOS); |
914 } | 915 } |
915 | 916 |
916 | 917 |
917 void Scanner::ScanTo(intptr_t token_index) { | 918 void Scanner::ScanTo(intptr_t token_index) { |
918 ASSERT(token_index >= 0); | 919 ASSERT(token_index >= 0); |
919 intptr_t index = 0; | 920 intptr_t index = 0; |
920 Reset(); | 921 Reset(); |
921 do { | 922 do { |
922 Scan(); | 923 Scan(); |
923 | |
924 bool inserted_new_lines = false; | 924 bool inserted_new_lines = false; |
925 for (intptr_t diff = current_token_.position.line - prev_token_line_; | 925 for (intptr_t diff = current_token_.position.line - prev_token_line_; |
926 diff > 0; | 926 diff > 0; |
927 diff--) { | 927 diff--) { |
928 // Advance the index to account for tokens added in ScanAll. | 928 // Advance the index to account for tokens added in ScanAll. |
929 index++; | 929 index++; |
930 inserted_new_lines = true; | 930 inserted_new_lines = true; |
931 } | 931 } |
932 | |
933 if (inserted_new_lines && | 932 if (inserted_new_lines && |
934 ((current_token_.kind == Token::kINTERPOL_VAR) || | 933 ((current_token_.kind == Token::kINTERPOL_VAR) || |
935 (current_token_.kind == Token::kINTERPOL_START))) { | 934 (current_token_.kind == Token::kINTERPOL_START))) { |
936 // Advance the index to account for tokens added in ScanAll. | 935 // Advance the index to account for tokens added in ScanAll. |
937 index++; | 936 index++; |
938 } | 937 } |
939 index++; | 938 index++; |
940 prev_token_line_ = current_token_.position.line; | 939 prev_token_line_ = current_token_.position.line; |
941 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); | 940 } while ((token_index >= index) && (current_token_.kind != Token::kEOS)); |
942 } | 941 } |
943 | 942 |
944 | 943 |
945 const Scanner::GrowableTokenStream& Scanner::GetStream() { | |
946 GrowableTokenStream* ts = new(Z) GrowableTokenStream(128); | |
947 ScanAll(ts); | |
948 if (FLAG_print_tokens) { | |
949 Scanner::PrintTokens(*ts); | |
950 } | |
951 return *ts; | |
952 } | |
953 | |
954 | |
955 void Scanner::PrintTokens(const GrowableTokenStream& ts) { | |
956 int currentLine = -1; | |
957 for (int i = 0; i < ts.length(); i++) { | |
958 const TokenDescriptor& td = ts[i]; | |
959 if (currentLine != td.position.line) { | |
960 currentLine = td.position.line; | |
961 OS::Print("\n%d (%d): ", currentLine, i); | |
962 } | |
963 OS::Print("%s ", Token::Name(td.kind)); | |
964 } | |
965 OS::Print("\n"); | |
966 } | |
967 | |
968 | |
969 void Scanner::InitOnce() { | 944 void Scanner::InitOnce() { |
970 ASSERT(Isolate::Current() == Dart::vm_isolate()); | 945 ASSERT(Isolate::Current() == Dart::vm_isolate()); |
971 for (int i = 0; i < kNumLowercaseChars; i++) { | 946 for (int i = 0; i < kNumLowercaseChars; i++) { |
972 keywords_char_offset_[i] = Token::kNumKeywords; | 947 keywords_char_offset_[i] = Token::kNumKeywords; |
973 } | 948 } |
974 for (int i = 0; i < Token::kNumKeywords; i++) { | 949 for (int i = 0; i < Token::kNumKeywords; i++) { |
975 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); | 950 Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i); |
976 keywords_[i].kind = token; | 951 keywords_[i].kind = token; |
977 keywords_[i].keyword_chars = Token::Str(token); | 952 keywords_[i].keyword_chars = Token::Str(token); |
978 keywords_[i].keyword_len = strlen(Token::Str(token)); | 953 keywords_[i].keyword_len = strlen(Token::Str(token)); |
979 keywords_[i].keyword_symbol = &Symbols::Token(token); | 954 keywords_[i].keyword_symbol = &Symbols::Token(token); |
980 | 955 |
981 int ch = keywords_[i].keyword_chars[0] - 'a'; | 956 int ch = keywords_[i].keyword_chars[0] - 'a'; |
982 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 957 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
983 keywords_char_offset_[ch] = i; | 958 keywords_char_offset_[ch] = i; |
984 } | 959 } |
985 } | 960 } |
986 } | 961 } |
987 | 962 |
988 } // namespace dart | 963 } // namespace dart |
OLD | NEW |