| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 #include "vm/scanner.h" | 5 #include "vm/scanner.h" |
| 6 | 6 |
| 7 #include "platform/assert.h" | 7 #include "platform/assert.h" |
| 8 #include "vm/dart.h" | 8 #include "vm/dart.h" |
| 9 #include "vm/flags.h" | 9 #include "vm/flags.h" |
| 10 #include "vm/object.h" | 10 #include "vm/object.h" |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 51 c0_pos_.line = 1; | 51 c0_pos_.line = 1; |
| 52 c0_pos_.column = 0; | 52 c0_pos_.column = 0; |
| 53 ReadChar(); | 53 ReadChar(); |
| 54 } | 54 } |
| 55 | 55 |
| 56 | 56 |
| 57 Scanner::Scanner(const String& src, const String& private_key) | 57 Scanner::Scanner(const String& src, const String& private_key) |
| 58 : source_(src), | 58 : source_(src), |
| 59 source_length_(src.Length()), | 59 source_length_(src.Length()), |
| 60 saved_context_(NULL), | 60 saved_context_(NULL), |
| 61 private_key_(String::ZoneHandle(private_key.raw())) { | 61 private_key_(String::ZoneHandle(private_key.raw())), |
| 62 char_at_func_(src.CharAtFunc()) { |
| 62 Reset(); | 63 Reset(); |
| 63 } | 64 } |
| 64 | 65 |
| 65 Scanner::~Scanner() { | 66 Scanner::~Scanner() { |
| 66 while (saved_context_ != NULL) { | 67 while (saved_context_ != NULL) { |
| 67 ScanContext* ctx = saved_context_; | 68 ScanContext* ctx = saved_context_; |
| 68 saved_context_ = ctx->next; | 69 saved_context_ = ctx->next; |
| 69 delete ctx; | 70 delete ctx; |
| 70 } | 71 } |
| 71 } | 72 } |
| (...skipping 75 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 147 | 148 |
| 148 bool Scanner::IsIdentChar(int32_t c) { | 149 bool Scanner::IsIdentChar(int32_t c) { |
| 149 return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$'); | 150 return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$'); |
| 150 } | 151 } |
| 151 | 152 |
| 152 | 153 |
| 153 bool Scanner::IsIdent(const String& str) { | 154 bool Scanner::IsIdent(const String& str) { |
| 154 if (!str.IsOneByteString()) { | 155 if (!str.IsOneByteString()) { |
| 155 return false; | 156 return false; |
| 156 } | 157 } |
| 157 if (str.Length() == 0 || !IsIdentStartChar(str.CharAt(0))) { | 158 if (str.Length() == 0 || !IsIdentStartChar(CallCharAt()(str, 0))) { |
| 158 return false; | 159 return false; |
| 159 } | 160 } |
| 160 for (int i = 1; i < str.Length(); i++) { | 161 for (int i = 1; i < str.Length(); i++) { |
| 161 if (!IsIdentChar(str.CharAt(i))) { | 162 if (!IsIdentChar(CallCharAt()(str, i))) { |
| 162 return false; | 163 return false; |
| 163 } | 164 } |
| 164 } | 165 } |
| 165 return true; | 166 return true; |
| 166 } | 167 } |
| 167 | 168 |
| 168 | 169 |
| 169 // This method is used when parsing integers and doubles in Dart code. We | 170 // This method is used when parsing integers and doubles in Dart code. We |
| 170 // are reusing the Scanner's handling of number literals in that situation. | 171 // are reusing the Scanner's handling of number literals in that situation. |
| 171 bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens, | 172 bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens, |
| (...skipping 23 matching lines...) Expand all Loading... |
| 195 return false; | 196 return false; |
| 196 } | 197 } |
| 197 | 198 |
| 198 | 199 |
| 199 void Scanner::ReadChar() { | 200 void Scanner::ReadChar() { |
| 200 if (lookahead_pos_ < source_length_) { | 201 if (lookahead_pos_ < source_length_) { |
| 201 if (c0_ == '\n') { | 202 if (c0_ == '\n') { |
| 202 newline_seen_ = true; | 203 newline_seen_ = true; |
| 203 c0_pos_.line++; | 204 c0_pos_.line++; |
| 204 c0_pos_.column = 0; | 205 c0_pos_.column = 0; |
| 205 if (source_.CharAt(lookahead_pos_) == '\r') { | 206 if (CallCharAt()(source_, lookahead_pos_) == '\r') { |
| 206 // Replace a sequence of '\r' '\n' with a single '\n'. | 207 // Replace a sequence of '\r' '\n' with a single '\n'. |
| 207 if (LookaheadChar(1) == '\n') { | 208 if (LookaheadChar(1) == '\n') { |
| 208 lookahead_pos_++; | 209 lookahead_pos_++; |
| 209 } | 210 } |
| 210 } | 211 } |
| 211 } | 212 } |
| 212 lookahead_pos_++; | 213 lookahead_pos_++; |
| 213 c0_pos_.column++; | 214 c0_pos_.column++; |
| 214 c0_ = LookaheadChar(0); | 215 c0_ = LookaheadChar(0); |
| 215 // Replace '\r' with '\n'. | 216 // Replace '\r' with '\n'. |
| 216 if (c0_ == '\r') { | 217 if (c0_ == '\r') { |
| 217 c0_ = '\n'; | 218 c0_ = '\n'; |
| 218 } | 219 } |
| 219 } | 220 } |
| 220 } | 221 } |
| 221 | 222 |
| 222 | 223 |
| 223 // Look ahead 'how_many' characters. Returns the character, or '\0' if | 224 // Look ahead 'how_many' characters. Returns the character, or '\0' if |
| 224 // the lookahead position is beyond the end of the string. Does not | 225 // the lookahead position is beyond the end of the string. Does not |
| 225 // normalize line end characters into '\n'. | 226 // normalize line end characters into '\n'. |
| 226 int32_t Scanner::LookaheadChar(int how_many) { | 227 int32_t Scanner::LookaheadChar(int how_many) { |
| 227 ASSERT(how_many >= 0); | 228 ASSERT(how_many >= 0); |
| 228 int32_t lookahead_char = '\0'; | 229 int32_t lookahead_char = '\0'; |
| 229 if (lookahead_pos_ + how_many < source_length_) { | 230 if (lookahead_pos_ + how_many < source_length_) { |
| 230 lookahead_char = source_.CharAt(lookahead_pos_ + how_many); | 231 lookahead_char = CallCharAt()(source_, lookahead_pos_ + how_many); |
| 231 } | 232 } |
| 232 return lookahead_char; | 233 return lookahead_char; |
| 233 } | 234 } |
| 234 | 235 |
| 235 | 236 |
| 236 void Scanner::ConsumeWhiteSpace() { | 237 void Scanner::ConsumeWhiteSpace() { |
| 237 while (c0_ == ' ' || c0_ == '\t' || c0_ == '\n') { | 238 while (c0_ == ' ' || c0_ == '\t' || c0_ == '\n') { |
| 238 ReadChar(); | 239 ReadChar(); |
| 239 } | 240 } |
| 240 } | 241 } |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 275 current_token_.kind = | 276 current_token_.kind = |
| 276 (nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL; | 277 (nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL; |
| 277 } | 278 } |
| 278 | 279 |
| 279 | 280 |
| 280 void Scanner::ScanIdentChars(bool allow_dollar) { | 281 void Scanner::ScanIdentChars(bool allow_dollar) { |
| 281 ASSERT(IsIdentStartChar(c0_)); | 282 ASSERT(IsIdentStartChar(c0_)); |
| 282 ASSERT(allow_dollar || (c0_ != '$')); | 283 ASSERT(allow_dollar || (c0_ != '$')); |
| 283 int ident_length = 0; | 284 int ident_length = 0; |
| 284 int ident_pos = lookahead_pos_; | 285 int ident_pos = lookahead_pos_; |
| 285 int32_t ident_char0 = source_.CharAt(ident_pos); | 286 int32_t ident_char0 = CallCharAt()(source_, ident_pos); |
| 286 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { | 287 while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) { |
| 287 ReadChar(); | 288 ReadChar(); |
| 288 ident_length++; | 289 ident_length++; |
| 289 } | 290 } |
| 290 | 291 |
| 291 // Check whether the characters we read are a known keyword. | 292 // Check whether the characters we read are a known keyword. |
| 292 // Note, can't use strcmp since token_chars is not null-terminated. | 293 // Note, can't use strcmp since token_chars is not null-terminated. |
| 293 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { | 294 if (('a' <= ident_char0) && (ident_char0 <= 'z')) { |
| 294 int i = keywords_char_offset_[ident_char0 - 'a']; | 295 int i = keywords_char_offset_[ident_char0 - 'a']; |
| 295 while (i < Token::kNumKeywords && | 296 while (i < Token::kNumKeywords && |
| 296 keywords_[i].keyword_chars[0] <= ident_char0) { | 297 keywords_[i].keyword_chars[0] <= ident_char0) { |
| 297 if (keywords_[i].keyword_len == ident_length) { | 298 if (keywords_[i].keyword_len == ident_length) { |
| 298 const char* keyword = keywords_[i].keyword_chars; | 299 const char* keyword = keywords_[i].keyword_chars; |
| 299 int char_pos = 1; | 300 int char_pos = 1; |
| 300 while ((char_pos < ident_length) && | 301 while ((char_pos < ident_length) && |
| 301 (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) { | 302 (keyword[char_pos] == |
| 303 CallCharAt()(source_, ident_pos + char_pos))) { |
| 302 char_pos++; | 304 char_pos++; |
| 303 } | 305 } |
| 304 if (char_pos == ident_length) { | 306 if (char_pos == ident_length) { |
| 305 current_token_.literal = keywords_[i].keyword_symbol; | 307 current_token_.literal = keywords_[i].keyword_symbol; |
| 306 current_token_.kind = keywords_[i].kind; | 308 current_token_.kind = keywords_[i].kind; |
| 307 return; | 309 return; |
| 308 } | 310 } |
| 309 } | 311 } |
| 310 i++; | 312 i++; |
| 311 } | 313 } |
| (...skipping 639 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 951 keywords_[i].keyword_symbol = &Symbols::Keyword(token); | 953 keywords_[i].keyword_symbol = &Symbols::Keyword(token); |
| 952 | 954 |
| 953 int ch = keywords_[i].keyword_chars[0] - 'a'; | 955 int ch = keywords_[i].keyword_chars[0] - 'a'; |
| 954 if (keywords_char_offset_[ch] == Token::kNumKeywords) { | 956 if (keywords_char_offset_[ch] == Token::kNumKeywords) { |
| 955 keywords_char_offset_[ch] = i; | 957 keywords_char_offset_[ch] = i; |
| 956 } | 958 } |
| 957 } | 959 } |
| 958 } | 960 } |
| 959 | 961 |
| 960 } // namespace dart | 962 } // namespace dart |
| OLD | NEW |