| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // | 4 // |
| 5 // Scanner class for the Dart language. The scanner reads source text | 5 // Scanner class for the Dart language. The scanner reads source text |
| 6 // and produces a stream of tokens which is used by the parser. | 6 // and produces a stream of tokens which is used by the parser. |
| 7 // | 7 // |
| 8 | 8 |
| 9 #ifndef RUNTIME_VM_SCANNER_H_ | 9 #ifndef RUNTIME_VM_SCANNER_H_ |
| 10 #define RUNTIME_VM_SCANNER_H_ | 10 #define RUNTIME_VM_SCANNER_H_ |
| (...skipping 28 matching lines...) Expand all Loading... |
| 39 // the source text. | 39 // the source text. |
| 40 struct TokenDescriptor { | 40 struct TokenDescriptor { |
| 41 Token::Kind kind; | 41 Token::Kind kind; |
| 42 int offset; // Offset in source string. | 42 int offset; // Offset in source string. |
| 43 SourcePosition position; // Text position in source. | 43 SourcePosition position; // Text position in source. |
| 44 const String* literal; // Identifier, number or string literal. | 44 const String* literal; // Identifier, number or string literal. |
| 45 }; | 45 }; |
| 46 | 46 |
| 47 class TokenCollector : public ValueObject { | 47 class TokenCollector : public ValueObject { |
| 48 public: | 48 public: |
| 49 TokenCollector() { } | 49 TokenCollector() {} |
| 50 virtual ~TokenCollector() { } | 50 virtual ~TokenCollector() {} |
| 51 virtual void AddToken(const TokenDescriptor& token) { } | 51 virtual void AddToken(const TokenDescriptor& token) {} |
| 52 |
| 52 private: | 53 private: |
| 53 DISALLOW_COPY_AND_ASSIGN(TokenCollector); | 54 DISALLOW_COPY_AND_ASSIGN(TokenCollector); |
| 54 }; | 55 }; |
| 55 | 56 |
| 56 // Initializes scanner to scan string source. | 57 // Initializes scanner to scan string source. |
| 57 Scanner(const String& source, const String& private_key); | 58 Scanner(const String& source, const String& private_key); |
| 58 ~Scanner(); | 59 ~Scanner(); |
| 59 | 60 |
| 60 // Scans one token at a time. | 61 // Scans one token at a time. |
| 61 void Scan(); | 62 void Scan(); |
| (...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 159 bool ScanHexDigits(int digits, int32_t* value); | 160 bool ScanHexDigits(int digits, int32_t* value); |
| 160 | 161 |
| 161 // Reads a variable number of hexadecimal digits. | 162 // Reads a variable number of hexadecimal digits. |
| 162 bool ScanHexDigits(int min_digits, int max_digits, int32_t* value); | 163 bool ScanHexDigits(int min_digits, int max_digits, int32_t* value); |
| 163 | 164 |
| 164 // Reads an escaped code point from within a string literal. | 165 // Reads an escaped code point from within a string literal. |
| 165 void ScanEscapedCodePoint(int32_t* escaped_char); | 166 void ScanEscapedCodePoint(int32_t* escaped_char); |
| 166 | 167 |
| 167 // Reads identifier. | 168 // Reads identifier. |
| 168 void ScanIdentChars(bool allow_dollar); | 169 void ScanIdentChars(bool allow_dollar); |
| 169 void ScanIdent() { | 170 void ScanIdent() { ScanIdentChars(true); } |
| 170 ScanIdentChars(true); | 171 void ScanIdentNoDollar() { ScanIdentChars(false); } |
| 171 } | |
| 172 void ScanIdentNoDollar() { | |
| 173 ScanIdentChars(false); | |
| 174 } | |
| 175 | 172 |
| 176 // Reads a number literal. | 173 // Reads a number literal. |
| 177 void ScanNumber(bool dec_point_seen); | 174 void ScanNumber(bool dec_point_seen); |
| 178 | 175 |
| 179 void ScanScriptTag(); | 176 void ScanScriptTag(); |
| 180 | 177 |
| 181 CharAtFunc CallCharAt() const { return char_at_func_; } | 178 CharAtFunc CallCharAt() const { return char_at_func_; } |
| 182 | 179 |
| 183 Thread* thread() const { return thread_; } | 180 Thread* thread() const { return thread_; } |
| 184 Zone* zone() const { return zone_; } | 181 Zone* zone() const { return zone_; } |
| 185 | 182 |
| 186 TokenDescriptor current_token_; // Current token. | 183 TokenDescriptor current_token_; // Current token. |
| 187 TokenDescriptor newline_token_; // Newline token. | 184 TokenDescriptor newline_token_; // Newline token. |
| 188 TokenDescriptor empty_string_token_; // Token for "". | 185 TokenDescriptor empty_string_token_; // Token for "". |
| 189 const String& source_; // The source text being tokenized. | 186 const String& source_; // The source text being tokenized. |
| 190 intptr_t source_length_; // The length of the source text. | 187 intptr_t source_length_; // The length of the source text. |
| 191 intptr_t lookahead_pos_; // Position of lookahead character | 188 intptr_t lookahead_pos_; // Position of lookahead character |
| 192 // within source_. | 189 // within source_. |
| 193 intptr_t token_start_; // Begin of current token in src_. | 190 intptr_t token_start_; // Begin of current token in src_. |
| 194 int32_t c0_; // Lookahead character. | 191 int32_t c0_; // Lookahead character. |
| 195 bool newline_seen_; // Newline before current token. | 192 bool newline_seen_; // Newline before current token. |
| 196 intptr_t prev_token_line_; // Line number of the previous token. | 193 intptr_t prev_token_line_; // Line number of the previous token. |
| 197 | 194 |
| 198 // The following fields keep track whether we are scanning a string literal | 195 // The following fields keep track whether we are scanning a string literal |
| 199 // and its interpolated expressions. | 196 // and its interpolated expressions. |
| 200 ScanContext* saved_context_; | 197 ScanContext* saved_context_; |
| 201 int32_t string_delimiter_; | 198 int32_t string_delimiter_; |
| 202 bool string_is_multiline_; | 199 bool string_is_multiline_; |
| 203 int brace_level_; | 200 int brace_level_; |
| 204 | 201 |
| 205 const String& private_key_; | 202 const String& private_key_; |
| 206 | 203 |
| 207 SourcePosition c0_pos_; // Source position of lookahead character c0_. | 204 SourcePosition c0_pos_; // Source position of lookahead character c0_. |
| 208 | 205 |
| 209 const CharAtFunc char_at_func_; | 206 const CharAtFunc char_at_func_; |
| 210 | 207 |
| 211 Thread* thread_; | 208 Thread* thread_; |
| 212 Zone* zone_; | 209 Zone* zone_; |
| 213 | 210 |
| 214 static KeywordTable keywords_[Token::kNumKeywords]; | 211 static KeywordTable keywords_[Token::kNumKeywords]; |
| 215 static int keywords_char_offset_[kNumLowercaseChars]; | 212 static int keywords_char_offset_[kNumLowercaseChars]; |
| 216 }; | 213 }; |
| 217 | 214 |
| 218 | 215 |
| 219 } // namespace dart | 216 } // namespace dart |
| 220 | 217 |
| 221 #endif // RUNTIME_VM_SCANNER_H_ | 218 #endif // RUNTIME_VM_SCANNER_H_ |
| OLD | NEW |