| OLD | NEW |
| 1 library angular.core.parser.lexer; | 1 library angular.core.parser.lexer; |
| 2 | 2 |
| 3 import 'package:angular/core/module.dart' show NgInjectableService; | 3 import 'package:angular/core/annotation_src.dart'; |
| 4 import 'package:angular/core/parser/characters.dart'; | 4 import 'package:angular/core/parser/characters.dart'; |
| 5 | 5 |
| 6 class Token { | 6 part 'tokens.dart'; |
| 7 final int index; | |
| 8 final String text; | |
| 9 | 7 |
| 10 var value; | 8 @Injectable() |
| 11 // Tokens should have one of these set. | |
| 12 String opKey; | |
| 13 String key; | |
| 14 | |
| 15 Token(this.index, this.text); | |
| 16 | |
| 17 withOp(op) { | |
| 18 this.opKey = op; | |
| 19 } | |
| 20 | |
| 21 withGetterSetter(key) { | |
| 22 this.key = key; | |
| 23 } | |
| 24 | |
| 25 withValue(value) { this.value = value; } | |
| 26 | |
| 27 toString() => "Token($text)"; | |
| 28 } | |
| 29 | |
| 30 | |
| 31 @NgInjectableService() | |
| 32 class Lexer { | 9 class Lexer { |
| 33 List<Token> call(String text) { | 10 List<Token> call(String text) { |
| 34 Scanner scanner = new Scanner(text); | 11 Scanner scanner = new Scanner(text); |
| 35 List<Token> tokens = []; | 12 List<Token> tokens = []; |
| 36 Token token = scanner.scanToken(); | 13 Token token = scanner.scanToken(); |
| 37 while (token != null) { | 14 while (token != null) { |
| 38 tokens.add(token); | 15 tokens.add(token); |
| 39 token = scanner.scanToken(); | 16 token = scanner.scanToken(); |
| 40 } | 17 } |
| 41 return tokens; | 18 return tokens; |
| (...skipping 23 matching lines...) Expand all Loading... |
| 65 } | 42 } |
| 66 | 43 |
| 67 // Handle identifiers and numbers. | 44 // Handle identifiers and numbers. |
| 68 if (isIdentifierStart(peek)) return scanIdentifier(); | 45 if (isIdentifierStart(peek)) return scanIdentifier(); |
| 69 if (isDigit(peek)) return scanNumber(index); | 46 if (isDigit(peek)) return scanNumber(index); |
| 70 | 47 |
| 71 int start = index; | 48 int start = index; |
| 72 switch (peek) { | 49 switch (peek) { |
| 73 case $PERIOD: | 50 case $PERIOD: |
| 74 advance(); | 51 advance(); |
| 75 return isDigit(peek) ? scanNumber(start) : new Token(start, '.'); | 52 return isDigit(peek) ? scanNumber(start) : new CharacterToken(start, $PE
RIOD); |
| 76 case $LPAREN: | 53 case $LPAREN: |
| 77 case $RPAREN: | 54 case $RPAREN: |
| 78 case $LBRACE: | 55 case $LBRACE: |
| 79 case $RBRACE: | 56 case $RBRACE: |
| 80 case $LBRACKET: | 57 case $LBRACKET: |
| 81 case $RBRACKET: | 58 case $RBRACKET: |
| 82 case $COMMA: | 59 case $COMMA: |
| 83 case $COLON: | 60 case $COLON: |
| 84 case $SEMICOLON: | 61 case $SEMICOLON: |
| 85 return scanCharacter(start, new String.fromCharCode(peek)); | 62 return scanCharacter(start, peek); |
| 86 case $SQ: | 63 case $SQ: |
| 87 case $DQ: | 64 case $DQ: |
| 88 return scanString(); | 65 return scanString(); |
| 89 case $PLUS: | 66 case $PLUS: |
| 90 case $MINUS: | 67 case $MINUS: |
| 91 case $STAR: | 68 case $STAR: |
| 92 case $SLASH: | 69 case $SLASH: |
| 93 case $PERCENT: | 70 case $PERCENT: |
| 94 case $CARET: | 71 case $CARET: |
| 95 case $QUESTION: | 72 case $QUESTION: |
| (...skipping 12 matching lines...) Expand all Loading... |
| 108 case $NBSP: | 85 case $NBSP: |
| 109 while (isWhitespace(peek)) advance(); | 86 while (isWhitespace(peek)) advance(); |
| 110 return scanToken(); | 87 return scanToken(); |
| 111 } | 88 } |
| 112 | 89 |
| 113 String character = new String.fromCharCode(peek); | 90 String character = new String.fromCharCode(peek); |
| 114 error('Unexpected character [$character]'); | 91 error('Unexpected character [$character]'); |
| 115 return null; | 92 return null; |
| 116 } | 93 } |
| 117 | 94 |
| 118 Token scanCharacter(int start, String string) { | 95 Token scanCharacter(int start, int code) { |
| 119 assert(peek == string.codeUnitAt(0)); | 96 assert(peek == code); |
| 120 advance(); | 97 advance(); |
| 121 return new Token(start, string); | 98 return new CharacterToken(start, code); |
| 122 } | 99 } |
| 123 | 100 |
| 124 Token scanOperator(int start, String string) { | 101 Token scanOperator(int start, String string) { |
| 125 assert(peek == string.codeUnitAt(0)); | 102 assert(peek == string.codeUnitAt(0)); |
| 126 assert(OPERATORS.contains(string)); | 103 assert(OPERATORS.contains(string)); |
| 127 advance(); | 104 advance(); |
| 128 return new Token(start, string)..withOp(string); | 105 return new OperatorToken(start, string); |
| 129 } | 106 } |
| 130 | 107 |
| 131 Token scanComplexOperator(int start, int code, String one, String two) { | 108 Token scanComplexOperator(int start, int code, String one, String two) { |
| 132 assert(peek == one.codeUnitAt(0)); | 109 assert(peek == one.codeUnitAt(0)); |
| 133 advance(); | 110 advance(); |
| 134 String string = one; | 111 String string = one; |
| 135 if (peek == code) { | 112 if (peek == code) { |
| 136 advance(); | 113 advance(); |
| 137 string += two; | 114 string += two; |
| 138 } | 115 } |
| 139 assert(OPERATORS.contains(string)); | 116 assert(OPERATORS.contains(string)); |
| 140 return new Token(start, string)..withOp(string); | 117 return new OperatorToken(start, string); |
| 141 } | 118 } |
| 142 | 119 |
| 143 Token scanIdentifier() { | 120 Token scanIdentifier() { |
| 144 assert(isIdentifierStart(peek)); | 121 assert(isIdentifierStart(peek)); |
| 145 int start = index; | 122 int start = index; |
| 146 advance(); | 123 advance(); |
| 147 while (isIdentifierPart(peek)) advance(); | 124 while (isIdentifierPart(peek)) advance(); |
| 148 String string = input.substring(start, index); | 125 String string = input.substring(start, index); |
| 149 Token result = new Token(start, string); | 126 return new IdentifierToken(start, string, KEYWORDS.contains(string)); |
| 150 // TODO(kasperl): Deal with null, undefined, true, and false in | |
| 151 // a cleaner and faster way. | |
| 152 if (OPERATORS.contains(string)) { | |
| 153 result.withOp(string); | |
| 154 } else { | |
| 155 result.withGetterSetter(string); | |
| 156 } | |
| 157 return result; | |
| 158 } | 127 } |
| 159 | 128 |
| 160 Token scanNumber(int start) { | 129 Token scanNumber(int start) { |
| 161 assert(isDigit(peek)); | 130 assert(isDigit(peek)); |
| 162 bool simple = (index == start); | 131 bool simple = (index == start); |
| 163 advance(); // Skip initial digit. | 132 advance(); // Skip initial digit. |
| 164 while (true) { | 133 while (true) { |
| 165 if (isDigit(peek)) { | 134 if (isDigit(peek)) { |
| 166 // Do nothing. | 135 // Do nothing. |
| 167 } else if (peek == $PERIOD) { | 136 } else if (peek == $PERIOD) { |
| 168 simple = false; | 137 simple = false; |
| 169 } else if (isExponentStart(peek)) { | 138 } else if (isExponentStart(peek)) { |
| 170 advance(); | 139 advance(); |
| 171 if (isExponentSign(peek)) advance(); | 140 if (isExponentSign(peek)) advance(); |
| 172 if (!isDigit(peek)) error('Invalid exponent', -1); | 141 if (!isDigit(peek)) error('Invalid exponent', -1); |
| 173 simple = false; | 142 simple = false; |
| 174 } else { | 143 } else { |
| 175 break; | 144 break; |
| 176 } | 145 } |
| 177 advance(); | 146 advance(); |
| 178 } | 147 } |
| 179 String string = input.substring(start, index); | 148 String string = input.substring(start, index); |
| 180 num value = simple ? int.parse(string) : double.parse(string); | 149 num value = simple ? int.parse(string) : double.parse(string); |
| 181 return new Token(start, string)..withValue(value); | 150 return new NumberToken(start, value); |
| 182 } | 151 } |
| 183 | 152 |
| 184 Token scanString() { | 153 Token scanString() { |
| 185 assert(peek == $SQ || peek == $DQ); | 154 assert(peek == $SQ || peek == $DQ); |
| 186 int start = index; | 155 int start = index; |
| 187 int quote = peek; | 156 int quote = peek; |
| 188 advance(); // Skip initial quote. | 157 advance(); // Skip initial quote. |
| 189 | 158 |
| 190 StringBuffer buffer; | 159 StringBuffer buffer; |
| 191 int marker = index; | 160 int marker = index; |
| 192 | 161 |
| 193 while (peek != quote) { | 162 while (peek != quote) { |
| 194 if (peek == $BACKSLASH) { | 163 if (peek == $BACKSLASH) { |
| 195 if (buffer == null) buffer = new StringBuffer(); | 164 if (buffer == null) buffer = new StringBuffer(); |
| 196 buffer.write(input.substring(marker, index)); | 165 buffer.write(input.substring(marker, index)); |
| 197 advance(); | 166 advance(); |
| 198 int unescaped; | 167 int unescaped; |
| 199 if (peek == $u) { | 168 if (peek == $u) { |
| 200 // TODO(kasperl): Check bounds? Make sure we have test | 169 // TODO(kasperl): Check bounds? Make sure we have test |
| 201 // coverage for this. | 170 // coverage for this. |
| 202 String hex = input.substring(index + 1, index + 5); | 171 String hex = input.substring(index + 1, index + 5); |
| 203 unescaped = int.parse(hex, radix: 16, onError: (ignore) { | 172 unescaped = int.parse(hex, radix: 16, onError: (ignore) { |
| 204 error('Invalid unicode escape [\\u$hex]'); }); | 173 error('Invalid unicode escape [\\u$hex]'); }); |
| 205 for (int i = 0; i < 5; i++) advance(); | 174 for (int i = 0; i < 5; i++) { |
| 175 advance(); |
| 176 } |
| 206 } else { | 177 } else { |
| 207 unescaped = unescape(peek); | 178 unescaped = unescape(peek); |
| 208 advance(); | 179 advance(); |
| 209 } | 180 } |
| 210 buffer.writeCharCode(unescaped); | 181 buffer.writeCharCode(unescaped); |
| 211 marker = index; | 182 marker = index; |
| 212 } else if (peek == $EOF) { | 183 } else if (peek == $EOF) { |
| 213 error('Unterminated quote'); | 184 error('Unterminated quote'); |
| 214 } else { | 185 } else { |
| 215 advance(); | 186 advance(); |
| 216 } | 187 } |
| 217 } | 188 } |
| 218 | 189 |
| 219 String last = input.substring(marker, index); | 190 String last = input.substring(marker, index); |
| 220 advance(); // Skip terminating quote. | 191 advance(); // Skip terminating quote. |
| 221 String string = input.substring(start, index); | 192 String string = input.substring(start, index); |
| 222 | 193 |
| 223 // Compute the unescaped string value. | 194 // Compute the unescaped string value. |
| 224 String unescaped = last; | 195 String unescaped = last; |
| 225 if (buffer != null) { | 196 if (buffer != null) { |
| 226 buffer.write(last); | 197 buffer.write(last); |
| 227 unescaped = buffer.toString(); | 198 unescaped = buffer.toString(); |
| 228 } | 199 } |
| 229 return new Token(start, string)..withValue(unescaped); | 200 return new StringToken(start, string, unescaped); |
| 230 } | 201 } |
| 231 | 202 |
| 232 void advance() { | 203 void advance() { |
| 233 if (++index >= length) peek = $EOF; | 204 peek = ++index >= length ? $EOF : input.codeUnitAt(index); |
| 234 else peek = input.codeUnitAt(index); | |
| 235 } | 205 } |
| 236 | 206 |
| 237 void error(String message, [int offset = 0]) { | 207 void error(String message, [int offset = 0]) { |
| 238 // TODO(kasperl): Try to get rid of the offset. It is only used to match | 208 // TODO(kasperl): Try to get rid of the offset. It is only used to match |
| 239 // the error expectations in the lexer tests for numbers with exponents. | 209 // the error expectations in the lexer tests for numbers with exponents. |
| 240 int position = index + offset; | 210 int position = index + offset; |
| 241 throw "Lexer Error: $message at column $position in expression [$input]"; | 211 throw "Lexer Error: $message at column $position in expression [$input]"; |
| 242 } | 212 } |
| 243 } | 213 } |
| 244 | 214 |
| 245 Set<String> OPERATORS = new Set<String>.from([ | 215 Set<String> KEYWORDS = new Set<String>.from([ |
| 216 'null', |
| 246 'undefined', | 217 'undefined', |
| 247 'null', | |
| 248 'true', | 218 'true', |
| 249 'false', | 219 'false', |
| 220 ]); |
| 221 |
| 222 Set<String> OPERATORS = new Set<String>.from([ |
| 250 '+', | 223 '+', |
| 251 '-', | 224 '-', |
| 252 '*', | 225 '*', |
| 253 '/', | 226 '/', |
| 254 '~/', | 227 '~/', |
| 255 '%', | 228 '%', |
| 256 '^', | 229 '^', |
| 257 '=', | 230 '=', |
| 258 '==', | 231 '==', |
| 259 '!=', | 232 '!=', |
| 260 '<', | 233 '<', |
| 261 '>', | 234 '>', |
| 262 '<=', | 235 '<=', |
| 263 '>=', | 236 '>=', |
| 264 '&&', | 237 '&&', |
| 265 '||', | 238 '||', |
| 266 '&', | 239 '&', |
| 267 '|', | 240 '|', |
| 268 '!', | 241 '!', |
| 269 '?', | 242 '?', |
| 270 ]); | 243 ]); |
| 271 | 244 |
| OLD | NEW |