| OLD | NEW |
| 1 library tokenizer; | 1 library tokenizer; |
| 2 | 2 |
| 3 import 'dart:collection'; | 3 import 'dart:collection'; |
| 4 import 'package:html5lib/parser.dart' show HtmlParser; | 4 import 'package:html5lib/parser.dart' show HtmlParser; |
| 5 import 'package:source_maps/span.dart' show Span, FileSpan; | |
| 6 import 'constants.dart'; | 5 import 'constants.dart'; |
| 7 import 'inputstream.dart'; | 6 import 'inputstream.dart'; |
| 8 import 'token.dart'; | 7 import 'token.dart'; |
| 9 import 'utils.dart'; | 8 import 'utils.dart'; |
| 10 | 9 |
| 11 // Group entities by their first character, for faster lookups | 10 // Group entities by their first character, for faster lookups |
| 12 | 11 |
| 13 // TODO(jmesserly): we could use a better data structure here like a trie, if | 12 // TODO(jmesserly): we could use a better data structure here like a trie, if |
| 14 // we had it implemented in Dart. | 13 // we had it implemented in Dart. |
| 15 Map<String, List<String>> entitiesByFirstChar = (() { | 14 Map<String, List<String>> entitiesByFirstChar = (() { |
| (...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 150 temporaryBuffer = null; | 149 temporaryBuffer = null; |
| 151 _attributes = null; | 150 _attributes = null; |
| 152 _attributeNames = null; | 151 _attributeNames = null; |
| 153 state = dataState; | 152 state = dataState; |
| 154 } | 153 } |
| 155 | 154 |
| 156 /// Adds a token to the queue. Sets the span if needed. | 155 /// Adds a token to the queue. Sets the span if needed. |
| 157 void _addToken(Token token) { | 156 void _addToken(Token token) { |
| 158 if (generateSpans && token.span == null) { | 157 if (generateSpans && token.span == null) { |
| 159 int offset = stream.position; | 158 int offset = stream.position; |
| 160 token.span = new FileSpan(stream.fileInfo, _lastOffset, offset); | 159 token.span = stream.fileInfo.span(_lastOffset, offset); |
| 161 if (token is! ParseErrorToken) { | 160 if (token is! ParseErrorToken) { |
| 162 _lastOffset = offset; | 161 _lastOffset = offset; |
| 163 } | 162 } |
| 164 } | 163 } |
| 165 tokenQueue.add(token); | 164 tokenQueue.add(token); |
| 166 } | 165 } |
| 167 | 166 |
| 168 /// This function returns either U+FFFD or the character based on the | 167 /// This function returns either U+FFFD or the character based on the |
| 169 /// decimal or hexadecimal representation. It also discards ";" if present. | 168 /// decimal or hexadecimal representation. It also discards ";" if present. |
| 170 /// If not present it will add a [ParseErrorToken]. | 169 /// If not present it will add a [ParseErrorToken]. |
| (...skipping 1707 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1878 } | 1877 } |
| 1879 | 1878 |
| 1880 if (data.length > 0) { | 1879 if (data.length > 0) { |
| 1881 _addToken(new CharactersToken(data.join())); | 1880 _addToken(new CharactersToken(data.join())); |
| 1882 } | 1881 } |
| 1883 state = dataState; | 1882 state = dataState; |
| 1884 return true; | 1883 return true; |
| 1885 } | 1884 } |
| 1886 } | 1885 } |
| 1887 | 1886 |
| OLD | NEW |