Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(6)

Side by Side Diff: pkg/third_party/html5lib/lib/src/tokenizer.dart

Issue 421503004: Switch transformers over to source_span (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Code review changes Created 6 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 library tokenizer; 1 library tokenizer;
2 2
3 import 'dart:collection'; 3 import 'dart:collection';
4 import 'package:html5lib/parser.dart' show HtmlParser; 4 import 'package:html5lib/parser.dart' show HtmlParser;
5 import 'package:source_maps/span.dart' show Span, FileSpan;
6 import 'constants.dart'; 5 import 'constants.dart';
7 import 'inputstream.dart'; 6 import 'inputstream.dart';
8 import 'token.dart'; 7 import 'token.dart';
9 import 'utils.dart'; 8 import 'utils.dart';
10 9
11 // Group entities by their first character, for faster lookups 10 // Group entities by their first character, for faster lookups
12 11
13 // TODO(jmesserly): we could use a better data structure here like a trie, if 12 // TODO(jmesserly): we could use a better data structure here like a trie, if
14 // we had it implemented in Dart. 13 // we had it implemented in Dart.
15 Map<String, List<String>> entitiesByFirstChar = (() { 14 Map<String, List<String>> entitiesByFirstChar = (() {
(...skipping 134 matching lines...) Expand 10 before | Expand all | Expand 10 after
150 temporaryBuffer = null; 149 temporaryBuffer = null;
151 _attributes = null; 150 _attributes = null;
152 _attributeNames = null; 151 _attributeNames = null;
153 state = dataState; 152 state = dataState;
154 } 153 }
155 154
156 /// Adds a token to the queue. Sets the span if needed. 155 /// Adds a token to the queue. Sets the span if needed.
157 void _addToken(Token token) { 156 void _addToken(Token token) {
158 if (generateSpans && token.span == null) { 157 if (generateSpans && token.span == null) {
159 int offset = stream.position; 158 int offset = stream.position;
160 token.span = new FileSpan(stream.fileInfo, _lastOffset, offset); 159 token.span = stream.fileInfo.span(_lastOffset, offset);
161 if (token is! ParseErrorToken) { 160 if (token is! ParseErrorToken) {
162 _lastOffset = offset; 161 _lastOffset = offset;
163 } 162 }
164 } 163 }
165 tokenQueue.add(token); 164 tokenQueue.add(token);
166 } 165 }
167 166
168 /// This function returns either U+FFFD or the character based on the 167 /// This function returns either U+FFFD or the character based on the
169 /// decimal or hexadecimal representation. It also discards ";" if present. 168 /// decimal or hexadecimal representation. It also discards ";" if present.
170 /// If not present it will add a [ParseErrorToken]. 169 /// If not present it will add a [ParseErrorToken].
(...skipping 1707 matching lines...) Expand 10 before | Expand all | Expand 10 after
1878 } 1877 }
1879 1878
1880 if (data.length > 0) { 1879 if (data.length > 0) {
1881 _addToken(new CharactersToken(data.join())); 1880 _addToken(new CharactersToken(data.join()));
1882 } 1881 }
1883 state = dataState; 1882 state = dataState;
1884 return true; 1883 return true;
1885 } 1884 }
1886 } 1885 }
1887 1886
OLDNEW
« no previous file with comments | « pkg/third_party/html5lib/lib/src/token.dart ('k') | pkg/third_party/html5lib/lib/src/treebuilder.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698