Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(248)

Issues Search
    My Issues | Starred     Open | Closed | All

Unified Diff: pkg/csslib/lib/src/tokenizer_base.dart

Issue 23168002: move csslib into dart svn (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 7 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « pkg/csslib/lib/src/tokenizer.dart ('k') | pkg/csslib/lib/src/tokenkind.dart » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: pkg/csslib/lib/src/tokenizer_base.dart
diff --git a/utils/css/tokenizer_base.dart b/pkg/csslib/lib/src/tokenizer_base.dart
similarity index 74%
copy from utils/css/tokenizer_base.dart
copy to pkg/csslib/lib/src/tokenizer_base.dart
index f98f245f910dcbfc2ba12681521bf31d70766dab..f33a3629ceee9f7c465d8677da9d40daefc27e47 100644
--- a/utils/css/tokenizer_base.dart
+++ b/pkg/csslib/lib/src/tokenizer_base.dart
@@ -1,54 +1,58 @@
-// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// Generated by scripts/tokenizer_gen.py.
+part of csslib.parser;
-abstract class TokenSource {
- Token next();
-}
-
-class InterpStack {
- InterpStack next, previous;
- final int quote;
- final bool isMultiline;
- int depth;
-
- InterpStack(this.previous, this.quote, this.isMultiline): depth = -1;
-
- InterpStack pop() {
- return this.previous;
- }
+/** Tokenizer state to support look ahead for Less' nested selectors. */
+class TokenizerState {
+ final int index;
+ final int startIndex;
+ final bool selectorExpression;
- static InterpStack push(InterpStack stack, int quote, bool isMultiline) {
- var newStack = new InterpStack(stack, quote, isMultiline);
- if (stack != null) newStack.previous = stack;
- return newStack;
- }
+ TokenizerState(TokenizerBase base) :
+ this.index = base._index,
+ this.startIndex = base._startIndex,
+ this.selectorExpression = base.selectorExpression;
}
/**
* The base class for our tokenizer. The hand coded parts are in this file, with
* the generated parts in the subclass Tokenizer.
*/
-class CSSTokenizerBase implements TokenSource {
- final SourceFile _source;
+abstract class TokenizerBase {
+ final SourceFile _file;
final bool _skipWhitespace;
- String _text;
+ final String _text;
+
+ /**
+ * Changes tokenization when in a pseudo function expression. If true then
+ * minus signs are handled as operators instead of identifiers.
+ */
+ bool selectorExpression = false;
int _index;
int _startIndex;
- /** Keeps track of string interpolation state. */
- InterpStack _interpStack;
+ static const String _CDATA_START = '<![CDATA[';
+ static const String _CDATA_END = ']]>';
- CSSTokenizerBase(this._source, this._skipWhitespace, [index = 0])
- : this._index = index {
- _text = _source.text;
- }
+ TokenizerBase(this._file, this._text, this._skipWhitespace,
+ [this._index = 0]);
+
+ Token next();
+ int getIdentifierKind();
+
+ /** Snapshot of Tokenizer scanning state. */
+ TokenizerState get mark => new TokenizerState(this);
- abstract Token next();
- abstract int getIdentifierKind();
+ /** Restore Tokenizer scanning state. */
+ void restore(TokenizerState markedData) {
+ _index = markedData.index;
+ _startIndex = markedData.startIndex;
+ selectorExpression = markedData.selectorExpression;
+ }
int _nextChar() {
if (_index < _text.length) {
@@ -88,21 +92,23 @@ class CSSTokenizerBase implements TokenSource {
}
Token _finishToken(int kind) {
- return new Token(kind, _source, _startIndex, _index);
+ return new Token(kind, _file.span(_startIndex, _index));
}
Token _errorToken([String message = null]) {
return new ErrorToken(
- TokenKind.ERROR, _source, _startIndex, _index, message);
+ TokenKind.ERROR, _file.span(_startIndex, _index), message);
}
Token finishWhitespace() {
_index--;
while (_index < _text.length) {
final ch = _text.codeUnitAt(_index++);
- if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) {
+ if (ch == TokenChar.SPACE ||
+ ch == TokenChar.TAB ||
+ ch == TokenChar.RETURN) {
// do nothing
- } else if (ch == 10/*'\n'*/) {
+ } else if (ch == TokenChar.NEWLINE) {
if (!_skipWhitespace) {
return _finishToken(TokenKind.WHITESPACE); // note the newline?
}
@@ -122,7 +128,7 @@ class CSSTokenizerBase implements TokenSource {
Token finishSingleLineComment() {
while (true) {
int ch = _nextChar();
- if (ch == 0 || ch == 10/*'\n'*/ || ch == 13/*'\r'*/) {
+ if (ch == 0 || ch == TokenChar.NEWLINE || ch == TokenChar.RETURN) {
if (_skipWhitespace) {
return next();
} else {
@@ -138,12 +144,12 @@ class CSSTokenizerBase implements TokenSource {
int ch = _nextChar();
if (ch == 0) {
return _errorToken();
- } else if (ch == 42/*'*'*/) {
- if (_maybeEatChar(47/*'/'*/)) {
+ } else if (ch == TokenChar.ASTERISK) {
+ if (_maybeEatChar(TokenChar.SLASH)) {
nesting--;
}
- } else if (ch == 47/*'/'*/) {
- if (_maybeEatChar(42/*'*'*/)) {
+ } else if (ch == TokenChar.SLASH) {
+ if (_maybeEatChar(TokenChar.ASTERISK)) {
nesting++;
}
}
@@ -212,7 +218,7 @@ class CSSTokenizerBase implements TokenSource {
Token finishNumber() {
eatDigits();
- if (_peekChar() == 46/*.*/) {
+ if (_peekChar() == TokenChar.DOT) {
// Handle the case of 1.toString().
_nextChar();
if (TokenizerHelpers.isDigit(_peekChar())) {
@@ -229,8 +235,8 @@ class CSSTokenizerBase implements TokenSource {
Token finishNumberExtra(int kind) {
if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) {
kind = TokenKind.DOUBLE;
- _maybeEatChar(45/*-*/);
- _maybeEatChar(43/*+*/);
+ _maybeEatChar(TokenKind.MINUS);
+ _maybeEatChar(TokenKind.PLUS);
eatDigits();
}
if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) {
@@ -244,20 +250,26 @@ class CSSTokenizerBase implements TokenSource {
Token _makeStringToken(List<int> buf, bool isPart) {
final s = new String.fromCharCodes(buf);
final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING;
- return new LiteralToken(kind, _source, _startIndex, _index, s);
+ return new LiteralToken(kind, _file.span(_startIndex, _index), s);
+ }
+
+ Token makeIEFilter(int start, int end) {
+ var filter = _text.substring(start, end);
+ return new LiteralToken(TokenKind.STRING, _file.span(start, end), filter);
}
Token _makeRawStringToken(bool isMultiline) {
- String s;
+ var s;
if (isMultiline) {
// Skip initial newline in multiline strings
int start = _startIndex + 4;
- if (_source.text[start] == '\n') start++;
- s = _source.text.substring(start, _index - 3);
+ if (_text[start] == '\n') start++;
+ s = _text.substring(start, _index - 3);
} else {
- s = _source.text.substring(_startIndex + 2, _index - 1);
+ s = _text.substring(_startIndex + 2, _index - 1);
}
- return new LiteralToken(TokenKind.STRING, _source, _startIndex, _index, s);
+ return new LiteralToken(TokenKind.STRING,
+ _file.span(_startIndex, _index), s);
}
Token finishMultilineString(int quote) {
@@ -274,11 +286,7 @@ class CSSTokenizerBase implements TokenSource {
buf.add(quote);
}
buf.add(quote);
- } else if (ch == 36/*$*/) {
- // start of string interp
- _interpStack = InterpStack.push(_interpStack, quote, true);
- return _makeStringToken(buf, true);
- } else if (ch == 92/*\*/) {
+ } else if (ch == TokenChar.BACKSLASH) {
var escapeVal = readEscapeSequence();
if (escapeVal == -1) {
return _errorToken("invalid hex escape sequence");
@@ -292,22 +300,10 @@ class CSSTokenizerBase implements TokenSource {
}
Token _finishOpenBrace() {
- if (_interpStack != null) {
- if (_interpStack.depth == -1) {
- _interpStack.depth = 1;
- } else {
- assert(_interpStack.depth >= 0);
- _interpStack.depth += 1;
- }
- }
return _finishToken(TokenKind.LBRACE);
}
Token _finishCloseBrace() {
- if (_interpStack != null) {
- _interpStack.depth -= 1;
- assert(_interpStack.depth >= 0);
- }
return _finishToken(TokenKind.RBRACE);
}
@@ -315,7 +311,7 @@ class CSSTokenizerBase implements TokenSource {
if (_maybeEatChar(quote)) {
if (_maybeEatChar(quote)) {
// skip an initial newline
- _maybeEatChar(10/*'\n'*/);
+ _maybeEatChar(TokenChar.NEWLINE);
return finishMultilineString(quote);
} else {
return _makeStringToken(new List<int>(), false);
@@ -359,13 +355,9 @@ class CSSTokenizerBase implements TokenSource {
int ch = _nextChar();
if (ch == quote) {
return _makeStringToken(buf, false);
- } else if (ch == 36/*$*/) {
- // start of string interp
- _interpStack = InterpStack.push(_interpStack, quote, false);
- return _makeStringToken(buf, true);
} else if (ch == 0) {
return _errorToken();
- } else if (ch == 92/*\*/) {
+ } else if (ch == TokenChar.BACKSLASH) {
var escapeVal = readEscapeSequence();
if (escapeVal == -1) {
return _errorToken("invalid hex escape sequence");
@@ -383,32 +375,30 @@ class CSSTokenizerBase implements TokenSource {
int hexValue;
switch (ch) {
case 110/*n*/:
- return 0x0a/*'\n'*/;
+ return TokenChar.NEWLINE;
case 114/*r*/:
- return 0x0d/*'\r'*/;
+ return TokenChar.RETURN;
case 102/*f*/:
- return 0x0c/*'\f'*/;
+ return TokenChar.FF;
case 98/*b*/:
- return 0x08/*'\b'*/;
+ return TokenChar.BACKSPACE;
case 116/*t*/:
- return 0x09/*'\t'*/;
+ return TokenChar.TAB;
case 118/*v*/:
- return 0x0b/*'\v'*/;
+ return TokenChar.FF;
case 120/*x*/:
hexValue = readHex(2);
break;
case 117/*u*/:
- if (_maybeEatChar(123/*{*/)) {
+ if (_maybeEatChar(TokenChar.LBRACE)) {
hexValue = readHex();
- if (!_maybeEatChar(125/*}*/)) {
+ if (!_maybeEatChar(TokenChar.RBRACE)) {
return -1;
- } else {
- break;
}
} else {
hexValue = readHex(4);
- break;
}
+ break;
default: return ch;
}
@@ -420,7 +410,8 @@ class CSSTokenizerBase implements TokenSource {
if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) {
return hexValue;
} else if (hexValue <= 0x10FFFF){
- world.fatal('unicode values greater than 2 bytes not implemented yet');
+ messages.error('unicode values greater than 2 bytes not implemented yet',
+ _file.span(_startIndex, _startIndex + 1));
return -1;
} else {
return -1;
« no previous file with comments | « pkg/csslib/lib/src/tokenizer.dart ('k') | pkg/csslib/lib/src/tokenkind.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698