move csslib into dart svn

pkg/csslib/lib/src/tokenizer_base.dart

-// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 // Generated by scripts/tokenizer_gen.py.
 
+part of csslib.parser;
 
-abstract class TokenSource {
-  Token next();
-}
+/** Tokenizer state to support look ahead for Less' nested selectors. */
+class TokenizerState {
+  final int index;
+  final int startIndex;
+  final bool selectorExpression;
 
-class InterpStack {
-  InterpStack next, previous;
-  final int quote;
-  final bool isMultiline;
-  int depth;
-
-  InterpStack(this.previous, this.quote, this.isMultiline): depth = -1;
-
-  InterpStack pop() {
-    return this.previous;
-  }
-
-  static InterpStack push(InterpStack stack, int quote, bool isMultiline) {
-    var newStack = new InterpStack(stack, quote, isMultiline);
-    if (stack != null) newStack.previous = stack;
-    return newStack;
-  }
+  TokenizerState(TokenizerBase base) :
+      this.index = base._index,
+      this.startIndex = base._startIndex,
+      this.selectorExpression = base.selectorExpression;
 }
 
 /**
  * The base class for our tokenizer. The hand coded parts are in this file, with
  * the generated parts in the subclass Tokenizer.
  */
-class CSSTokenizerBase implements TokenSource {
-  final SourceFile _source;
+abstract class TokenizerBase {
+  final SourceFile _file;
   final bool _skipWhitespace;
-  String _text;
+  final String _text;
+
+  /**
+   * Changes tokenization when in a pseudo function expression.  If true then
+   * minus signs are handled as operators instead of identifiers.
+   */
+  bool selectorExpression = false;
 
   int _index;
   int _startIndex;
 
-  /** Keeps track of string interpolation state. */
-  InterpStack _interpStack;
+  static const String _CDATA_START = '<![CDATA[';
+  static const String _CDATA_END = ']]>';
 
-  CSSTokenizerBase(this._source, this._skipWhitespace, [index = 0])
-      : this._index = index {
-    _text = _source.text;
+  TokenizerBase(this._file, this._text, this._skipWhitespace,
+      [this._index = 0]);
+
+  Token next();
+  int getIdentifierKind();
+
+  /** Snapshot of Tokenizer scanning state. */
+  TokenizerState get mark => new TokenizerState(this);
+
+  /** Restore Tokenizer scanning state. */
+  void restore(TokenizerState markedData) {
+    _index = markedData.index;
+    _startIndex = markedData.startIndex;
+    selectorExpression = markedData.selectorExpression;
   }
 
-  abstract Token next();
-  abstract int getIdentifierKind();
-
   int _nextChar() {
     if (_index < _text.length) {
       return _text.codeUnitAt(_index++);
     } else {
       return 0;
     }
   }
 
   int _peekChar() {
     if (_index < _text.length) {
@@ skipping 18 matching lines @@
 
   String _tokenText() {
     if (_index < _text.length) {
       return _text.substring(_startIndex, _index);
     } else {
       return _text.substring(_startIndex, _text.length);
     }
   }
 
   Token _finishToken(int kind) {
-    return new Token(kind, _source, _startIndex, _index);
+    return new Token(kind, _file.span(_startIndex, _index));
   }
 
   Token _errorToken([String message = null]) {
     return new ErrorToken(
-        TokenKind.ERROR, _source, _startIndex, _index, message);
+        TokenKind.ERROR, _file.span(_startIndex, _index), message);
   }
 
   Token finishWhitespace() {
     _index--;
     while (_index < _text.length) {
       final ch = _text.codeUnitAt(_index++);
-      if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) {
+      if (ch == TokenChar.SPACE ||
+          ch == TokenChar.TAB ||
+          ch == TokenChar.RETURN) {
         // do nothing
-      } else if (ch == 10/*'\n'*/) {
+      } else if (ch == TokenChar.NEWLINE) {
         if (!_skipWhitespace) {
           return _finishToken(TokenKind.WHITESPACE); // note the newline?
         }
       } else {
         _index--;
         if (_skipWhitespace) {
           return next();
         } else {
           return _finishToken(TokenKind.WHITESPACE);
         }
       }
 
     }
     return _finishToken(TokenKind.END_OF_FILE);
   }
 
   Token finishSingleLineComment() {
     while (true) {
       int ch = _nextChar();
-      if (ch == 0 || ch == 10/*'\n'*/ || ch == 13/*'\r'*/) {
+      if (ch == 0 || ch == TokenChar.NEWLINE || ch == TokenChar.RETURN) {
         if (_skipWhitespace) {
           return next();
         } else {
           return _finishToken(TokenKind.COMMENT);
         }
       }
     }
   }
 
   Token finishMultiLineComment() {
     int nesting = 1;
     do {
       int ch = _nextChar();
       if (ch == 0) {
         return _errorToken();
-      } else if (ch == 42/*'*'*/) {
-        if (_maybeEatChar(47/*'/'*/)) {
+      } else if (ch == TokenChar.ASTERISK) {
+        if (_maybeEatChar(TokenChar.SLASH)) {
           nesting--;
         }
-      } else if (ch == 47/*'/'*/) {
-        if (_maybeEatChar(42/*'*'*/)) {
+      } else if (ch == TokenChar.SLASH) {
+        if (_maybeEatChar(TokenChar.ASTERISK)) {
           nesting++;
         }
       }
     } while (nesting > 0);
 
     if (_skipWhitespace) {
       return next();
     } else {
       return _finishToken(TokenKind.COMMENT);
     }
@@ skipping 48 matching lines @@
       result = (result * 16) + digit;
       _index++;
     }
 
     return result;
   }
 
   Token finishNumber() {
     eatDigits();
 
-    if (_peekChar() == 46/*.*/) {
+    if (_peekChar() == TokenChar.DOT) {
       // Handle the case of 1.toString().
       _nextChar();
       if (TokenizerHelpers.isDigit(_peekChar())) {
         eatDigits();
         return finishNumberExtra(TokenKind.DOUBLE);
       } else {
         _index--;
       }
     }
 
     return finishNumberExtra(TokenKind.INTEGER);
   }
 
   Token finishNumberExtra(int kind) {
     if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) {
       kind = TokenKind.DOUBLE;
-      _maybeEatChar(45/*-*/);
-      _maybeEatChar(43/*+*/);
+      _maybeEatChar(TokenKind.MINUS);
+      _maybeEatChar(TokenKind.PLUS);
       eatDigits();
     }
     if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) {
       _nextChar();
       return _errorToken("illegal character in number");
     }
 
     return _finishToken(kind);
   }
 
   Token _makeStringToken(List<int> buf, bool isPart) {
     final s = new String.fromCharCodes(buf);
     final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING;
-    return new LiteralToken(kind, _source, _startIndex, _index, s);
+    return new LiteralToken(kind, _file.span(_startIndex, _index), s);
+  }
+
+  Token makeIEFilter(int start, int end) {
+    var filter = _text.substring(start, end);
+    return new LiteralToken(TokenKind.STRING, _file.span(start, end), filter);
   }
 
   Token _makeRawStringToken(bool isMultiline) {
-    String s;
+    var s;
     if (isMultiline) {
       // Skip initial newline in multiline strings
       int start = _startIndex + 4;
-      if (_source.text[start] == '\n') start++;
-      s = _source.text.substring(start, _index - 3);
+      if (_text[start] == '\n') start++;
+      s = _text.substring(start, _index - 3);
     } else {
-      s = _source.text.substring(_startIndex + 2, _index - 1);
+      s = _text.substring(_startIndex + 2, _index - 1);
     }
-    return new LiteralToken(TokenKind.STRING, _source, _startIndex, _index, s);
+    return new LiteralToken(TokenKind.STRING,
+        _file.span(_startIndex, _index), s);
   }
 
   Token finishMultilineString(int quote) {
     var buf = <int>[];
     while (true) {
       int ch = _nextChar();
       if (ch == 0) {
         return _errorToken();
       } else if (ch == quote) {
         if (_maybeEatChar(quote)) {
           if (_maybeEatChar(quote)) {
             return _makeStringToken(buf, false);
           }
           buf.add(quote);
         }
         buf.add(quote);
-      } else if (ch == 36/*$*/) {
-        // start of string interp
-        _interpStack = InterpStack.push(_interpStack, quote, true);
-        return _makeStringToken(buf, true);
-      } else if (ch == 92/*\*/) {
+      } else if (ch == TokenChar.BACKSLASH) {
         var escapeVal = readEscapeSequence();
         if (escapeVal == -1) {
           return _errorToken("invalid hex escape sequence");
         } else {
           buf.add(escapeVal);
         }
       } else {
         buf.add(ch);
       }
     }
   }
 
   Token _finishOpenBrace() {
-    if (_interpStack != null) {
-      if (_interpStack.depth == -1) {
-        _interpStack.depth = 1;
-      } else {
-        assert(_interpStack.depth >= 0);
-        _interpStack.depth += 1;
-      }
-    }
     return _finishToken(TokenKind.LBRACE);
   }
 
   Token _finishCloseBrace() {
-    if (_interpStack != null) {
-      _interpStack.depth -= 1;
-      assert(_interpStack.depth >= 0);
-    }
     return _finishToken(TokenKind.RBRACE);
   }
 
   Token finishString(int quote) {
     if (_maybeEatChar(quote)) {
       if (_maybeEatChar(quote)) {
         // skip an initial newline
-        _maybeEatChar(10/*'\n'*/);
+        _maybeEatChar(TokenChar.NEWLINE);
         return finishMultilineString(quote);
       } else {
         return _makeStringToken(new List<int>(), false);
       }
     }
     return finishStringBody(quote);
   }
 
   Token finishRawString(int quote) {
     if (_maybeEatChar(quote)) {
@@ skipping 23 matching lines @@
       }
     }
   }
 
   Token finishStringBody(int quote) {
     var buf = new List<int>();
     while (true) {
       int ch = _nextChar();
       if (ch == quote) {
         return _makeStringToken(buf, false);
-      } else if (ch == 36/*$*/) {
-        // start of string interp
-        _interpStack = InterpStack.push(_interpStack, quote, false);
-        return _makeStringToken(buf, true);
       } else if (ch == 0) {
         return _errorToken();
-      } else if (ch == 92/*\*/) {
+      } else if (ch == TokenChar.BACKSLASH) {
         var escapeVal = readEscapeSequence();
         if (escapeVal == -1) {
           return _errorToken("invalid hex escape sequence");
         } else {
           buf.add(escapeVal);
         }
       } else {
         buf.add(ch);
       }
     }
   }
 
   int readEscapeSequence() {
     final ch = _nextChar();
     int hexValue;
     switch (ch) {
       case 110/*n*/:
-        return 0x0a/*'\n'*/;
+        return TokenChar.NEWLINE;
       case 114/*r*/:
-        return 0x0d/*'\r'*/;
+        return TokenChar.RETURN;
       case 102/*f*/:
-        return 0x0c/*'\f'*/;
+        return TokenChar.FF;
       case 98/*b*/:
-        return 0x08/*'\b'*/;
+        return TokenChar.BACKSPACE;
       case 116/*t*/:
-        return 0x09/*'\t'*/;
+        return TokenChar.TAB;
       case 118/*v*/:
-        return 0x0b/*'\v'*/;
+        return TokenChar.FF;
       case 120/*x*/:
         hexValue = readHex(2);
         break;
       case 117/*u*/:
-        if (_maybeEatChar(123/*{*/)) {
+        if (_maybeEatChar(TokenChar.LBRACE)) {
           hexValue = readHex();
-          if (!_maybeEatChar(125/*}*/)) {
+          if (!_maybeEatChar(TokenChar.RBRACE)) {
             return -1;
-          } else {
-            break;
           }
         } else {
           hexValue = readHex(4);
-          break;
         }
+        break;
       default: return ch;
     }
 
     if (hexValue == -1) return -1;
 
     // According to the Unicode standard the high and low surrogate halves
     // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF
     // are not legal Unicode values.
     if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) {
       return hexValue;
     } else if (hexValue <= 0x10FFFF){
-      world.fatal('unicode values greater than 2 bytes not implemented yet');
+      messages.error('unicode values greater than 2 bytes not implemented yet',
+          _file.span(_startIndex, _startIndex + 1));
       return -1;
     } else {
       return -1;
     }
   }
 
   Token finishDot() {
     if (TokenizerHelpers.isDigit(_peekChar())) {
       eatDigits();
       return finishNumberExtra(TokenKind.DOUBLE);
@@ skipping 11 matching lines @@
     }
     int kind = getIdentifierKind();
     if (kind == TokenKind.IDENTIFIER) {
       return _finishToken(TokenKind.IDENTIFIER);
     } else {
       return _finishToken(kind);
     }
   }
 }