[html5lib] implement querySelector/querySelectorAll

pkg/csslib/lib/src/tokenizer_base.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 // Generated by scripts/tokenizer_gen.py.
 
 part of csslib.parser;
 
 /** Tokenizer state to support look ahead for Less' nested selectors. */
 class TokenizerState {
   final int index;
   final int startIndex;
-  final bool selectorExpression;
+  final bool inSelectorExpression;
+  final bool inSelector;
 
   TokenizerState(TokenizerBase base) :
-      this.index = base._index,
-      this.startIndex = base._startIndex,
-      this.selectorExpression = base.selectorExpression;
+      index = base._index,
+      startIndex = base._startIndex,
+      inSelectorExpression = base.inSelectorExpression,
+      inSelector = base.inSelector;
 }
 
 /**
  * The base class for our tokenizer. The hand coded parts are in this file, with
  * the generated parts in the subclass Tokenizer.
  */
 abstract class TokenizerBase {
   final SourceFile _file;
-  final bool _skipWhitespace;
   final String _text;
 
+  bool _skipWhitespace;
+
   /**
    * Changes tokenization when in a pseudo function expression.  If true then
    * minus signs are handled as operators instead of identifiers.
    */
-  bool selectorExpression = false;
+  bool inSelectorExpression = false;
+
+  /**
+   * Changes tokenization when in selectors. If true, it prevents identifiers
+   * from being treated as units. This would break things like ":lang(fr)" or
+   * the HTML (unknown) tag name "px", which is legal to use in a selector.
+   */
+  // TODO(jmesserly): is this a problem elsewhere? "fr" for example will be
+  // processed as a "fraction" unit token, preventing it from working in
+  // places where an identifier is expected. This was breaking selectors like:
+  //     :lang(fr)
+  // The assumption that "fr" always means fraction (and similar issue with
+  // other units) doesn't seem valid. We probably should defer this
+  // analysis until we reach places in the parser where units are expected.
+  // I'm not sure this is tokenizing as described in the specs:
+  //     http://dev.w3.org/csswg/css-syntax/
+  //     http://dev.w3.org/csswg/selectors4/
+  bool inSelector = false;
 
   int _index;
   int _startIndex;
 
   static const String _CDATA_START = '<![CDATA[';
   static const String _CDATA_END = ']]>';
 
   TokenizerBase(this._file, this._text, this._skipWhitespace,
       [this._index = 0]);
 
   Token next();
   int getIdentifierKind();
 
   /** Snapshot of Tokenizer scanning state. */
   TokenizerState get mark => new TokenizerState(this);
 
   /** Restore Tokenizer scanning state. */
   void restore(TokenizerState markedData) {
     _index = markedData.index;
     _startIndex = markedData.startIndex;
-    selectorExpression = markedData.selectorExpression;
+    inSelectorExpression = markedData.inSelectorExpression;
+    inSelector = markedData.inSelector;
   }
 
   int _nextChar() {
     if (_index < _text.length) {
       return _text.codeUnitAt(_index++);
     } else {
       return 0;
     }
   }
 
@@ skipping 53 matching lines @@
           return next();
         } else {
           return _finishToken(TokenKind.WHITESPACE);
         }
       }
 
     }
     return _finishToken(TokenKind.END_OF_FILE);
   }
 
-  Token finishSingleLineComment() {
-    while (true) {
-      int ch = _nextChar();
-      if (ch == 0 || ch == TokenChar.NEWLINE || ch == TokenChar.RETURN) {
-        if (_skipWhitespace) {
-          return next();
-        } else {
-          return _finishToken(TokenKind.COMMENT);
-        }
-      }
-    }
-  }
-
   Token finishMultiLineComment() {
     int nesting = 1;
     do {
       int ch = _nextChar();
       if (ch == 0) {
         return _errorToken();
       } else if (ch == TokenChar.ASTERISK) {
         if (_maybeEatChar(TokenChar.SLASH)) {
           nesting--;
         }
@@ skipping 268 matching lines @@
   }
 
   Token finishDot() {
     if (TokenizerHelpers.isDigit(_peekChar())) {
       eatDigits();
       return finishNumberExtra(TokenKind.DOUBLE);
     } else {
       return _finishToken(TokenKind.DOT);
     }
   }
-
-  Token finishIdentifier(int ch) {
-    while (_index < _text.length) {
-      if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) {
-        _index--;
-        break;
-      }
-    }
-    int kind = getIdentifierKind();
-    if (kind == TokenKind.IDENTIFIER) {
-      return _finishToken(TokenKind.IDENTIFIER);
-    } else {
-      return _finishToken(kind);
-    }
-  }
 }