[html5lib] implement querySelector/querySelectorAll

pkg/csslib/lib/src/tokenizer.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of csslib.parser;
 
 class Tokenizer extends TokenizerBase {
   /** U+ prefix for unicode characters. */
   final UNICODE_U = 'U'.codeUnitAt(0);
   final UNICODE_LOWER_U = 'u'.codeUnitAt(0);
@@ skipping 23 matching lines @@
       case TokenChar.END_OF_FILE:
         return _finishToken(TokenKind.END_OF_FILE);
       case TokenChar.AT:
         int peekCh = _peekChar();
         if (TokenizerHelpers.isIdentifierStart(peekCh)) {
           var oldIndex = _index;
           var oldStartIndex = _startIndex;
 
           _startIndex = _index;
           ch = _nextChar();
-          Token ident = this.finishIdentifier(ch);
+          Token ident = finishIdentifier();
 
           // Is it a directive?
           int tokId = TokenKind.matchDirectives(_text, _startIndex,
               _index - _startIndex);
           if (tokId == -1) {
             // No, is it a margin directive?
             tokId = TokenKind.matchMarginDirectives(_text, _startIndex,
                 _index - _startIndex);
           }
 
@@ skipping 39 matching lines @@
           // ]]>
           return next();
         }
         return _finishToken(TokenKind.RBRACK);
       case TokenChar.HASH:
         return _finishToken(TokenKind.HASH);
       case TokenChar.PLUS:
         if (maybeEatDigit()) return finishNumber();
         return _finishToken(TokenKind.PLUS);
       case TokenChar.MINUS:
-        if (selectorExpression || unicodeRange) {
+        if (inSelectorExpression || unicodeRange) {
           // If parsing in pseudo function expression then minus is an operator
           // not part of identifier e.g., interval value range (e.g. U+400-4ff)
           // or minus operator in selector expression.
           return _finishToken(TokenKind.MINUS);
         } else if (maybeEatDigit()) {
           return finishNumber();
         } else if (TokenizerHelpers.isIdentifierStart(ch)) {
-          return this.finishIdentifier(ch);
+          return finishIdentifier();
         }
         return _finishToken(TokenKind.MINUS);
       case TokenChar.GREATER:
         return _finishToken(TokenKind.GREATER);
       case TokenChar.TILDE:
         if (_maybeEatChar(TokenChar.EQUALS)) {
           return _finishToken(TokenKind.INCLUDES);          // ~=
         }
         return _finishToken(TokenKind.TILDE);
       case TokenChar.ASTERISK:
         if (_maybeEatChar(TokenChar.EQUALS)) {
           return _finishToken(TokenKind.SUBSTRING_MATCH);   // *=
         }
         return _finishToken(TokenKind.ASTERISK);
       case TokenChar.AMPERSAND:
         return _finishToken(TokenKind.AMPERSAND);
       case TokenChar.NAMESPACE:
+        if (_maybeEatChar(TokenChar.EQUALS)) {
+          return _finishToken(TokenKind.DASH_MATCH);      // |=
+        }
         return _finishToken(TokenKind.NAMESPACE);
       case TokenChar.COLON:
         return _finishToken(TokenKind.COLON);
       case TokenChar.COMMA:
         return _finishToken(TokenKind.COMMA);
       case TokenChar.SEMICOLON:
         return _finishToken(TokenKind.SEMICOLON);
       case TokenChar.PERCENT:
         return _finishToken(TokenKind.PERCENT);
       case TokenChar.SINGLE_QUOTE:
@@ skipping 15 matching lines @@
               _maybeEatChar(CDATA_NAME[3]) &&
               _maybeEatChar(CDATA_NAME[4]) &&
               _maybeEatChar(TokenChar.LBRACK)) {
             // <![CDATA[
             return next();
           }
         }
         return _finishToken(TokenKind.LESS);
       case TokenChar.EQUALS:
         return _finishToken(TokenKind.EQUALS);
-      case TokenChar.OR:
-        if (_maybeEatChar(TokenChar.EQUALS)) {
-          return _finishToken(TokenKind.DASH_MATCH);      // |=
-        }
-        return _finishToken(TokenKind.OR);
       case TokenChar.CARET:
         if (_maybeEatChar(TokenChar.EQUALS)) {
           return _finishToken(TokenKind.PREFIX_MATCH);    // ^=
         }
         return _finishToken(TokenKind.CARET);
       case TokenChar.DOLLAR:
         if (_maybeEatChar(TokenChar.EQUALS)) {
           return _finishToken(TokenKind.SUFFIX_MATCH);    // $=
         }
         return _finishToken(TokenKind.DOLLAR);
       case TokenChar.BANG:
-        Token tok = finishIdentifier(ch);
+        Token tok = finishIdentifier();
         return (tok == null) ? _finishToken(TokenKind.BANG) : tok;
-      case TokenChar.BACKSLASH:
-        return _finishToken(TokenKind.BACKSLASH);
       default:
+        // TODO(jmesserly): this is used for IE8 detection; I'm not sure it's
+        // appropriate outside of a few specific places; certainly shouldn't
+        // be parsed in selectors.
+        if (!inSelector && ch == TokenChar.BACKSLASH) {
+          return _finishToken(TokenKind.BACKSLASH);
+        }
+
         if (unicodeRange) {
           // Three types of unicode ranges:
           //   - single code point (e.g. U+416)
           //   - interval value range (e.g. U+400-4ff)
           //   - range where trailing ‘?’ characters imply ‘any digit value’
           //   (e.g. U+4??)
           if (maybeEatHexDigit()) {
             var t = finishHexNumber();
             // Any question marks then it's a HEX_RANGE not HEX_NUMBER.
             if (maybeEatQuestionMark()) finishUnicodeRange();
             return t;
           } else if (maybeEatQuestionMark()) {
             // HEX_RANGE U+N???
             return finishUnicodeRange();
           } else {
             return _errorToken();
           }
         } else if ((ch == UNICODE_U || ch == UNICODE_LOWER_U) &&
             (_peekChar() == UNICODE_PLUS)) {
           // Unicode range: U+uNumber[-U+uNumber]
           //   uNumber = 0..10FFFF
           _nextChar();                                // Skip +
           _startIndex = _index;                       // Starts at the number
           return _finishToken(TokenKind.UNICODE_RANGE);
         } else if (varDef(ch)) {
           return _finishToken(TokenKind.VAR_DEFINITION);
         } else if (varUsage(ch)) {
           return _finishToken(TokenKind.VAR_USAGE);
         } else if (TokenizerHelpers.isIdentifierStart(ch)) {
-          return finishIdentifier(ch);
+          return finishIdentifier();
         } else if (TokenizerHelpers.isDigit(ch)) {
           return finishNumber();
         }
         return _errorToken();
     }
   }
 
   bool varDef(int ch) {
     return ch == 'v'.codeUnitAt(0) && _maybeEatChar('a'.codeUnitAt(0)) &&
         _maybeEatChar('r'.codeUnitAt(0)) && _maybeEatChar('-'.codeUnitAt(0));
   }
 
   bool varUsage(int ch) {
     return ch == 'v'.codeUnitAt(0) && _maybeEatChar('a'.codeUnitAt(0)) &&
         _maybeEatChar('r'.codeUnitAt(0)) && (_peekChar() == '-'.codeUnitAt(0));
   }
 
   Token _errorToken([String message = null]) {
     return _finishToken(TokenKind.ERROR);
   }
 
   int getIdentifierKind() {
     // Is the identifier a unit type?
-    int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
+    int tokId = -1;
+
+    // Don't match units in selectors or selector expressions.
+    if (!inSelectorExpression && !inSelector) {
+      tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
+    }
     if (tokId == -1) {
       tokId = (_text.substring(_startIndex, _index) == '!important') ?
           TokenKind.IMPORTANT : -1;
     }
 
     return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
   }
 
-  // Need to override so CSS version of isIdentifierPart is used.
-  Token finishIdentifier(int ch) {
+  Token finishIdentifier() {
+    // If we encounter an escape sequence, remember it so we can post-process
+    // to unescape.
+    bool hasEscapedChars = false;
+    var chars = [];
+
+    // backup so we can start with the first character
+    int validateFrom = _index;
+    _index = _startIndex;
     while (_index < _text.length) {
-      // If parsing in pseudo function expression then minus is an operator
-      // not part of identifier.
-      var isIdentifier = selectorExpression
-          ? TokenizerHelpers.isIdentifierPartExpr(_text.codeUnitAt(_index))
-          : TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index));
-      if (!isIdentifier) {
-          break;
+      int ch = _text.codeUnitAt(_index);
+
+      // If the previous character was "\" we need to escape. T
+      // http://www.w3.org/TR/CSS21/syndata.html#characters
+      // if followed by hexadecimal digits, create the appropriate character.
+      // otherwise, include the character in the identifier and don't treat it
+      // specially.
+      if (ch == 92/*\*/) {
+        int startHex = ++_index;
+        eatHexDigits(startHex + 6);
+        if (_index != startHex) {
+          // Parse the hex digits and add that character.
+          chars.add(int.parse('0x' + _text.substring(startHex, _index)));
+
+          if (_index == _text.length) break;
+
+          // if we stopped the hex because of a whitespace char, skip it
+          ch = _text.codeUnitAt(_index);
+          if (_index - startHex != 6 &&
+              (ch == TokenChar.SPACE || ch == TokenChar.TAB ||
+              ch == TokenChar.RETURN || ch == TokenChar.NEWLINE)) {
+            _index++;
+          }
+        } else {
+          // not a digit, just add the next character literally
+          if (_index == _text.length) break;
+          chars.add(_text.codeUnitAt(_index++));
+        }
+      } else if (_index < validateFrom || (inSelectorExpression
+          ? TokenizerHelpers.isIdentifierPartExpr(ch)
+          : TokenizerHelpers.isIdentifierPart(ch))) {
+        chars.add(ch);
+        _index++;
       } else {
-        _index += 1;
+        // Not an identifier or escaped character.
+        break;
       }
     }
 
-    int kind = getIdentifierKind();
-    if (kind == TokenKind.IDENTIFIER) {
-      return _finishToken(TokenKind.IDENTIFIER);
-    } else {
-      return _finishToken(kind);
-    }
-  }
+    var span = _file.span(_startIndex, _index);
+    var text = new String.fromCharCodes(chars);
 
-  Token finishImportant() {
-
+    return new IdentifierToken(text, getIdentifierKind(), span);
   }
 
   Token finishNumber() {
     eatDigits();
 
     if (_peekChar() == 46/*.*/) {
       // Handle the case of 1.toString().
       _nextChar();
       if (TokenizerHelpers.isDigit(_peekChar())) {
         eatDigits();
         return _finishToken(TokenKind.DOUBLE);
       } else {
         _index -= 1;
       }
     }
 
     return _finishToken(TokenKind.INTEGER);
   }
 
   bool maybeEatDigit() {
     if (_index < _text.length
         && TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) {
       _index += 1;
       return true;
     }
     return false;
   }
 
   Token finishHexNumber() {
-    eatHexDigits();
+    eatHexDigits(_text.length);
     return _finishToken(TokenKind.HEX_INTEGER);
   }
 
-  void eatHexDigits() {
-    while (_index < _text.length) {
+  void eatHexDigits(int end) {
+    end = math.min(end, _text.length);
+    while (_index < end) {
      if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
        _index += 1;
      } else {
        return;
      }
     }
   }
 
   bool maybeEatHexDigit() {
     if (_index < _text.length
@@ skipping 74 matching lines @@
         || (c >= 65/*A*/ && c <= 70/*F*/));
   }
 
   static bool isIdentifierPart(int c) {
     return isIdentifierPartExpr(c) || c == 45 /*-*/;
   }
 
   /** Pseudo function expressions identifiers can't have a minus sign. */
   static bool isIdentifierStartExpr(int c) {
     return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
-        c == 95/*_*/);
+        // Note: Unicode 10646 chars U+00A0 or higher are allowed, see:
+        // http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier
+        // http://www.w3.org/TR/CSS21/syndata.html#characters
+        // Also, escaped character should be allowed.
+        c == 95/*_*/ || c >= 0xA0 || c == 92/*\*/);
   }
 
   /** Pseudo function expressions identifiers can't have a minus sign. */
   static bool isIdentifierPartExpr(int c) {
     return (isIdentifierStartExpr(c) || isDigit(c));
   }
 }