Beginning of CSS parser using frog parsering infrastructure.

frog/css/tokenizer_css.dart

Index: frog/css/tokenizer_css.dart
diff --git a/frog/css/tokenizer_css.dart b/frog/css/tokenizer_css.dart
new file mode 100644
index 0000000000000000000000000000000000000000..2d60515de5df83955785e116a29e1a430c5c26b6
--- /dev/null
+++ b/frog/css/tokenizer_css.dart
@@ -0,0 +1,114 @@
+// Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+class Tokenizer extends TokenizerBase {

[jimhug, 2011/11/09 16:04:03]

It's weird to me that the file is tokenizer_css but the type is Tokenizer.  I'd
probably rename the type to CssTokenizer, but you could also rename the file to
tokenizer.dart

[terry, 2011/11/09 21:56:06]

Done.

+  TokenKind cssTokens;
+
+  Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
+    : super(source, skipWhitespace, index) {
+    cssTokens = new TokenKind();
+  }
+
+  Token next() {
+    // keep track of our starting position
+    _startIndex = _index;
+
+    if (_interpStack != null && _interpStack.depth == 0) {

[nweiz, 2011/11/10 00:04:39]

Unused code

[terry, 2011/11/22 16:40:47]

I expect to be using this in next checkin.
On 2011/11/10 00:04:39, nweiz wrote:

+      var istack = _interpStack;
+      _interpStack = _interpStack.pop();
+/*
+      if (istack.isMultiline) {
+        return finishMultilineStringBody(istack.quote);
+      } else {
+        return finishStringBody(istack.quote);
+      }
+*/
+    }
+
+    int ch;
+    ch = _nextChar();
+    switch(ch) {
+      case 0:
+        return _finishToken(TokenKind.END_OF_FILE);
+      case TokenizerHelpers.WS_SPACE:
+      case TokenizerHelpers.WS_TAB:
+      case TokenizerHelpers.WS_NEWLINE:
+      case TokenizerHelpers.WS_RETURN:
+        return finishWhitespace();
+      case cssTokens.tokens[TokenKind.END_OF_FILE]:
+        return _finishToken(TokenKind.END_OF_FILE);
+      case cssTokens.tokens[TokenKind.AT]:
+        return _finishToken(TokenKind.AT);
+      case cssTokens.tokens[TokenKind.DOT]:
+        return _finishToken(TokenKind.DOT);
+      case cssTokens.tokens[TokenKind.LBRACE]:
+        return _finishToken(TokenKind.LBRACE);
+      case cssTokens.tokens[TokenKind.RBRACE]:
+        return _finishToken(TokenKind.RBRACE);
+      case cssTokens.tokens[TokenKind.HASH]:
+        return _finishToken(TokenKind.HASH);
+      case cssTokens.tokens[TokenKind.COMBINATOR_PLUS]:
+        return _finishToken(TokenKind.COMBINATOR_PLUS);
+      case cssTokens.tokens[TokenKind.COMBINATOR_GREATER]:
+        return _finishToken(TokenKind.COMBINATOR_GREATER);
+      case cssTokens.tokens[TokenKind.COMBINATOR_TILDE]:
+        return _finishToken(TokenKind.COMBINATOR_TILDE);
+      case cssTokens.tokens[TokenKind.ASTERISK]:
+        return _finishToken(TokenKind.ASTERISK);
+      case cssTokens.tokens[TokenKind.NAMESPACE]:
+        return _finishToken(TokenKind.NAMESPACE);
+      case cssTokens.tokens[TokenKind.PSEUDO]:
+        return _finishToken(TokenKind.PSEUDO);
+      case cssTokens.tokens[TokenKind.COMMA]:
+        return _finishToken(TokenKind.COMMA);
+
+      default:
+        if (isIdentifierStart(ch)) {
+          return this.finishIdentifier();
+        } else if (isDigit(ch)) {
+          return this.finishNumber();
+        } else {
+          return _errorToken();
+        }
+    }
+  }
+
+  // TODO(jmesserly): we need a way to emit human readable error messages from
+  // the tokenizer.
+  Token _errorToken() {
+    return _finishToken(TokenKind.ERROR);
+  }
+
+  int getIdentifierKind() {

[nweiz, 2011/11/10 00:04:39]

Not useful, remove

[terry, 2011/11/22 16:40:47]

This will be used real soon with other known idents.

On 2011/11/10 00:04:39, nweiz wrote:

+    return TokenKind.IDENTIFIER;
+  }
+}
+
+/** Static helper methods. */
+class TokenizerHelpers {
+  final static int WS_SPACE = 32;                       // ' '
+  final static int WS_TAB = 9;                          // '\t'
+  final static int WS_NEWLINE = 10;                     // '\n'
+  final static int WS_RETURN = 13;                      // '\r'
+
+  static bool isIdentifierStart(int c) {
+    return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || c == 95/*_*/);

[nweiz, 2011/11/10 00:04:39]

Line length (also below)

[terry, 2011/11/22 16:40:47]

Done.

+  }
+
+  static bool isDigit(int c) {
+    return (c >= 48/*0*/ && c <= 57/*9*/);
+  }
+
+  static bool isHexDigit(int c) {
+    return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c <= 70/*F*/));
+  }
+
+  static bool isWhitespace(int c) {
+    return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'*/);
+  }
+
+  static bool isIdentifierPart(int c) {
+    return (isIdentifierStart(c) || isDigit(c) || c == 45/*-*/);
+  }
+}