Support for nested comments

frog/tokenizer.dart

Index: frog/tokenizer.dart
diff --git a/frog/tokenizer.dart b/frog/tokenizer.dart
index 1fb86e1a455fcf28a85d986b0268d5739833aa1d..a944cf93b5bcd870ded107dd2d417c6e071950cf 100644
--- a/frog/tokenizer.dart
+++ b/frog/tokenizer.dart
@@ -4,8 +4,6 @@
 // Generated by scripts/tokenizer_gen.py.
 
 
-/**
- */
 interface TokenSource {
   Token next();
 }
@@ -159,21 +157,27 @@ class TokenizerBase extends TokenizerHelpers implements TokenSource {
   }
 
   Token finishMultiLineComment() {
-    while (true) {
+    int nesting = 1;
+    do {
       int ch = _nextChar();
       if (ch == 0) {
         return _finishToken(TokenKind.INCOMPLETE_COMMENT);
       } else if (ch == 42/*'*'*/) {
         if (_maybeEatChar(47/*'/'*/)) {
-          if (_skipWhitespace) {
-            return next();
-          } else {
-            return _finishToken(TokenKind.COMMENT);
-          }
+          nesting--;
+        }
+      } else if (ch == 47/*'/'*/) {
+        if (_maybeEatChar(42/*'*'*/)) {
+          nesting++;
         }
       }
+    } while (nesting > 0);
+
+    if (_skipWhitespace) {
+      return next();
+    } else {
+      return _finishToken(TokenKind.COMMENT);
     }
-    return _errorToken();

[jimhug, 2012/01/05 18:32:29]

Nice to see this cleaned up as part of the change.

   }
 
   void eatDigits() {