Support for nested comments

frog/tokenizer.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 // Generated by scripts/tokenizer_gen.py.
 
 
-/**
- */
 interface TokenSource {
   Token next();
 }
 
 class InterpStack {
   InterpStack next, previous;
   final int quote;
   final bool isMultiline;
   int depth;
 
@@ skipping 133 matching lines @@
         if (_skipWhitespace) {
           return next();
         } else {
           return _finishToken(TokenKind.COMMENT);
         }
       }
     }
   }
 
   Token finishMultiLineComment() {
-    while (true) {
+    int nesting = 1;
+    do {
       int ch = _nextChar();
       if (ch == 0) {
         return _finishToken(TokenKind.INCOMPLETE_COMMENT);
       } else if (ch == 42/*'*'*/) {
         if (_maybeEatChar(47/*'/'*/)) {
-          if (_skipWhitespace) {
-            return next();
-          } else {
-            return _finishToken(TokenKind.COMMENT);
-          }
+          nesting--;
+        }
+      } else if (ch == 47/*'/'*/) {
+        if (_maybeEatChar(42/*'*'*/)) {
+          nesting++;
         }
       }
+    } while (nesting > 0);
+
+    if (_skipWhitespace) {
+      return next();
+    } else {
+      return _finishToken(TokenKind.COMMENT);
     }
-    return _errorToken();

[jimhug, 2012/01/05 18:32:29]

Nice to see this cleaned up as part of the change.

   }
 
   void eatDigits() {
     while (_index < _text.length) {
       if (isDigit(_text.charCodeAt(_index))) {
         _index++;
       } else {
         return;
       }
     }
@@ skipping 227 matching lines @@
       }
     }
     int kind = getIdentifierKind();
     if (kind == TokenKind.IDENTIFIER) {
       return _finishToken(TokenKind.IDENTIFIER);
     } else {
       return _finishToken(kind);
     }
   }
 }