Fixes issues in string interpolation tests

frog/tokenizer.g.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 // Generated by scripts/tokenizer_gen.py.
 
 
 /** A generated file that extends the hand coded methods in TokenizerBase. */
 class Tokenizer extends TokenizerBase {
 
   Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
@@ skipping 36 matching lines @@
           return finishHashBang();
         } else {
           return _finishToken(TokenKind.HASH);
         }
       case 36/*$*/:
         if (_maybeEatChar(34/*"*/)) {
           return finishString(34/*"*/);
         } else if (_maybeEatChar(39/*'*/)) {
           return finishString(39/*'*/);
         } else {
-          return finishIdentifier();
+          return finishIdentifier(36/*$*/);
         }
       case 37/*%*/:
         if (_maybeEatChar(61/*=*/)) {
           return _finishToken(TokenKind.ASSIGN_MOD);
         } else {
           return _finishToken(TokenKind.MOD);
         }
       case 38/*&*/:
         if (_maybeEatChar(38/*&*/)) {
           return _finishToken(TokenKind.AND);
@@ skipping 151 matching lines @@
           if (_maybeEatChar(61/*=*/)) {
             return _finishToken(TokenKind.ASSIGN_TRUNCDIV);
           } else {
             return _finishToken(TokenKind.TRUNCDIV);
           }
         } else {
           return _finishToken(TokenKind.BIT_NOT);
         }
       default:
         if (isIdentifierStart(ch)) {
-          return this.finishIdentifier();
+          return this.finishIdentifier(ch);
         } else if (isDigit(ch)) {
           return this.finishNumber();
         } else {
           return _errorToken();
         }
       }
 
   }
 
   int getIdentifierKind() {
@@ skipping 126 matching lines @@
         return TokenKind.IDENTIFIER;
       default: return TokenKind.IDENTIFIER;
     }
   }
 
 }
 
 /** Static helper methods. */
 class TokenizerHelpers {
   
   static bool isIdentifierStart(int c) {

[jimhug, 2011/11/18 17:05:29]

This definition is now incorrect - if you want to follow the spec. I believe
that '$' is a legal identifier start character.  In the tokenizer, you handle
this in the '$' case so I don't think there's a real but in the code, but this
method name should be changed or its behavior should be changed to be correct.

     return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || c == 95/*_*/);
   }
 
   static bool isDigit(int c) {
     return (c >= 48/*0*/ && c <= 57/*9*/);
   }
 
   static bool isHexDigit(int c) {
     return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c <= 70/*F*/));
   }
 
   static bool isWhitespace(int c) {
     return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'*/);
   }
 
   static bool isIdentifierPart(int c) {
+    return (isIdentifierStart(c) || isDigit(c) || c == 36/*$*/);
+  }
+
+  static bool isInterpIdentifierPart(int c) {

[jimhug, 2011/11/18 17:05:29]

FYI - I really hate this -but I can't fault you for just following the spec...

     return (isIdentifierStart(c) || isDigit(c));
   }
 
 
 }