Various scanner fixes:

dart/frog/leg/scanner/scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
  */
 class AbstractScanner<T> implements Scanner {
   // TODO(ahe): following makes frog happy.
   abstract int advance();
   abstract int nextByte();
   abstract int peek();
   abstract int select(int choice, String yes, String no);
   abstract void appendStringToken(int kind, String value);
   abstract void appendByteStringToken(int kind, T value);
   abstract void appendKeywordToken(Keyword keyword);
   abstract void appendWhiteSpace(int next);
   abstract void appendEofToken();
-  abstract T asciiString(int start);
+  abstract T asciiString(int start, int offset);
   abstract T utf8String(int start, int offset);
   abstract Token firstToken();
   abstract void beginToken();
   abstract void addToCharOffset(int offset);
   abstract int get charOffset();
   abstract int get byteOffset();
   abstract void appendBeginGroup(int kind, String value);
-  abstract void appendEndGroup(int kind, String value, int openKind);
+  abstract int appendEndGroup(int kind, String value, int openKind);
   abstract void appendGtGt(int kind, String value);
   abstract void appendGtGtGt(int kind, String value);
+  abstract void discardOpenLt();
 
   // TODO(ahe): Move this class to implementation.
 
   Token tokenize() {
     int next = advance();
-    while (next != $EOF) {
+    while (next !== $EOF) {
       next = bigSwitch(next);
     }
     appendEofToken();
     return firstToken();
   }
 
   int bigSwitch(int next) {
     beginToken();
     if (next === $TAB || next === $LF || next === $CR || next === $SPACE) {
       appendWhiteSpace(next);
@@ skipping 68 matching lines @@
     if (next === $HASH) {
       return tokenizeTag(next);
     }
 
     if (next === $LPAREN) {
       appendBeginGroup(LPAREN_TOKEN, "(");
       return advance();
     }
 
     if (next === $RPAREN) {
-      appendEndGroup(RPAREN_TOKEN, ")", LPAREN_TOKEN);
-      return advance();
+      return appendEndGroup(RPAREN_TOKEN, ")", LPAREN_TOKEN);
     }
 
     if (next === $COMMA) {
       appendStringToken(COMMA_TOKEN, ",");
       return advance();
     }
 
     if (next === $COLON) {
       appendStringToken(COLON_TOKEN, ":");
       return advance();
     }
 
     if (next === $SEMICOLON) {
       appendStringToken(SEMICOLON_TOKEN, ";");
+      discardOpenLt();
       return advance();
     }
 
     if (next === $QUESTION) {
       appendStringToken(QUESTION_TOKEN, "?");
       return advance();
     }
 
     if (next === $CLOSE_SQUARE_BRACKET) {
-      appendEndGroup(CLOSE_SQUARE_BRACKET_TOKEN, "]",
-                     OPEN_SQUARE_BRACKET_TOKEN);
-      return advance();
+      return appendEndGroup(CLOSE_SQUARE_BRACKET_TOKEN, "]",
+                            OPEN_SQUARE_BRACKET_TOKEN);
     }
 
     if (next === $BACKPING) {
       appendStringToken(BACKPING_TOKEN, "`");
       return advance();
     }
 
     if (next === $OPEN_CURLY_BRACKET) {
       appendBeginGroup(OPEN_CURLY_BRACKET_TOKEN, "{");
       return advance();
     }
 
     if (next === $CLOSE_CURLY_BRACKET) {
-      appendEndGroup(CLOSE_CURLY_BRACKET_TOKEN, "}", OPEN_CURLY_BRACKET_TOKEN);
-      return advance();
+      return appendEndGroup(CLOSE_CURLY_BRACKET_TOKEN, "}",
+                            OPEN_CURLY_BRACKET_TOKEN);
     }
 
     if (next === $SLASH) {
       return tokenizeSlashOrComment(next);
     }
 
     if (next === $AT) {
       return tokenizeRawString(next);
     }
 
@@ skipping 178 matching lines @@
           return advance();
         } else {
           appendGtGtGt(GT_TOKEN, ">>>");
           return next;
         }
       } else {
         appendGtGt(GT_TOKEN, ">>");
         return next;
       }
     } else {
-      appendEndGroup(GT_TOKEN, ">", LT_TOKEN);
+      appendGt(GT_TOKEN, ">");
       return next;
     }
   }
 
   int tokenizeLessThan(int next) {
     // < <= << <<=
     next = advance();
     if ($EQ === next) {
       appendStringToken(LT_EQ_TOKEN, "<=");
       return advance();
     } else if ($LT === next) {
       return select($EQ, "<<=", "<<");
     } else {
       appendBeginGroup(LT_TOKEN, "<");
       return next;
     }
   }
 
   int tokenizeNumber(int next) {
     int start = byteOffset;
     while (true) {
       next = advance();
       if ($0 <= next && next <= $9) {
         continue;
       } else if (next === $PERIOD) {
         return tokenizeFractionPart(advance(), start);
       } else if (next === $e || next === $E || next === $d || next === $D) {
         return tokenizeFractionPart(next, start);
       } else {
-        appendByteStringToken(INT_TOKEN, asciiString(start));
+        appendByteStringToken(INT_TOKEN, asciiString(start, 0));
         return next;
       }
     }
   }
 
   int tokenizeHexOrNumber(int next) {
     int x = peek();
     if (x === $x || x === $X) {
       advance();
       return tokenizeHex(x);
     }
     return tokenizeNumber(next);
   }
 
   int tokenizeHex(int next) {
-    int start = byteOffset;
+    int start = byteOffset - 1;
     bool hasDigits = false;
     while (true) {
       next = advance();
       if (($0 <= next && next <= $9)
           || ($A <= next && next <= $F)
           || ($a <= next && next <= $f)) {
         hasDigits = true;
       } else {
         if (!hasDigits) {
           throw new MalformedInputException(charOffset);
         }
-        appendByteStringToken(HEXADECIMAL_TOKEN, asciiString(start));
+        appendByteStringToken(HEXADECIMAL_TOKEN, asciiString(start, 0));
         return next;
       }
     }
   }
 
   int tokenizeDotOrNumber(int next) {
     int start = byteOffset;
     next = advance();
     if (($0 <= next && next <= $9)) {
       return tokenizeFractionPart(next, start);
     } else if ($PERIOD === next) {
       return select($PERIOD, "...", "..");
     } else {
       appendStringToken(PERIOD_TOKEN, ".");
       return next;
     }
   }
 
   int tokenizeFractionPart(int next, int start) {
     bool done = false;
+    bool hasDigit = false;
     LOOP: while (!done) {
       if ($0 <= next && next <= $9) {
+        hasDigit = true;
       } else if ($e === next || $E === next) {
+        hasDigit = true;
         next = tokenizeExponent(advance());
         done = true;
         continue LOOP;
       } else {
         done = true;
         continue LOOP;
       }
       next = advance();
     }
+    if (!hasDigit) {
+      appendByteStringToken(INT_TOKEN, asciiString(start, -2));
+      // TODO(ahe): Wrong offset for the period.
+      appendStringToken(PERIOD_TOKEN, ".");
+      return bigSwitch(next);
+    }
     if (next === $d || next === $D) {
       next = advance();
     }
-    appendByteStringToken(DOUBLE_TOKEN, asciiString(start));
+    appendByteStringToken(DOUBLE_TOKEN, asciiString(start, 0));
     return next;
   }
 
   int tokenizeExponent(int next) {
     if (next === $PLUS || next === $MINUS) {
       next = advance();
     }
     bool hasDigits = false;
     while (true) {
       if ($0 <= next && next <= $9) {
@@ skipping 50 matching lines @@
     }
   }
 
   int tokenizeKeywordOrIdentifier(int next) {
     KeywordState state = KeywordState.KEYWORD_STATE;
     int start = byteOffset;
     while (state !== null && $a <= next && next <= $z) {
       state = state.next(next);
       next = advance();
     }
-    if (state === null || !state.isLeaf()) {
+    if (state === null || state.keyword === null) {
       return tokenizeIdentifier(next, start);
     }
     if (($A <= next && next <= $Z) ||
         ($0 <= next && next <= $9) ||
         next === $_ ||
         next === $$) {
       return tokenizeIdentifier(next, start);
     } else if (next < 128) {
       appendKeywordToken(state.keyword);
       return next;
     } else {
       return tokenizeIdentifier(next, start);
     }
   }
 
   int tokenizeIdentifier(int next, int start) {
     bool isAscii = true;
     while (true) {
       if (($a <= next && next <= $z) ||
           ($A <= next && next <= $Z) ||
           ($0 <= next && next <= $9) ||
           next === $_ ||
           next === $$) {
         next = advance();
       } else if (next < 128) {
         if (isAscii) {
-          appendByteStringToken(IDENTIFIER_TOKEN, asciiString(start));
+          appendByteStringToken(IDENTIFIER_TOKEN, asciiString(start, 0));
         } else {
           appendByteStringToken(IDENTIFIER_TOKEN, utf8String(start, -1));
         }
         return next;
       } else {
         int nonAsciiStart = byteOffset;
         do {
           next = nextByte();
         } while (next > 127);
         String string = utf8String(nonAsciiStart, -1).toString();
@@ skipping 29 matching lines @@
       }
     }
     if (raw) {
       return tokenizeSingleLineRawString(next, q, start);
     } else {
       return tokenizeSingleLineString(next, q, start);
     }
   }
 
   int tokenizeSingleLineString(int next, int q1, int start) {
-    while (next != $EOF) {
+    while (next !== $EOF) {
       if (next === q1) {
         appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
         return advance();
       } else if (next === $BACKSLASH) {
         next = advance();
         if (next === $EOF) {
           throw new MalformedInputException(charOffset);
         }
+      } else if (next === $$) {
+        beginToken();
+        next = advance();
+        if (next === $OPEN_CURLY_BRACKET) {
+          next = tokenizeInterpolatedExpression(next, q1, start);
+        }
+        continue;
       } else if (next === $LF || next === $CR) {
         throw new MalformedInputException(charOffset);
       }
       next = advance();
     }
     throw new MalformedInputException(charOffset);
   }
 
+  int tokenizeInterpolatedExpression(int next, int q, int start) {
+    appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
+    appendBeginGroup(STRING_INTERPOLATION_TOKEN, "\${");
+    next = advance();
+    while (next !== $EOF && next !== $STX) {
+      next = bigSwitch(next);
+    }
+    if (next === $EOF) return next;
+    return advance();
+  }
+
   int tokenizeSingleLineRawString(int next, int q1, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === q1) {
         appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
         throw new MalformedInputException(charOffset);
       }
       next = advance();
@@ skipping 19 matching lines @@
     }
     return next;
   }
 }
 
 class MalformedInputException {
   final message;
   MalformedInputException(this.message);
   toString() => message.toString();
 }