Implement string interpolation up to SSA.

dart/frog/leg/scanner/scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
@@ skipping 546 matching lines @@
     }
   }
 
   int tokenizeIdentifier(int next, int start) {
     bool isAscii = true;
     while (true) {
       if (($a <= next && next <= $z) ||
           ($A <= next && next <= $Z) ||
           ($0 <= next && next <= $9) ||
           next === $_ ||
           next === $$) {

[ngeoffray, 2011/12/20 11:54:45]

Is a boolean check really that bad here, compared to duplicate code between
tokenizeIdentifier and tokenizeIdentifierNoDollar?

[ahe, 2011/12/20 13:19:24]

I don't know. I'll try to clean this up in a follow up CL.

         next = advance();
       } else if (next < 128) {
         if (isAscii) {
           appendByteStringToken(IDENTIFIER_TOKEN, asciiString(start, 0));
         } else {
           appendByteStringToken(IDENTIFIER_TOKEN, utf8String(start, -1));
         }
         return next;
       } else {
         int nonAsciiStart = byteOffset;
         do {
           next = nextByte();
         } while (next > 127);
         String string = utf8String(nonAsciiStart, -1).toString();
         isAscii = false;
         int byteLength = nonAsciiStart - byteOffset;
         addToCharOffset(string.length - byteLength);
       }
     }
   }
 
+  int tokenizeIdentifierOrKeywordNoDollar(int next) {

[ngeoffray, 2011/12/20 11:54:45]

The other method is called tokenizeKeywordOrIdentifier, maybe follow the same
naming? And again, a boolean check was too expensive?

[ahe, 2011/12/20 13:19:24]

Next CL.

+    KeywordState state = KeywordState.KEYWORD_STATE;
+    int start = byteOffset;
+    while (state !== null && $a <= next && next <= $z) {
+      state = state.next(next);
+      next = advance();
+    }
+    if (state === null || state.keyword === null) {
+      return tokenizeIdentifierNoDollar(next, start);
+    }
+    if (($A <= next && next <= $Z) ||
+        ($0 <= next && next <= $9) ||
+        next === $_ ||
+        next === $$) {
+      return tokenizeIdentifierNoDollar(next, start);
+    } else if (next < 128) {
+      appendKeywordToken(state.keyword);
+      return next;
+    } else {
+      return tokenizeIdentifierNoDollar(next, start);
+    }
+  }
+
+  int tokenizeIdentifierNoDollar(int next, int start) {
+    bool isAscii = true;
+    while (true) {
+      if (($a <= next && next <= $z) ||
+          ($A <= next && next <= $Z) ||
+          ($0 <= next && next <= $9) ||
+          next === $_) {
+        next = advance();
+      } else if (next < 128) {
+        if (isAscii) {
+          appendByteStringToken(IDENTIFIER_TOKEN, asciiString(start, 0));
+        } else {
+          appendByteStringToken(IDENTIFIER_TOKEN, utf8String(start, -1));
+        }
+        return next;
+      } else {
+        int nonAsciiStart = byteOffset;
+        do {
+          next = nextByte();
+        } while (next > 127);
+        String string = utf8String(nonAsciiStart, -1).toString();
+        isAscii = false;
+        int byteLength = nonAsciiStart - byteOffset;
+        addToCharOffset(string.length - byteLength);
+      }
+    }
+  }
+
   int tokenizeRawString(int next) {
     int start = byteOffset;
     next = advance();
     if (next === $DQ || next === $SQ) {
       return tokenizeString(next, start, true);
     } else {
       throw new MalformedInputException(charOffset);
     }
   }
 
@@ skipping 25 matching lines @@
         return advance();
       } else if (next === $BACKSLASH) {
         next = advance();
         if (next === $EOF) {
           throw new MalformedInputException(charOffset);
         }
       } else if (next === $$) {
         beginToken();
         next = advance();
         if (next === $OPEN_CURLY_BRACKET) {
-          next = tokenizeInterpolatedExpression(next, q1, start);
+          next = tokenizeInterpolatedExpression(next, start);
+        } else {
+          next = tokenizeInterpolatedIdentifier(next, start);
         }
+        start = byteOffset;
         continue;
       } else if (next === $LF || next === $CR) {
         throw new MalformedInputException(charOffset);
       }
       next = advance();
     }
     throw new MalformedInputException(charOffset);
   }
 
-  int tokenizeInterpolatedExpression(int next, int q, int start) {
-    appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
+  int tokenizeInterpolatedExpression(int next, int start) {
+    appendByteStringToken(STRING_TOKEN, utf8String(start, -2));
     appendBeginGroup(STRING_INTERPOLATION_TOKEN, "\${");
     next = advance();
     while (next !== $EOF && next !== $STX) {
       next = bigSwitch(next);
     }
     if (next === $EOF) return next;
     return advance();
   }
 
+  int tokenizeInterpolatedIdentifier(int next, int start) {
+    appendByteStringToken(STRING_TOKEN, utf8String(start, -2));
+    appendBeginGroup(STRING_INTERPOLATION_TOKEN, "\${");
+    next = tokenizeIdentifierOrKeywordNoDollar(next);
+    appendEndGroup(CLOSE_CURLY_BRACKET_TOKEN, "}", OPEN_CURLY_BRACKET_TOKEN);
+    return next;
+  }
+
   int tokenizeSingleLineRawString(int next, int q1, int start) {
     next = advance();
     while (next != $EOF) {
       if (next === q1) {
         appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
         return advance();
       } else if (next === $LF || next === $CR) {
         throw new MalformedInputException(charOffset);
       }
       next = advance();
     }
     throw new MalformedInputException(charOffset);
   }
 
   int tokenizeMultiLineString(int q, int start, bool raw) {
     // TODO(ahe): Handle escapes.
+    // TODO(ahe): Handle string interpolation.
     int next = advance();
     while (next != $EOF) {
       if (next === q) {
         next = advance();
         if (next === q) {
           next = advance();
           if (next === q) {
             appendByteStringToken(STRING_TOKEN, utf8String(start, 0));
             return advance();
           }
         }
       }
       next = advance();
     }
     return next;
   }
 }
 
 class MalformedInputException {
   final message;
   MalformedInputException(this.message);
   toString() => message.toString();
 }