Handle escapes in string literals.

frog/leg/scanner/scanner.dart

Index: frog/leg/scanner/scanner.dart
diff --git a/frog/leg/scanner/scanner.dart b/frog/leg/scanner/scanner.dart
index bfce2342cb9597ab318b32124ef838c973f29a26..f4bdfdd6aed06a7d2267c34adf3f2f9c9e42db4f 100644
--- a/frog/leg/scanner/scanner.dart
+++ b/frog/leg/scanner/scanner.dart
@@ -629,6 +629,62 @@ class AbstractScanner<T> implements Scanner {
     }
   }
 
+  static bool isHexDigit(int character) {
+    if ($0 <= character && character <= $9) return true;
+    character |= 0x20;
+    return ($a <= character && character <= $f);
+  }
+
+  int tokenizeStringEscape(int next) {
+    if (next === $u) {
+      next = advance();
+      bool isBracketed = false;
+      int maxLength = 4;
+      if (next === $OPEN_CURLY_BRACKET) {
+        isBracketed = true;
+        maxLength = 7;
+        next = advance();
+      }
+      int startOffset = charOffset;
+      int unicodePoint = 0;
+      int length = 0;
+      while (length < maxLength && isHexDigit(next)) {
+        int digit = next <= $9 ? next - $0 : (next | 0x20) - ($a - 10);
+        unicodePoint = unicodePoint * 16 + digit;
+        length += 1;
+        next = advance();
+      }
+      if (isBracketed) {
+        if (length === 0) {
+          throw new MalformedInputException(startOffset);
+        }
+        if (next !== $CLOSE_CURLY_BRACKET) {
+          throw new MalformedInputException(charOffset);
+        }
+        next = advance();
+      } else if (length != 4) {
+        throw new MalformedInputException(startOffset + length);
+      }
+      if ((0xD800 <= unicodePoint && unicodePoint <= 0xDFFF) ||
+          unicodePoint > 0x10ffff) {
+        // Don't allow invalid code points in strings.
+        throw new MalformedInputException(startOffset);
+      }
+      return next;
+    } else if (next === $x) {
+      for (int i = 0; i < 2; i++) {
+        next = advance();
+        if (!isHexDigit(next)) {
+          throw new MalformedInputException(charOffset);
+        }
+      }
+    }
+    // All other characters following a backslash are always valid escapes
+    // in Dart.
+    return advance();
+  }
+
+
   int tokenizeSingleLineString(int next, int q1, int start) {
     while (next !== $EOF) {
       if (next === q1) {
@@ -636,9 +692,11 @@ class AbstractScanner<T> implements Scanner {
         return advance();
       } else if (next === $BACKSLASH) {
         next = advance();
-        if (next === $EOF) {
+        if (next === $EOF || next === $LF || next === $CR) {

[ahe, 2012/01/17 13:21:39]

Shouldn't this be in tokenizeStringEscape?

[Lasse Reichstein Nielsen, 2012/01/18 08:22:34]

This method has been reverted.

This shouldn't have been in tokenizeStringEscape, since I wanted to reuse it for
multiline strings (where $LF and $CR are valid).

           throw new MalformedInputException(charOffset);
         }
+        next = tokenizeStringEscape(next);
+        continue;
       } else if (next === $$) {
         beginToken();
         next = advance();