Handle escapes in string literals.

frog/leg/scanner/scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 interface Scanner {
   Token tokenize();
 }
 
 /**
  * Common base class for a Dart scanner.
@@ skipping 611 matching lines @@
         return next;
       }
     }
     if (raw) {
       return tokenizeSingleLineRawString(next, q, start);
     } else {
       return tokenizeSingleLineString(next, q, start);
     }
   }
 
+  static bool isHexDigit(int character) {
+    if ($0 <= character && character <= $9) return true;
+    character |= 0x20;
+    return ($a <= character && character <= $f);
+  }
+
+  int tokenizeStringEscape(int next) {
+    if (next === $u) {
+      next = advance();
+      bool isBracketed = false;
+      int maxLength = 4;
+      if (next === $OPEN_CURLY_BRACKET) {
+        isBracketed = true;
+        maxLength = 7;
+        next = advance();
+      }
+      int startOffset = charOffset;
+      int unicodePoint = 0;
+      int length = 0;
+      while (length < maxLength && isHexDigit(next)) {
+        int digit = next <= $9 ? next - $0 : (next | 0x20) - ($a - 10);
+        unicodePoint = unicodePoint * 16 + digit;
+        length += 1;
+        next = advance();
+      }
+      if (isBracketed) {
+        if (length === 0) {
+          throw new MalformedInputException(startOffset);
+        }
+        if (next !== $CLOSE_CURLY_BRACKET) {
+          throw new MalformedInputException(charOffset);
+        }
+        next = advance();
+      } else if (length != 4) {
+        throw new MalformedInputException(startOffset + length);
+      }
+      if ((0xD800 <= unicodePoint && unicodePoint <= 0xDFFF) ||
+          unicodePoint > 0x10ffff) {
+        // Don't allow invalid code points in strings.
+        throw new MalformedInputException(startOffset);
+      }
+      return next;
+    } else if (next === $x) {
+      for (int i = 0; i < 2; i++) {
+        next = advance();
+        if (!isHexDigit(next)) {
+          throw new MalformedInputException(charOffset);
+        }
+      }
+    }
+    // All other characters following a backslash are always valid escapes
+    // in Dart.
+    return advance();
+  }
+
+
   int tokenizeSingleLineString(int next, int q1, int start) {
     while (next !== $EOF) {
       if (next === q1) {
         appendByteStringToken(STRING_INFO, utf8String(start, 0));
         return advance();
       } else if (next === $BACKSLASH) {
         next = advance();
-        if (next === $EOF) {
+        if (next === $EOF || next === $LF || next === $CR) {

[ahe, 2012/01/17 13:21:39]

Shouldn't this be in tokenizeStringEscape?

[Lasse Reichstein Nielsen, 2012/01/18 08:22:34]

This method has been reverted.

This shouldn't have been in tokenizeStringEscape, since I wanted to reuse it for
multiline strings (where $LF and $CR are valid).

           throw new MalformedInputException(charOffset);
         }
+        next = tokenizeStringEscape(next);
+        continue;
       } else if (next === $$) {
         beginToken();
         next = advance();
         if (next === $OPEN_CURLY_BRACKET) {
           next = tokenizeInterpolatedExpression(next, start);
         } else {
           next = tokenizeInterpolatedIdentifier(next, start);
         }
         start = byteOffset;
         continue;
@@ skipping 57 matching lines @@
     }
     return next;
   }
 }
 
 class MalformedInputException {
   final message;
   MalformedInputException(this.message);
   toString() => message.toString();
 }