Scanner for UTF-8 byte arrays

sdk/lib/_internal/compiler/implementation/scanner/scanner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 abstract class Scanner {
   Token tokenize();
+
+  factory Scanner(SourceFile file, {bool includeComments: false}) {
+    if (file is Utf8BytesSourceFile) {
+      return new Utf8BytesScanner(file, includeComments: includeComments);
+    } else {
+      return new StringScanner(file, includeComments: includeComments);
+    }
+  }
 }
 
-/**
- * Common base class for a Dart scanner.
- */
-abstract class AbstractScanner<T extends SourceString> implements Scanner {
-  int advance();
-  int nextByte();
+abstract class AbstractScanner implements Scanner {
+  final bool includeComments;
 
   /**
-   * Returns the current character or byte depending on the underlying input
-   * kind. For example, [StringScanner] operates on [String] and thus returns
-   * characters (Unicode codepoints represented as int) whereas
-   * [ByteArrayScanner] operates on byte arrays and thus returns bytes.
+   * The string offset for the next token that will be created.
+   *
+   * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values
+   * are different. One string character can be encoded using multiple UTF-8
+   * bytes.
+   */
+  int tokenStart = -1;
+
+  /**
+   * A pointer to the token stream created by this scanner. The first token
+   * is a special token and not part of the source file. This is an
+   * implementation detail to avoids special cases in the scanner. This token
+   * is not exposed to clients of the scanner, which are expected to invoke
+   * [firstToken] to access the token stream.
+   */
+  final Token tokens = new SymbolToken(EOF_INFO, -1);
+
+  /**
+   * A pointer to the last scanned token.
+   */
+  Token tail;
+
+  /**
+   * The source file that is being scanned. This field can be [:null:].
+   * If the source file is available, the scanner assigns its [:lineStarts:] and
+   * [:length:] fields at the end of [tokenize].
+   */
+  final SourceFile file;
+
+  final List<int> lineStarts = [0];

[ngeoffray, 2013/10/18 10:19:37]

<int>[0]

[lukas, 2013/10/24 16:48:36]

Done.

+
+  AbstractScanner(this.file, this.includeComments) {
+    this.tail = this.tokens;
+  }
+
+

[ngeoffray, 2013/10/18 10:19:37]

Extra line.

[lukas, 2013/10/24 16:48:36]

Done.

+  /**
+   * Advances and returns the next character.
+   *
+   * If the next character is non-ASCII, then the returned value depends on the
+   * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while
+   * the [StringScanner] returns a UTF-16 code unit.
+   *
+   * The scanner ensures that [advance] is not invoked after it returned [$EOF].
+   * This allows implementations to omit bound checks if the data structure ends
+   * with '0'.
+   */
+  int advance();
+
+  /**
+   * Returns the current unicode character.
+   *
+   * If the current character is ASCII, then it is returned unchanged.
+   *
+   * The [Utf8BytesScanner] decodes the next unicode code point starting at the
+   * current position. Note that every unicode character is returned as a single
+   * code point, i.e., for '\u{1d11e}' it returns 119070, and the following
+   * [advance] returns the next character.
+   *
+   * The [StringScanner] returns the current character unchanged, which might
+   * be a surrogate character. In the case of '\u{1d11e}', it returns the first
+   * code unit 55348, and the following [advance] returns the second code unit
+   * 56606.
+   *
+   * Invoking [currentAsUnicode] multiple times is safe, i.e.,

[ngeoffray, 2013/10/18 10:19:37]

i.e. -> that is

[lukas, 2013/10/24 16:48:36]

Done.

+   * [:currentAsUnicode(next) == currentAsUnicode(currentAsUnicode(next)):].
+   */
+  int currentAsUnicode(int next);
+
+  /**
+   * Returns the character at the next poisition. Like in [advance], the
+   * [Utf8BytesScanner] returns a UTF-8 byte, while the [StringScanner] returns
+   * a UTF-16 code unit.
    */
   int peek();
 
   /**
-   * Appends a fixed token based on whether the current char is [choice] or not.
-   * If the current char is [choice] a fixed token whose kind and content
-   * is determined by [yes] is appended, otherwise a fixed token whose kind
-   * and content is determined by [no] is appended.
+   * Notifies the scanner that unicode characters were detected in either a
+   * comment or a string literal between [startScanOffset] and the current
+   * scan offset.
    */
+  void handleUnicode(int startScanOffset);
+
+  /**
+   * Returns the current scan offset.
+   *
+   * In the [Utf8BytesScanner] this is the offset into the byte list, in the
+   * [StringScanner] the offset in the source string.
+   */
+  int get scanOffset;
+
+  /**
+   * Returns the current string offset.
+   *
+   * In the [StringScanner] this is identical to the [scanOffset]. In the
+   * [Utf8BytesScanner] it is computed based on encountered UTF-8 characters.
+   */
+  int get stringOffset;
+
+  /**
+   * Returns the first token scanned by this [Scanner].
+   */
+  Token firstToken();
+
+  /**
+   * Returns the last token scanned by this [Scanner].
+   */
+  Token previousToken();
+
+  /**
+   * Notifies that a new token starts at current offset.
+   */
+  void beginToken() {
+    tokenStart = stringOffset;
+  }
+
+  /**
+   * Appends a substring from the scan offset [:start:] to the current
+   * [:scanOffset:] plus the [:extraOffset:]. For example, if the current
+   * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the
+   * substring string [5,9).
+   *
+   * Note that [extraOffset] can only be used if the covered character(s) are
+   * known to be ASCII.
+   */
+  void appendSubstringToken(PrecedenceInfo info, int start,
+                            bool asciiOnly, [int extraOffset]);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendStringToken(PrecedenceInfo info, String value);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendPrecedenceToken(PrecedenceInfo info);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
   int select(int choice, PrecedenceInfo yes, PrecedenceInfo no);
 
-  /**
-   * Appends a fixed token whose kind and content is determined by [info].
-   */
-  void appendPrecedenceToken(PrecedenceInfo info);
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendKeywordToken(Keyword keyword);
 
-  /**
-   * Appends a token whose kind is determined by [info] and content is [value].
-   */
-  void appendStringToken(PrecedenceInfo info, String value);
-
-  /**
-   * Appends a token whose kind is determined by [info] and content is defined
-   * by the SourceString [value].
-   */
-  void appendByteStringToken(PrecedenceInfo info, T value);
-
-  /**
-   * Appends a keyword token whose kind is determined by [keyword].
-   */
-  void appendKeywordToken(Keyword keyword);
-  void appendWhiteSpace(int next);
+  /** Documentation in subclass [ArrayBasedScanner]. */
   void appendEofToken();
 
-  /**
-   * Creates an ASCII SourceString whose content begins at the source byte
-   * offset [start] and ends at [offset] bytes from the current byte offset of
-   * the scanner. For example, if the current byte offset is 10,
-   * [:asciiString(0,-1):] creates an ASCII SourceString whose content is found
-   * at the [0,9[ byte interval of the source text.
-   */
-  T asciiString(int start, int offset);
-  T utf8String(int start, int offset);
-  Token firstToken();
-  Token previousToken();
-  void beginToken();
-  void addToCharOffset(int offset);
-  int get charOffset;
-  int get byteOffset;
-  void appendBeginGroup(PrecedenceInfo info, String value);
-  int appendEndGroup(PrecedenceInfo info, String value, int openKind);
-  void appendGt(PrecedenceInfo info, String value);
-  void appendGtGt(PrecedenceInfo info, String value);
-  void appendGtGtGt(PrecedenceInfo info, String value);
-  void appendComment();
+  /** Documentation in subclass [ArrayBasedScanner]. */

[ngeoffray, 2013/10/18 10:19:37]

So do the following methods only apply to the ArrayBasedScanner?

[lukas, 2013/10/24 16:48:36]

I just decided to put the documentation together with the actual implementation.
Technically the implementations could go here, but Peter preferred to separate
scanning code from token building code.

+  void appendWhiteSpace(int next);
 
-  /**
-   * We call this method to discard '<' from the "grouping" stack
-   * (maintained by subclasses).
-   *
-   * [PartialParser.skipExpression] relies on the fact that we do not
-   * create groups for stuff like:
-   * [:a = b < c, d = e > f:].
-   *
-   * In other words, this method is called when the scanner recognizes
-   * something which cannot possibly be part of a type
-   * parameter/argument list.
-   */
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void lineFeedInMultiline();
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendBeginGroup(PrecedenceInfo info);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  int appendEndGroup(PrecedenceInfo info, int openKind);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendGt(PrecedenceInfo info);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendGtGt(PrecedenceInfo info);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
+  void appendComment(start, bool asciiOnly);
+
+  /** Documentation in subclass [ArrayBasedScanner]. */
   void discardOpenLt();
 
   // TODO(ahe): Move this class to implementation.
 
   Token tokenize() {
     int next = advance();
     while (!identical(next, $EOF)) {
       next = bigSwitch(next);
     }
     appendEofToken();
+
+    if (file != null) {
+      file.length = stringOffset;
+      // One additional line start at the end, see [SourceFile.lineStarts].
+      lineStarts.add(stringOffset + 1);
+      file.lineStarts = lineStarts;
+    }
+
     return firstToken();
   }
 
   int bigSwitch(int next) {
     beginToken();
     if (identical(next, $SPACE) || identical(next, $TAB)
         || identical(next, $LF) || identical(next, $CR)) {
       appendWhiteSpace(next);
       next = advance();
+      // Sequences of spaces are common, so advance through them fast.
       while (identical(next, $SPACE)) {
-        appendWhiteSpace(next);
+        // We don't invoke [:appendWhiteSpace(next):] here for efficiency,
+        // assuming that it does not do anything for space characters.
         next = advance();
       }
       return next;
     }
 
     if ($a <= next && next <= $z) {
       if (identical($r, next)) {
         return tokenizeRawStringKeywordOrIdentifier(next);
       }
       return tokenizeKeywordOrIdentifier(next, true);
     }
 
-    if (($A <= next && next <= $Z) || identical(next, $_) || identical(next, $$)) {
-      return tokenizeIdentifier(next, byteOffset, true);
+    if (($A <= next && next <= $Z) ||
+        identical(next, $_) ||
+        identical(next, $$)) {
+      return tokenizeIdentifier(next, scanOffset, true);
     }
 
     if (identical(next, $LT)) {
       return tokenizeLessThan(next);
     }
 
     if (identical(next, $GT)) {
       return tokenizeGreaterThan(next);
     }
 
@@ skipping 44 matching lines @@
     if (identical(next, $BACKSLASH)) {
       appendPrecedenceToken(BACKSLASH_INFO);
       return advance();
     }
 
     if (identical(next, $HASH)) {
       return tokenizeTag(next);
     }
 
     if (identical(next, $OPEN_PAREN)) {
-      appendBeginGroup(OPEN_PAREN_INFO, "(");
+      appendBeginGroup(OPEN_PAREN_INFO);
       return advance();
     }
 
     if (identical(next, $CLOSE_PAREN)) {
-      return appendEndGroup(CLOSE_PAREN_INFO, ")", OPEN_PAREN_TOKEN);
+      return appendEndGroup(CLOSE_PAREN_INFO, OPEN_PAREN_TOKEN);
     }
 
     if (identical(next, $COMMA)) {
       appendPrecedenceToken(COMMA_INFO);
       return advance();
     }
 
     if (identical(next, $COLON)) {
       appendPrecedenceToken(COLON_INFO);
       return advance();
     }
 
     if (identical(next, $SEMICOLON)) {
       appendPrecedenceToken(SEMICOLON_INFO);
       // Type parameters and arguments cannot contain semicolon.
       discardOpenLt();
       return advance();
     }
 
     if (identical(next, $QUESTION)) {
       appendPrecedenceToken(QUESTION_INFO);
       return advance();
     }
 
     if (identical(next, $CLOSE_SQUARE_BRACKET)) {
-      return appendEndGroup(CLOSE_SQUARE_BRACKET_INFO, "]",
+      return appendEndGroup(CLOSE_SQUARE_BRACKET_INFO,
                             OPEN_SQUARE_BRACKET_TOKEN);
     }
 
     if (identical(next, $BACKPING)) {
       appendPrecedenceToken(BACKPING_INFO);
       return advance();
     }
 
     if (identical(next, $OPEN_CURLY_BRACKET)) {
-      appendBeginGroup(OPEN_CURLY_BRACKET_INFO, "{");
+      appendBeginGroup(OPEN_CURLY_BRACKET_INFO);
       return advance();
     }
 
     if (identical(next, $CLOSE_CURLY_BRACKET)) {
-      return appendEndGroup(CLOSE_CURLY_BRACKET_INFO, "}",
+      return appendEndGroup(CLOSE_CURLY_BRACKET_INFO,
                             OPEN_CURLY_BRACKET_TOKEN);
     }
 
     if (identical(next, $SLASH)) {
       return tokenizeSlashOrComment(next);
     }
 
     if (identical(next, $AT)) {
       return tokenizeAt(next);
     }
 
     if (identical(next, $DQ) || identical(next, $SQ)) {
-      return tokenizeString(next, byteOffset, false);
+      return tokenizeString(next, scanOffset, false);
     }
 
     if (identical(next, $PERIOD)) {
       return tokenizeDotsOrNumber(next);
     }
 
     if (identical(next, $0)) {
       return tokenizeHexOrNumber(next);
     }
 
     // TODO(ahe): Would a range check be faster?
     if (identical(next, $1) || identical(next, $2) || identical(next, $3)
         || identical(next, $4) ||  identical(next, $5) || identical(next, $6)
         || identical(next, $7) || identical(next, $8) || identical(next, $9)) {
       return tokenizeNumber(next);
     }
 
     if (identical(next, $EOF)) {
       return $EOF;
     }
     if (next < 0x1f) {
-      return error(new SourceString("unexpected character $next"));
+      return error("unexpected character $next");
     }
 
+    next = currentAsUnicode(next);
+
     // The following are non-ASCII characters.
 
     if (identical(next, $NBSP)) {
       appendWhiteSpace(next);
       return advance();
     }
 
-    return tokenizeIdentifier(next, byteOffset, true);
+    return error("unexpected unicode character $next");
   }
 
   int tokenizeTag(int next) {
     // # or #!.*[\n\r]
-    if (byteOffset == 0) {
+    if (scanOffset == 0) {
       if (identical(peek(), $BANG)) {
+        int start = scanOffset + 1;
+        bool asciiOnly = true;
         do {
           next = advance();
-        } while (!identical(next, $LF) && !identical(next, $CR) && !identical(next, $EOF));
+          if (next > 127) asciiOnly = false;
+        } while (!identical(next, $LF) &&
+                 !identical(next, $CR) &&
+                 !identical(next, $EOF));
+        if (!asciiOnly) handleUnicode(start);
         return next;
       }
     }
     appendPrecedenceToken(HASH_INFO);
     return advance();
   }
 
   int tokenizeTilde(int next) {
     // ~ ~/ ~/=
     next = advance();
     if (identical(next, $SLASH)) {
       return select($EQ, TILDE_SLASH_EQ_INFO, TILDE_SLASH_INFO);
     } else {
       appendPrecedenceToken(TILDE_INFO);
       return next;
     }
   }
 
   int tokenizeOpenSquareBracket(int next) {
     // [ [] []=
     next = advance();
     if (identical(next, $CLOSE_SQUARE_BRACKET)) {
       Token token = previousToken();
-      if (token is KeywordToken && identical(token.value.stringValue, 'operator')) {
+      if (token is KeywordToken &&
+          identical((token as KeywordToken).keyword.syntax, 'operator')) {
         return select($EQ, INDEX_EQ_INFO, INDEX_INFO);
       }
     }
-    appendBeginGroup(OPEN_SQUARE_BRACKET_INFO, "[");
+    appendBeginGroup(OPEN_SQUARE_BRACKET_INFO);
     return next;
   }
 
   int tokenizeCaret(int next) {
     // ^ ^=
     return select($EQ, CARET_EQ_INFO, CARET_INFO);
   }
 
   int tokenizeBar(int next) {
     // | || |=
@@ skipping 43 matching lines @@
       return advance();
     } else if (identical(next, $EQ)) {
       appendPrecedenceToken(MINUS_EQ_INFO);
       return advance();
     } else {
       appendPrecedenceToken(MINUS_INFO);
       return next;
     }
   }
 
-
   int tokenizePlus(int next) {
     // + ++ +=
     next = advance();
     if (identical($PLUS, next)) {
       appendPrecedenceToken(PLUS_PLUS_INFO);
       return advance();
     } else if (identical($EQ, next)) {
       appendPrecedenceToken(PLUS_EQ_INFO);
       return advance();
     } else {
       appendPrecedenceToken(PLUS_INFO);
       return next;
     }
   }
 
   int tokenizeExclamation(int next) {
-    // ! != !==
+    // ! !=
+    // !== is kept for user-friendly error reporting.
+
     next = advance();
     if (identical(next, $EQ)) {
       return select($EQ, BANG_EQ_EQ_INFO, BANG_EQ_INFO);
     }
     appendPrecedenceToken(BANG_INFO);
     return next;
   }
 
   int tokenizeEquals(int next) {
-    // = == ===
+    // = == =>
+    // === is kept for user-friendly error reporting.
 
     // Type parameters and arguments cannot contain any token that
     // starts with '='.
     discardOpenLt();
 
     next = advance();
     if (identical(next, $EQ)) {
       return select($EQ, EQ_EQ_EQ_INFO, EQ_EQ_INFO);
     } else if (identical(next, $GT)) {
       appendPrecedenceToken(FUNCTION_INFO);
       return advance();
     }
     appendPrecedenceToken(EQ_INFO);
     return next;
   }
 
   int tokenizeGreaterThan(int next) {
-    // > >= >> >>= >>> >>>=
+    // > >= >> >>=
     next = advance();
     if (identical($EQ, next)) {
       appendPrecedenceToken(GT_EQ_INFO);
       return advance();
     } else if (identical($GT, next)) {
       next = advance();
       if (identical($EQ, next)) {
         appendPrecedenceToken(GT_GT_EQ_INFO);
         return advance();
       } else {
-        appendGtGt(GT_GT_INFO, ">>");
+        appendGtGt(GT_GT_INFO);
         return next;
       }
     } else {
-      appendGt(GT_INFO, ">");
+      appendGt(GT_INFO);
       return next;
     }
   }
 
   int tokenizeLessThan(int next) {
     // < <= << <<=
     next = advance();
     if (identical($EQ, next)) {
       appendPrecedenceToken(LT_EQ_INFO);
       return advance();
     } else if (identical($LT, next)) {
       return select($EQ, LT_LT_EQ_INFO, LT_LT_INFO);
     } else {
-      appendBeginGroup(LT_INFO, "<");
+      appendBeginGroup(LT_INFO);
       return next;
     }
   }
 
   int tokenizeNumber(int next) {
-    int start = byteOffset;
+    int start = scanOffset;
     while (true) {
       next = advance();
       if ($0 <= next && next <= $9) {
         continue;
       } else if (identical(next, $e) || identical(next, $E)) {
         return tokenizeFractionPart(next, start);
       } else {
         if (identical(next, $PERIOD)) {
           int nextnext = peek();
           if ($0 <= nextnext && nextnext <= $9) {
             return tokenizeFractionPart(advance(), start);
           }
         }
-        appendByteStringToken(INT_INFO, asciiString(start, 0));
+        appendSubstringToken(INT_INFO, start, true);
         return next;
       }
     }
   }
 
   int tokenizeHexOrNumber(int next) {
     int x = peek();
     if (identical(x, $x) || identical(x, $X)) {
-      advance();
-      return tokenizeHex(x);
+      return tokenizeHex(next);
     }
     return tokenizeNumber(next);
   }
 
   int tokenizeHex(int next) {
-    int start = byteOffset - 1;
+    int start = scanOffset;
+    next = advance(); // Advance past the $x or $X.
     bool hasDigits = false;
     while (true) {
       next = advance();
       if (($0 <= next && next <= $9)
           || ($A <= next && next <= $F)
           || ($a <= next && next <= $f)) {
         hasDigits = true;
       } else {
         if (!hasDigits) {
-          return error(const SourceString("hex digit expected"));
+          return error("hex digit expected");
         }
-        appendByteStringToken(HEXADECIMAL_INFO, asciiString(start, 0));
+        appendSubstringToken(HEXADECIMAL_INFO, start, true);
         return next;
       }
     }
   }
 
   int tokenizeDotsOrNumber(int next) {
-    int start = byteOffset;
+    int start = scanOffset;
     next = advance();
     if (($0 <= next && next <= $9)) {
       return tokenizeFractionPart(next, start);
     } else if (identical($PERIOD, next)) {
       return select($PERIOD, PERIOD_PERIOD_PERIOD_INFO, PERIOD_PERIOD_INFO);
     } else {
       appendPrecedenceToken(PERIOD_INFO);
       return next;
     }
   }
 
   int tokenizeFractionPart(int next, int start) {
     bool done = false;
     bool hasDigit = false;
     LOOP: while (!done) {
       if ($0 <= next && next <= $9) {
         hasDigit = true;
       } else if (identical($e, next) || identical($E, next)) {
         hasDigit = true;
         next = tokenizeExponent(advance());
         done = true;
         continue LOOP;
       } else {
         done = true;
         continue LOOP;
       }
       next = advance();
     }
     if (!hasDigit) {
-      appendByteStringToken(INT_INFO, asciiString(start, -1));
+      // Reduce offset, we already advanced to the token past the period.
+      appendSubstringToken(INT_INFO, start, true, -1);
+
+      // TODO(ahe): Wrong offset for the period. Cannot call beginToken because
+      // the scanner already advanced past the period.
       if (identical($PERIOD, next)) {
         return select($PERIOD, PERIOD_PERIOD_PERIOD_INFO, PERIOD_PERIOD_INFO);
       }
-      // TODO(ahe): Wrong offset for the period.
       appendPrecedenceToken(PERIOD_INFO);
-      return bigSwitch(next);
+      return next;
     }
-    appendByteStringToken(DOUBLE_INFO, asciiString(start, 0));
+    appendSubstringToken(DOUBLE_INFO, start, true);
     return next;
   }
 
   int tokenizeExponent(int next) {
     if (identical(next, $PLUS) || identical(next, $MINUS)) {
       next = advance();
     }
     bool hasDigits = false;
     while (true) {
       if ($0 <= next && next <= $9) {
         hasDigits = true;
       } else {
         if (!hasDigits) {
-          return error(const SourceString("digit expected"));
+          return error("digit expected");
         }
         return next;
       }
       next = advance();
     }
   }
 
   int tokenizeSlashOrComment(int next) {
+    int start = scanOffset;
     next = advance();
     if (identical($STAR, next)) {
-      return tokenizeMultiLineComment(next);
+      return tokenizeMultiLineComment(next, start);
     } else if (identical($SLASH, next)) {
-      return tokenizeSingleLineComment(next);
+      return tokenizeSingleLineComment(next, start);
     } else if (identical($EQ, next)) {
       appendPrecedenceToken(SLASH_EQ_INFO);
       return advance();
     } else {
       appendPrecedenceToken(SLASH_INFO);
       return next;
     }
   }
 
-  int tokenizeSingleLineComment(int next) {
+  int tokenizeSingleLineComment(int next, int start) {
+    bool asciiOnly = true;
     while (true) {
       next = advance();
-      if (identical($LF, next) || identical($CR, next) || identical($EOF, next)) {
-        appendComment();
+      if (next > 127) asciiOnly = false;
+      if (identical($LF, next) ||
+          identical($CR, next) ||
+          identical($EOF, next)) {
+        if (!asciiOnly) handleUnicode(start);
+        appendComment(start, asciiOnly);
         return next;
       }
     }
   }
 
-  int tokenizeMultiLineComment(int next) {
+
+  int tokenizeMultiLineComment(int next, int start) {
+    bool asciiOnlyComment = true; // Track if the entire comment is ASCII.
+    bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode.
+    int unicodeStart = start;
     int nesting = 1;
     next = advance();
     while (true) {
       if (identical($EOF, next)) {
-        // TODO(ahe): Report error.
+        if (!asciiOnlyLines) handleUnicode(unicodeStart);
+        appendStringToken(BAD_INPUT_INFO, "unterminated multi-line comment");
         return next;
       } else if (identical($STAR, next)) {
         next = advance();
         if (identical($SLASH, next)) {
           --nesting;
           if (0 == nesting) {
+            if (!asciiOnlyLines) handleUnicode(unicodeStart);
             next = advance();
-            appendComment();
+            appendComment(start, asciiOnlyComment);
             return next;
           } else {
             next = advance();
           }
         }
       } else if (identical($SLASH, next)) {
         next = advance();
         if (identical($STAR, next)) {
           next = advance();
           ++nesting;
         }
+      } else if (identical(next, $LF)) {
+        if (!asciiOnlyLines) {
+          // Synchronize the string offset in the utf8 scanner.
+          handleUnicode(unicodeStart);
+          asciiOnlyLines = true;
+          unicodeStart = scanOffset;
+        }
+        lineFeedInMultiline();
+        next = advance();
       } else {
+        if (next > 127) {
+          asciiOnlyLines = false;
+          asciiOnlyComment = false;
+        }
         next = advance();
       }
     }
   }
 
   int tokenizeRawStringKeywordOrIdentifier(int next) {
+    // [next] is $r.
     int nextnext = peek();
     if (identical(nextnext, $DQ) || identical(nextnext, $SQ)) {
-      int start = byteOffset;
+      int start = scanOffset;
       next = advance();
       return tokenizeString(next, start, true);
     }
     return tokenizeKeywordOrIdentifier(next, true);
   }
 
   int tokenizeKeywordOrIdentifier(int next, bool allowDollar) {
     KeywordState state = KeywordState.KEYWORD_STATE;
-    int start = byteOffset;
+    int start = scanOffset;
     while (state != null && $a <= next && next <= $z) {
       state = state.next(next);
       next = advance();
     }
     if (state == null || state.keyword == null) {
       return tokenizeIdentifier(next, start, allowDollar);
     }
     if (($A <= next && next <= $Z) ||
         ($0 <= next && next <= $9) ||
         identical(next, $_) ||
         identical(next, $$)) {
       return tokenizeIdentifier(next, start, allowDollar);
-    } else if (next < 128) {
+    } else {
       appendKeywordToken(state.keyword);
       return next;
-    } else {
-      return tokenizeIdentifier(next, start, allowDollar);
     }
   }
 
+  /**
+   * [allowDollar] can exclude '$', which is not allowed as part of a string
+   * interpolation identifier.
+   */
   int tokenizeIdentifier(int next, int start, bool allowDollar) {
-    bool isAscii = true;
-
     while (true) {
       if (($a <= next && next <= $z) ||
           ($A <= next && next <= $Z) ||
           ($0 <= next && next <= $9) ||
           identical(next, $_) ||
           (identical(next, $$) && allowDollar)) {
         next = advance();
-      } else if ((next < 128) || (identical(next, $NBSP))) {
+      } else {
         // Identifier ends here.
-        if (start == byteOffset) {
-          return error(const SourceString("expected identifier"));
-        } else if (isAscii) {
-          appendByteStringToken(IDENTIFIER_INFO, asciiString(start, 0));
+        if (start == scanOffset) {
+          return error("expected identifier");
         } else {
-          appendByteStringToken(BAD_INPUT_INFO, utf8String(start, -1));
+          appendSubstringToken(IDENTIFIER_INFO, start, true);
         }
         return next;
-      } else {
-        int nonAsciiStart = byteOffset;
-        do {
-          next = nextByte();
-          if (identical(next, $NBSP)) break;
-        } while (next > 127);
-        String string = utf8String(nonAsciiStart, -1).slowToString();
-        isAscii = false;
-        int byteLength = nonAsciiStart - byteOffset;
-        addToCharOffset(string.length - byteLength);
       }
     }
   }
 
   int tokenizeAt(int next) {
-    int start = byteOffset;
-    next = advance();
     appendPrecedenceToken(AT_INFO);
-    return next;
+    return advance();
   }
 
   int tokenizeString(int next, int start, bool raw) {
     int quoteChar = next;
     next = advance();
     if (identical(quoteChar, next)) {
       next = advance();
       if (identical(quoteChar, next)) {
         // Multiline string.
         return tokenizeMultiLineString(quoteChar, start, raw);
       } else {
         // Empty string.
-        appendByteStringToken(STRING_INFO, utf8String(start, -1));
+        appendSubstringToken(STRING_INFO, start, true);
         return next;
       }
     }
     if (raw) {
       return tokenizeSingleLineRawString(next, quoteChar, start);
     } else {
       return tokenizeSingleLineString(next, quoteChar, start);
     }
   }
 
-  static bool isHexDigit(int character) {
-    if ($0 <= character && character <= $9) return true;
-    character |= 0x20;
-    return ($a <= character && character <= $f);
-  }
-
+  /**
+   * [next] is the first character after the qoute.

[ngeoffray, 2013/10/18 10:19:37]

qoute -> quote

[lukas, 2013/10/24 16:48:36]

Done.

+   * [start] is the scanOffset of the quote.
+   *
+   * The token contains a substring of the source file, including the
+   * string quotes, backslashes for escaping. For interpolated strings,
+   * the parts before and after are separate tokens.
+   *
+   *   "a $b c"
+   *
+   * gives StringToken("a $), StringToken(b) and StringToken( c").
+   */
   int tokenizeSingleLineString(int next, int quoteChar, int start) {
+    bool asciiOnly = true;
     while (!identical(next, quoteChar)) {
       if (identical(next, $BACKSLASH)) {
         next = advance();
       } else if (identical(next, $$)) {
-        next = tokenizeStringInterpolation(start);
-        start = byteOffset;
+        if (!asciiOnly) handleUnicode(start);
+        next = tokenizeStringInterpolation(start, asciiOnly);
+        start = scanOffset;
+        asciiOnly = true;
         continue;
       }
       if (next <= $CR
-          && (identical(next, $LF) || identical(next, $CR) || identical(next, $EOF))) {
-        return error(const SourceString("unterminated string literal"));
+          && (identical(next, $LF) ||
+              identical(next, $CR) ||
+              identical(next, $EOF))) {
+        if (!asciiOnly) handleUnicode(start);
+        return error("unterminated string literal");
       }
+      if (next > 127) asciiOnly = false;
       next = advance();
     }
-    appendByteStringToken(STRING_INFO, utf8String(start, 0));
-    return advance();
+    if (!asciiOnly) handleUnicode(start);
+    // Advance past the quote character.
+    next = advance();
+    appendSubstringToken(STRING_INFO, start, asciiOnly);
+    return next;
   }
 
-  int tokenizeStringInterpolation(int start) {
-    appendByteStringToken(STRING_INFO, utf8String(start, -1));
+  int tokenizeStringInterpolation(int start, bool asciiOnly) {
+    appendSubstringToken(STRING_INFO, start, asciiOnly);
     beginToken(); // $ starts here.
     int next = advance();
     if (identical(next, $OPEN_CURLY_BRACKET)) {
-      return tokenizeInterpolatedExpression(next, start);
+      return tokenizeInterpolatedExpression(next);
     } else {
-      return tokenizeInterpolatedIdentifier(next, start);
+      return tokenizeInterpolatedIdentifier(next);
     }
   }
 
-  int tokenizeInterpolatedExpression(int next, int start) {
-    appendBeginGroup(STRING_INTERPOLATION_INFO, "\${");
+  int tokenizeInterpolatedExpression(int next) {
+    appendBeginGroup(STRING_INTERPOLATION_INFO);
     beginToken(); // The expression starts here.
-    next = advance();
+    next = advance(); // Move past the curly bracket.
     while (!identical(next, $EOF) && !identical(next, $STX)) {
       next = bigSwitch(next);
     }
     if (identical(next, $EOF)) return next;
-    next = advance();
+    next = advance();  // Move past the $STX.
     beginToken(); // The string interpolation suffix starts here.
     return next;
   }
 
-  int tokenizeInterpolatedIdentifier(int next, int start) {
+  int tokenizeInterpolatedIdentifier(int next) {
     appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO);
     beginToken(); // The identifier starts here.
     next = tokenizeKeywordOrIdentifier(next, false);
     beginToken(); // The string interpolation suffix starts here.
     return next;
   }
 
   int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
-    next = advance();
+    bool asciiOnly = true;
+    next = advance(); // Advance past the quote

[ngeoffray, 2013/10/18 10:19:37]

Missing .

[lukas, 2013/10/24 16:48:36]

Done.

     while (next != $EOF) {
       if (identical(next, quoteChar)) {
-        appendByteStringToken(STRING_INFO, utf8String(start, 0));
-        return advance();
+        if (!asciiOnly) handleUnicode(start);
+        next = advance();
+        appendSubstringToken(STRING_INFO, start, asciiOnly);
+        return next;
       } else if (identical(next, $LF) || identical(next, $CR)) {
-        return error(const SourceString("unterminated string literal"));
+        if (!asciiOnly) handleUnicode(start);
+        return error("unterminated string literal");
+      } else if (next > 127) {
+        asciiOnly = false;
       }
       next = advance();
     }
-    return error(const SourceString("unterminated string literal"));
+    if (!asciiOnly) handleUnicode(start);
+    return error("unterminated string literal");
   }
 
   int tokenizeMultiLineRawString(int quoteChar, int start) {
-    int next = advance();
+    bool asciiOnlyString = true;
+    bool asciiOnlyLine = true;
+    int unicodeStart = start;
+    int next = advance(); // Advance past the (last) quote (of three)

[ngeoffray, 2013/10/18 10:19:37]

Missing .

[lukas, 2013/10/24 16:48:36]

Done.

     outer: while (!identical(next, $EOF)) {
       while (!identical(next, quoteChar)) {
+        if (identical(next, $LF)) {
+          if (!asciiOnlyLine) {
+            // Synchronize the string offset in the utf8 scanner.
+            handleUnicode(unicodeStart);
+            asciiOnlyLine = true;
+            unicodeStart = scanOffset;
+          }
+          lineFeedInMultiline();
+        } else if (next > 127) {
+          asciiOnlyLine = false;
+          asciiOnlyString = false;
+        }
         next = advance();
         if (identical(next, $EOF)) break outer;
       }
       next = advance();
       if (identical(next, quoteChar)) {
         next = advance();
         if (identical(next, quoteChar)) {
-          appendByteStringToken(STRING_INFO, utf8String(start, 0));
-          return advance();
+          if (!asciiOnlyLine) handleUnicode(unicodeStart);
+          next = advance();
+          appendSubstringToken(STRING_INFO, start, asciiOnlyString);
+          return next;
         }
       }
     }
-    return error(const SourceString("unterminated string literal"));
+    if (!asciiOnlyLine) handleUnicode(unicodeStart);
+    return error("unterminated string literal");
   }
 
   int tokenizeMultiLineString(int quoteChar, int start, bool raw) {
     if (raw) return tokenizeMultiLineRawString(quoteChar, start);
-    int next = advance();
+    bool asciiOnlyString = true;
+    bool asciiOnlyLine = true;
+    int unicodeStart = start;
+    int next = advance(); // Advance past the (last) quote (of three).
     while (!identical(next, $EOF)) {
       if (identical(next, $$)) {
-        next = tokenizeStringInterpolation(start);
-        start = byteOffset;
+        if (!asciiOnlyLine) handleUnicode(unicodeStart);
+        next = tokenizeStringInterpolation(start, asciiOnlyString);
+        start = scanOffset;
+        unicodeStart = start;
+        asciiOnlyString = true; // A new string token is created for the rest.
+        asciiOnlyLine = true;
         continue;
       }
       if (identical(next, quoteChar)) {
         next = advance();
         if (identical(next, quoteChar)) {
           next = advance();
           if (identical(next, quoteChar)) {
-            appendByteStringToken(STRING_INFO, utf8String(start, 0));
-            return advance();
+            if (!asciiOnlyLine) handleUnicode(unicodeStart);
+            next = advance();
+            appendSubstringToken(STRING_INFO, start, asciiOnlyString);
+            return next;
           }
         }
         continue;
       }
       if (identical(next, $BACKSLASH)) {
         next = advance();
         if (identical(next, $EOF)) break;
       }
+      if (identical(next, $LF)) {
+        if (!asciiOnlyLine) {
+          // Synchronize the string offset in the utf8 scanner.
+          handleUnicode(unicodeStart);
+          asciiOnlyLine = true;
+          unicodeStart = scanOffset;
+        }
+        lineFeedInMultiline();
+      } else if (next > 127) {
+        asciiOnlyString = false;
+        asciiOnlyLine = false;
+      }
       next = advance();
     }
-    return error(const SourceString("unterminated string literal"));
+    if (!asciiOnlyLine) handleUnicode(unicodeStart);
+    return error("unterminated string literal");
   }
 
-  int error(SourceString message) {
-    appendByteStringToken(BAD_INPUT_INFO, message);
+  int error(String message) {
+    appendStringToken(BAD_INPUT_INFO, message);
     return advance(); // Ensure progress.
   }
+
+  void unmatchedBeginGroup(BeginGroupToken begin) {
+    String error = 'unmatched "${begin.stringValue}"';
+    Token close =
+        new StringToken.fromString(
+            BAD_INPUT_INFO, error, begin.charOffset, true);
+
+    // We want to ensure that unmatched BeginGroupTokens are reported
+    // as errors. However, the rest of the parser assume the groups

[ngeoffray, 2013/10/18 10:19:37]

assume -> assumes

[lukas, 2013/10/24 16:48:36]

Done.

+    // are well-balanced and will never look at the endGroup
+    // token. This is a nice property that allows us to skip quickly
+    // over correct code. By inserting an additional error token in
+    // the stream, we can keep ignoring endGroup tokens.
+    //
+    // [begin] --next--> [tail]
+    // [begin] --endG--> [close] --next--> [next] --next--> [tail]
+    //
+    // This allows the parser to skip from [begin] via endGroup to [close] and
+    // ignore the [close] token (assuming it's correct), then the error will be
+    // reported when parsing the [next] token.
+
+    Token next = new StringToken.fromString(
+        BAD_INPUT_INFO, error, begin.charOffset, true);
+    begin.endGroup = close;
+    close.next = next;
+    next.next = begin.next;
+  }
 }