Incorporates feedback by Nicolas for UTF-8 bytes based scanner CL

sdk/lib/_internal/compiler/implementation/scanner/array_based_scanner.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 abstract class ArrayBasedScanner extends AbstractScanner {
   ArrayBasedScanner(SourceFile file, bool includeComments)
       : super(file, includeComments);
 
   /**
    * The stack of open groups, e.g [: { ... ( .. :]
    * Each BeginGroupToken has a pointer to the token where the group
    * ends. This field is set when scanning the end group token.
    */
   Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>();
 
   /**
    * Appends a token whose kind is determined by [info] and content is defined
    * by the String [value].
    *
    * This method is invoked for class names, field names, method names, types,
    * etc.
    */
   void appendStringToken(PrecedenceInfo info, String value) {
-    tail.next = new StringToken.fromString(info, value, tokenStart, true);
+    tail.next = new StringToken.fromString(info, value, tokenStart,
+                                           canonicalize: true);
     tail = tail.next;
   }
 
   /**
    * Appends a fixed token whose kind and content is determined by [info].
    * Appends an *operator* token from [info].
    *
    * An operator token represent operators like ':', '.', ';', '&&', '==', '--',
    * '=>', etc.
    */
@@ skipping 52 matching lines @@
    * This method is used by the scanners to track line breaks and create the
    * [lineStarts] map.
    */
   void appendWhiteSpace(int next) {
     if (next == $LF && file != null) {
       lineStarts.add(stringOffset + 1); // +1, the line starts after the $LF.
     }
   }
 
   /**
-   * Notifies on [$LF] characters in multi-line commends or strings.
+   * Notifies on [$LF] characters in multi-line comments or strings.
    *
    * This method is used by the scanners to track line breaks and create the
    * [lineStarts] map.
    */
   void lineFeedInMultiline() {
     if (file != null) {
       lineStarts.add(stringOffset + 1);
     }
   }
 
   /**
    * Appends a token that begins a new group, represented by [value].
    * Group begin tokens are '{', '(', '[' and '${'.
    */
   void appendBeginGroup(PrecedenceInfo info) {
     Token token = new BeginGroupToken(info, tokenStart);
     tail.next = token;
     tail = tail.next;
 
     // { (  [ ${ cannot appear inside a type parameters / arguments.
     if (!identical(info.kind, LT_TOKEN)) discardOpenLt();
     groupingStack = groupingStack.prepend(token);
   }
 
   /**
-   * Appends a token that begins a ends group, represented by [value].
+   * Appends a token that begins an end group, represented by [value].
    * It handles the group end tokens '}', ')' and ']'. The tokens '>' and
    * '>>' are handled separately bo [appendGt] and [appendGtGt].
    */
   int appendEndGroup(PrecedenceInfo info, int openKind) {
     assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >>
     appendPrecedenceToken(info);
     // Don't report unmatched errors for <; it is also the less-than operator.
     discardOpenLt();
     if (groupingStack.isEmpty) {
       return advance();
@@ skipping 51 matching lines @@
       groupingStack = groupingStack.tail;
     }
   }
 
   void appendComment(start, bool asciiOnly) {
     if (!includeComments) return;
     appendSubstringToken(COMMENT_INFO, start, asciiOnly);
   }
 
   /**
-   * We call this method to discard '<' from the "grouping" stack
-   * (maintained by subclasses).
+   * This method is called to discard '<' from the "grouping" stack.
    *
    * [PartialParser.skipExpression] relies on the fact that we do not
    * create groups for stuff like:
    * [:a = b < c, d = e > f:].
    *
    * In other words, this method is called when the scanner recognizes
-   * something which cannot possibly be part of a type
-   * parameter/argument list.
+   * something which cannot possibly be part of a type parameter/argument
+   * list, like the '=' in the above example.
    */
   void discardOpenLt() {
     while (!groupingStack.isEmpty
         && identical(groupingStack.head.kind, LT_TOKEN)) {
       groupingStack = groupingStack.tail;
     }
   }
 }