Incorporates feedback by Nicolas for UTF-8 bytes based scanner CL

sdk/lib/_internal/compiler/implementation/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 const int EOF_TOKEN = 0;
 
 const int KEYWORD_TOKEN = $k;
 const int IDENTIFIER_TOKEN = $a;
@@ skipping 85 matching lines @@
    * The precedence info for this token. [info] determines the kind and the
    * precedence level of this token.
    *
    * Defined as getter to save a field in the [KeywordToken] subclass.
    */
   PrecedenceInfo get info;
 
   /**
    * The string represented by this token, a substring of the source code.
    *
-   * For [StringToken]s the value includes the quotes, explicit escapes, etc.
-   *
+   * For [StringToken]s the [value] includes the quotes, explicit escapes, etc.
    */
   String get value;
 
   /**
-   * For symbol and keyword tokens, returns the string value reprenseted by this
+   * For symbol and keyword tokens, returns the string value represented by this
    * token. For [StringToken]s this method returns [:null:].
    *
    * For [SymbolToken]s and [KeywordToken]s, the string value is a compile-time
    * constant originating in the [PrecedenceInfo] or in the [Keyword] instance.
    * This allows testing for keywords and symbols using [:identical:], e.g.,
    * [:identical('class', token.value):].
    *
    * Note that returning [:null:] for string tokens is important to identify
    * symbols and keywords, we cannot use [value] instead. The string literal
    *   "$a($b"
@@ skipping 10 matching lines @@
    */
   int get kind => info.kind;
 
   /**
    * The precedence level for this token.
    */
   int get precedence => info.precedence;
 
   /**
    * True if this token is an identifier. Some keywords allowed as identifiers,
-   * see implementaiton in [KeywordToken].
+   * see implementation in [KeywordToken].
    */
   bool isIdentifier();
 
   /**
    * Returns a textual representation of this token to be used for debugging
    * purposes. The resulting string might contain information about the
    * structure of the token, for example 'StringToken(foo)' for the identifier
    * token 'foo'.
    *
    * Use [value] for the text actually parsed by the token.
@@ skipping 10 matching lines @@
       return 1;
     } else {
       return value.length;
     }
   }
 
   int get hashCode => computeHashCode(charOffset, info, value);
 }
 
 /**
- * A symbol token represents the symbol in its precendence info.
+ * A [SymbolToken] represents the symbol in its precendence info.
  * Also used for end of file with EOF_INFO.
  */
 class SymbolToken extends Token {
 
   final PrecedenceInfo info;
 
   SymbolToken(this.info, int charOffset) : super(charOffset);
 
   String get value => info.value;
 
   String get stringValue => info.value;
 
   bool isIdentifier() => false;
 
   String toString() => "SymbolToken($value)";
 }
 
 /**
- * A [BeginGroupToken] reprsents a symbol that may be the beginning of
+ * A [BeginGroupToken] represents a symbol that may be the beginning of
  * a pair of brackets, i.e., ( { [ < or ${
  * The [endGroup] token points to the matching closing bracked in case
  * it can be identified during scanning.
  */
 class BeginGroupToken extends SymbolToken {
   Token endGroup;
 
   BeginGroupToken(PrecedenceInfo info, int charOffset)
-    : super(info, charOffset);
+      : super(info, charOffset);
 }
 
 /**
  * A keyword token.
  */
 class KeywordToken extends Token {
   final Keyword keyword;
 
   KeywordToken(this.keyword, int charOffset) : super(charOffset);
 
   PrecedenceInfo get info => keyword.info;
 
   String get value => keyword.syntax;
 
   String get stringValue => keyword.syntax;
 
   bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn;
 
   String toString() => "KeywordToken($value)";
 }
 
 /**
  * A String-valued token. Represents identifiers, string literals,
- * number literals, comments and error tokens, using the corresponding
+ * number literals, comments, and error tokens, using the corresponding
  * precedence info.
  */
 class StringToken extends Token {
   /**
    * The length threshold above which substring tokens are computed lazily.
    *
    * For string tokens that are substrings of the program source, the actual
    * substring extraction is performed lazily. This is beneficial because
    * not all scanned code is actually used. For unused parts, the substrings
    * are never computed and allocated.
    */
   static const int LAZY_THRESHOLD = 4;
 
-  var valueOrLazySubstring;
+  var /* String | LazySubtring */ valueOrLazySubstring;
 
   final PrecedenceInfo info;
 
   /**
    * Creates a non-lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromString(this.info, String value, int charOffset,
-                         [bool canonicalize = false])
+                         {bool canonicalize : false})
       : valueOrLazySubstring = canonicalizedString(value, canonicalize),
         super(charOffset);
 
   /**
    * Creates a lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromSubstring(this.info, String data, int start, int end,
-                            int charOffset, [bool canonicalize = false])
+                            int charOffset, {bool canonicalize : false})
       : super(charOffset) {
     int length = end - start;
     if (length <= LAZY_THRESHOLD) {
       valueOrLazySubstring = canonicalizedString(data.substring(start, end),
-                                            canonicalize);
+                                                 canonicalize);
     } else {
       valueOrLazySubstring =
           new LazySubstring(data, start, length, canonicalize);
     }
   }
 
   /**
    * Creates a lazy string token. If [asciiOnly] is false, the byte array
    * is passed through a UTF-8 decoder.
    */
@@ skipping 27 matching lines @@
     }
   }
 
   String get stringValue => null;
 
   bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN);
 
   String toString() => "StringToken($value)";
 
   static final HashSet<String> canonicalizedSubstrings =
-      new HashSet();
+      new HashSet<String>();
 
   static String canonicalizedString(String s, bool canonicalize) {
     if (!canonicalize) return s;
     var result = canonicalizedSubstrings.lookup(s);
     if (result != null) return result;
     canonicalizedSubstrings.add(s);
     return s;
   }
 
   static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) {
     var s;
     if (asciiOnly) {
       // getRange returns an iterator, it does not copy the data.
       s = new String.fromCharCodes(data.getRange(start, end));
     } else {
       // TODO(lry): this is measurably slow. Also sublist is copied eagerly.
       var bytes = data.sublist(start, end);
       s = UTF8.decode(bytes);
     }
     return canonicalizedString(s, true);
   }
 }
 
 /**
  * This class represents the necessary information to compute a substring
- * lazily. The substring can either originate in a string or in a [:List<int>:]
- * of UTF-8 bytes.
+ * lazily. The substring can either originate from a string or from
+ * a [:List<int>:] of UTF-8 bytes.
  */
 abstract class LazySubstring {
   /** The original data, either a string or a List<int> */
   get data;
 
   int get start;
   int get length;
 
   /**
-   * If this substring is based on a String, the boolean indicates wheter the
-   * resulting substring should be canonicalized.
+   * If this substring is based on a String, the [boolValue] indicates wheter
+   * the resulting substring should be canonicalized.
    *
-   * For substrings based on a byte array, the boolean value is true if the
+   * For substrings based on a byte array, the [boolValue] is true if the
    * array only holds ASCII characters. The resulting substring will be
    * canonicalized after decoding.
    */
   bool get boolValue;
 
   LazySubstring.internal();
 
   factory LazySubstring(data, int start, int length, bool b) {
     // See comment on [CompactLazySubstring].
     if (start < 0x100000 && length < 0x200) {
       int fields = (start << 9);
       fields = fields | length;
       fields = fields << 1;
       if (b) fields |= 1;
       return new CompactLazySubstring(data, fields);
     } else {
       return new FullLazySubstring(data, start, length, b);
     }
   }
 }
 
 /**
  * This class encodes [start], [length] and [boolValue] in a single
  * 30 bit integer. It uses 20 bits for [start], which covers source files
- * of 1M. [length] has 9 bits, which covers 512 characters.
+ * of 1MB. [length] has 9 bits, which covers 512 characters.
  *
- * The file html_dart2js.dart is currently around 1M.
+ * The file html_dart2js.dart is currently around 1MB.
  */
 class CompactLazySubstring extends LazySubstring {
   final data;
   final int fields;
 
   CompactLazySubstring(this.data, this.fields) : super.internal();
 
   int get start => fields >> 10;
   int get length => (fields >> 1) & 0x1ff;
   bool get boolValue => (fields & 1) == 1;
@@ skipping 269 matching lines @@
 
 const PrecedenceInfo STRING_INTERPOLATION_IDENTIFIER_INFO =
   const PrecedenceInfo('\$', 0,
                        STRING_INTERPOLATION_IDENTIFIER_TOKEN);
 
 const PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo('hexadecimal', 0, HEXADECIMAL_TOKEN);
 
 const PrecedenceInfo COMMENT_INFO =
   const PrecedenceInfo('comment', 0, COMMENT_TOKEN);