Avoiding creating strings for canonicalization

pkg/front_end/lib/src/fasta/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.token;
 
 import 'dart:collection' show
     HashSet;
 
 import 'dart:convert' show
     UTF8;
 
 import 'keyword.dart' show
     Keyword;
 
 import 'precedence.dart' show
     BAD_INPUT_INFO,
     EOF_INFO,
     PrecedenceInfo;
 
 import 'token_constants.dart' show
     IDENTIFIER_TOKEN;
 
+import 'canonicalizer.dart';
+
 /**
  * A token that doubles as a linked list.
  */
 abstract class Token {
   /**
    * The character offset of the start of this token within the source text.
    */
   final int charOffset;
 
   Token(this.charOffset);
@@ skipping 154 matching lines @@
   var /* String | LazySubtring */ valueOrLazySubstring;
 
   final PrecedenceInfo info;
 
   /**
    * Creates a non-lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromString(this.info, String value, int charOffset,
       {bool canonicalize: false})
-      : valueOrLazySubstring = canonicalizedString(value, canonicalize),
+      : valueOrLazySubstring = canonicalizedString(value, 
+          0, value.length, canonicalize),
         super(charOffset);
 
   /**
    * Creates a lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromSubstring(
       this.info, String data, int start, int end, int charOffset,
       {bool canonicalize: false})
       : super(charOffset) {
     int length = end - start;
     if (length <= LAZY_THRESHOLD) {
       valueOrLazySubstring =
-          canonicalizedString(data.substring(start, end), canonicalize);
+          canonicalizedString(data, start, end, canonicalize);
     } else {
       valueOrLazySubstring =
           new LazySubstring(data, start, length, canonicalize);
     }
   }
 
   /**
    * Creates a lazy string token. If [asciiOnly] is false, the byte array
    * is passed through a UTF-8 decoder.
    */
@@ skipping 11 matching lines @@
   String get value {
     if (valueOrLazySubstring is String) {
       return valueOrLazySubstring;
     } else {
       assert(valueOrLazySubstring is LazySubstring);
       var data = valueOrLazySubstring.data;
       int start = valueOrLazySubstring.start;
       int end = start + valueOrLazySubstring.length;
       if (data is String) {
         valueOrLazySubstring = canonicalizedString(
-            data.substring(start, end), valueOrLazySubstring.boolValue);
+            data, start, end, valueOrLazySubstring.boolValue);
       } else {
         valueOrLazySubstring =
             decodeUtf8(data, start, end, valueOrLazySubstring.boolValue);
       }
       return valueOrLazySubstring;
     }
   }
 
   /// See [Token.stringValue] for an explanation.
   String get stringValue => null;
 
   bool isIdentifier() => identical(kind, IDENTIFIER_TOKEN);
 
   String toString() => "StringToken($value)";
 
-  static final HashSet<String> canonicalizedSubstrings = new HashSet<String>();
+  static final StringCanonicalizer canonicalizer = new StringCanonicalizer();
 
-  static String canonicalizedString(String s, bool canonicalize) {
+  static String canonicalizedString(String s, int start, int end, bool canonicalize) {

[ahe, 2017/02/14 14:35:04]

Long line.

     if (!canonicalize) return s;
-    var result = canonicalizedSubstrings.lookup(s);
-    if (result != null) return result;
-    canonicalizedSubstrings.add(s);
-    return s;
+    return canonicalizer.canonicalize(s, start, end, false);
   }
 
   static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) {
-    var s;
-    if (asciiOnly) {
-      s = new String.fromCharCodes(data, start, end);
-    } else {
-      s = UTF8.decoder.convert(data, start, end);
-    }
-    return canonicalizedString(s, true);
+    return canonicalizer.canonicalize(data, start, end, asciiOnly);
   }
 }
 
 /**
  * This class represents the necessary information to compute a substring
  * lazily. The substring can either originate from a string or from
  * a [:List<int>:] of UTF-8 bytes.
  */
 abstract class LazySubstring {
   /** The original data, either a string or a List<int> */
@@ skipping 79 matching lines @@
       value == '<=' ||
       value == '<' ||
       value == '&' ||
       value == '^' ||
       value == '|';
 }
 
 bool isTernaryOperator(String value) => value == '[]=';
 
 bool isMinusOperator(String value) => value == '-';