Fix UTF8 encoder for Unicode runes > 0xFFFF.

sdk/lib/convert/utf.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart.convert;
 
 /** The Unicode Replacement character `U+FFFD` (�). */
 const UNICODE_REPLACEMENT_CHARACTER_RUNE = 0xFFFD;
 
+/** The Unicode Byte Order Marker (BOM) character `U+FEFF`. */
+const UNICODE_BOM_CHARACTER_RUNE = 0xFEFF;
+
 /**
  * An instance of the default implementation of the [Utf8Codec].
  *
  * This instance provides a convenient access to the most common UTF-8
  * use cases.
  *
  * Examples:
  *
  *     var encoded = UTF8.encode("Îñţérñåţîöñåļîžåţîờñ");
  *     var decoded = UTF8.decode([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6,
@@ skipping 21 matching lines @@
    */
   const Utf8Codec({ bool allowMalformed: false })
       : _allowMalformed = allowMalformed;
 
   String get name => "utf-8";
 
   /**
    * Decodes the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
    * corresponding string.
    *
+   * If the [codeUnits] start with a leading [UNICODE_BOM_CHARACTER_RUNE] this

[Anders Johnsen, 2013/10/01 11:32:29]

Is this really happening? I don't see any usage of 'UNICODE_BOM_CHARACTER_RUNE'.

[floitsch, 2013/10/01 11:37:42]

There was a private copy of it. I removed that one.

+   * character is discarded.
+   *
    * If [allowMalformed] is `true` the decoder replaces invalid (or
    * unterminated) character sequences with the Unicode Replacement character
    * `U+FFFD` (�). Otherwise it throws a [FormatException].
    *
    * If [allowMalformed] is not given, it defaults to the `allowMalformed` that
    * was used to instantiate `this`.
    */
   String decode(List<int> codeUnits, { bool allowMalformed }) {
     if (allowMalformed == null) allowMalformed = _allowMalformed;
     return new Utf8Decoder(allowMalformed: allowMalformed).convert(codeUnits);
@@ skipping 235 matching lines @@
    * If it is `true` [convert] replaces invalid (or unterminated) character
    * sequences with the Unicode Replacement character `U+FFFD` (�). Otherwise
    * it throws a [FormatException].
    */
   const Utf8Decoder({ bool allowMalformed: false })
       : this._allowMalformed = allowMalformed;
 
   /**
    * Converts the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
    * corresponding string.
+   *
+   * If the [codeUnits] start with a leading [UNICODE_BOM_CHARACTER_RUNE] this
+   * character is discarded.
    */
   String convert(List<int> codeUnits) {
     StringBuffer buffer = new StringBuffer();
     _Utf8Decoder decoder = new _Utf8Decoder(buffer, _allowMalformed);
     decoder.convert(codeUnits, 0, codeUnits.length);
     decoder.close();
     return buffer.toString();
   }
 
   /**
@@ skipping 33 matching lines @@
 const int _REPLACEMENT_CHARACTER = 0xFFFD;
 const int _BOM_CHARACTER = 0xFEFF;
 
 bool _isSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_MASK) == _LEAD_SURROGATE_MIN;
 bool _isLeadSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_TAG_MASK) == _LEAD_SURROGATE_MIN;
 bool _isTailSurrogate(int codeUnit) =>
     (codeUnit & _SURROGATE_TAG_MASK) == _TAIL_SURROGATE_MIN;
 int _combineSurrogatePair(int lead, int tail) =>
-    0x10000 | ((lead & _SURROGATE_VALUE_MASK) << 10)
+    0x10000 + ((lead & _SURROGATE_VALUE_MASK) << 10)
             | (tail & _SURROGATE_VALUE_MASK);
 
 
 /**
  * Decodes UTF-8.
  *
  * The decoder handles chunked input.
  */
 // TODO(floitsch): make this class public.
 class _Utf8Decoder {
@@ skipping 137 matching lines @@
       }
       break loop;
     }
     if (expectedUnits > 0) {
       _value = value;
       _expectedUnits = expectedUnits;
       _extraUnits = extraUnits;
     }
   }
 }