Add chunked conversion to converters.

sdk/lib/convert/utf.dart

 // Copyright (c) 2013, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart.convert;
 
 /**
  * An instance of the default implementation of the [Utf8Codec].
  *
  * This instance provides a convenient access to the most common UTF-8
@@ skipping 44 matching lines @@
     return new Utf8Decoder(allowMalformed: allowMalformed).convert(codeUnits);
   }
 
   Converter<String, List<int>> get encoder => new Utf8Encoder();
   Converter<List<int>, String> get decoder {
     return new Utf8Decoder(allowMalformed: _allowMalformed);
   }
 }
 
 /**
- * A [Utf8Encoder] converts strings to their UTF-8 code units (a list of
+ * This class converts strings to their UTF-8 code units (a list of
  * unsigned 8-bit integers).
  */
 class Utf8Encoder extends Converter<String, List<int>> {
   /**
    * Converts [string] to its UTF-8 code units (a list of
    * unsigned 8-bit integers).
    */
   List<int> convert(String string) {
     // Create a new encoder with a length that is guaranteed to be big enough.
     // A single code unit uses at most 3 bytes. Two code units at most 4.
     _Utf8Encoder encoder = new _Utf8Encoder.withBufferSize(string.length * 3);
     int endPosition = encoder._fillBuffer(string, 0, string.length);
     assert(endPosition >= string.length - 1);
     if (endPosition != string.length) {
       int lastCodeUnit = string.codeUnitAt(string.length - 1);
       assert(_isLeadSurrogate(lastCodeUnit));
       // We use a non-surrogate as `nextUnit` so that _writeSurrogate just
       // writes the lead-surrogate.
       bool wasCombined = encoder._writeSurrogate(lastCodeUnit, 0);
       assert(!wasCombined);
     }
     return encoder._buffer.sublist(0, encoder._bufferIndex);
   }
+
+  StringConversionSink startChunkedConversion(ChunkedConversionSink sink) {
+    ByteConversionSink ByteSink = sink.adaptTo(outputInterface);
+    return new _Utf8EncoderSink(sink);
+  }
+
+  ChunkedConversionInterface get inputInterface =>
+      StringConversionSink.INTERFACE;
+  ChunkedConversionInterface get outputInterface =>
+      ByteConversionSink.INTERFACE;
 }
 
 /**
  * This class encodes Strings to UTF-8 code units (unsigned 8 bit integers).
  */
 // TODO(floitsch): make this class public.
 class _Utf8Encoder {
   int _carry = 0;
   int _bufferIndex = 0;
   final List<int> _buffer;
 
   static const _DEFAULT_BYTE_BUFFER_SIZE = 1024;
 
   _Utf8Encoder() : this.withBufferSize(_DEFAULT_BYTE_BUFFER_SIZE);
 
   _Utf8Encoder.withBufferSize(int bufferSize)
       // TODO(11971, floitsch): use Uint8List instead of normal lists.
       : _buffer = new List<int>(bufferSize);
 
   /**
    * Tries to combine the given [leadingSurrogate] with the [nextCodeUnit] and
    * writes it to [_buffer].
    *
    * Returns true if the [nextCodeUnit] was combined with the
-   * [leadingSurrogate]. If it wasn't then nextCodeUnit has not been written
-   * yet.
+   * [leadingSurrogate]. If it wasn't then nextCodeUnit was not a trailing
+   * surrogate and has not been written yet.
+   *
+   * It is safe to pass 0 for [nextCodeUnit] in which case only the leading
+   * surrogate is written.
    */
   bool _writeSurrogate(int leadingSurrogate, int nextCodeUnit) {
     if (_isTailSurrogate(nextCodeUnit)) {
       int rune = _combineSurrogatePair(leadingSurrogate, nextCodeUnit);
       // If the rune is encoded with 2 code-units then it must be encoded
       // with 4 bytes in UTF-8.
       assert(rune > _THREE_BYTE_LIMIT);
       assert(rune <= _FOUR_BYTE_LIMIT);
       _buffer[_bufferIndex++] = 0xF0 | (rune >> 18);
       _buffer[_bufferIndex++] = 0x80 | ((rune >> 12) & 0x3f);
@@ skipping 55 matching lines @@
           _buffer[_bufferIndex++] = 0x80 | ((rune >> 6) & 0x3f);
           _buffer[_bufferIndex++] = 0x80 | (rune & 0x3f);
         }
       }
     }
     return stringIndex;
   }
 }
 
 /**
+ * This class encodes chunked strings to UTF-8 code units (unsigned 8-bit
+ * integers).
+ */
+class _Utf8EncoderSink extends _Utf8Encoder with StringConversionSinkMixin {
+
+  final ByteConversionSink _sink;
+
+  _Utf8EncoderSink(this._sink);
+
+  void close() {
+    if (_carry != 0) {
+      // addSlice will call close again, but then the carry must be equal to 0.
+      addSlice("", 0, 0, true);
+      return;
+    }
+    _sink.close();
+  }
+
+  void addSlice(String str, int start, int end, bool isLast) {
+    _bufferIndex = 0;
+
+    if (start == end && !isLast) {
+      return;
+    }
+
+    if (_carry != 0) {
+      int nextCodeUnit = 0;
+      if (start != end) {
+        nextCodeUnit = str.codeUnitAt(start);
+      }  else {

[Søren Gjesse, 2013/07/24 09:26:41]

Two spaces before else.

[floitsch, 2013/07/24 18:31:15]

good eye.
done.

+        assert(isLast);
+      }
+      bool wasCombined = _writeSurrogate(_carry, nextCodeUnit);
+      // Either we got a non-empty string, or we must not have been combined.
+      assert(!wasCombined || start != end );
+      if (wasCombined) start++;
+      _carry = 0;
+    }
+    do {
+      start = _fillBuffer(str, start, end);
+      bool isLastSlice = isLast && (start == end);
+      if (start == end - 1 && _isLeadSurrogate(str.codeUnitAt(start))) {
+        if (isLast && _bufferIndex < _buffer.length - 3) {
+          // There is still space for the last incomplete surrogate.
+          // We use a non-surrogate as second argument. This way the
+          // function will just add the surrogate-half to the buffer.
+          bool hasBeenCombined = _writeSurrogate(str.codeUnitAt(start), 0);
+          assert(!hasBeenCombined);
+        } else {
+          // Otherwise store it in the carry. If [isLast] is true, then

[Søren Gjesse, 2013/07/24 09:26:41]

No need for markup in non-dartdoc comment.

[floitsch, 2013/07/24 18:31:15]

Done.

+          // close will flush the last carry.
+          _carry = str.codeUnitAt(start);
+        }
+        start++;
+      }
+      _sink.addSlice(_buffer, 0, _bufferIndex, isLastSlice);
+      _bufferIndex = 0;
+    } while (start < end);
+    if (isLast) close();
+  }
+
+  // TODO(floitsch): implement asUtf8Sink. Sligthly complicated because it
+  // needs to deal with malformed input.
+}
+
+/**
  * This class converts UTF-8 code units (lists of unsigned 8-bit integers)
  * to a string.
  */
 class Utf8Decoder extends Converter<List<int>, String> {
   final bool _allowMalformed;
 
   /**
    * Instantiates a new [Utf8Decoder].
    *
    * The optional [allowMalformed] argument defines how [convert] deals
    * with invalid or unterminated character sequences.
    *
    * If it is `true` [convert] replaces invalid (or unterminated) character
    * sequences with the Unicode Replacement character `U+FFFD` (�). Otherwise
    * it throws a [FormatException].
    */
   Utf8Decoder({ bool allowMalformed: false })
       : this._allowMalformed = allowMalformed;
 
   /**
    * Converts the UTF-8 [codeUnits] (a list of unsigned 8-bit integers) to the
    * corresponding string.
    */
   String convert(List<int> codeUnits) {
     StringBuffer buffer = new StringBuffer();
-    _Utf8Decoder decoder = new _Utf8Decoder(_allowMalformed);
-    decoder.convert(codeUnits, 0, codeUnits.length, buffer);
-    decoder.close(buffer);
+    _Utf8Decoder decoder = new _Utf8Decoder(buffer, _allowMalformed);
+    decoder.convert(codeUnits, 0, codeUnits.length);
+    decoder.close();
     return buffer.toString();
   }
+
+  ByteConversionSink startChunkedConversion(ChunkedConversionSink sink) {
+    StringConversionSink stringSink = sink.adaptTo(outputInterface);
+    return stringSink.asUtf8Sink(_allowMalformed);
+  }
+
+  ChunkedConversionInterface get inputInterface =>
+      ByteConversionSink.INTERFACE;
+  ChunkedConversionInterface get outputInterface =>
+      StringConversionSink.INTERFACE;
 }
 
 // UTF-8 constants.
-const int _ONE_BYTE_LIMIT = 0x7f;   // 7 bytes
-const int _TWO_BYTE_LIMIT = 0x7ff;  // 11 bytes
-const int _THREE_BYTE_LIMIT = 0xffff;  // 16 bytes
-const int _FOUR_BYTE_LIMIT = 0x10ffff;  // 21 bytes, truncated to Unicode max.
+const int _ONE_BYTE_LIMIT = 0x7f;   // 7 bits
+const int _TWO_BYTE_LIMIT = 0x7ff;  // 11 bits
+const int _THREE_BYTE_LIMIT = 0xffff;  // 16 bits
+const int _FOUR_BYTE_LIMIT = 0x10ffff;  // 21 bits, truncated to Unicode max.
 
 // UTF-16 constants.
 const int _SURROGATE_MASK = 0xF800;
 const int _SURROGATE_TAG_MASK = 0xFC00;
 const int _SURROGATE_VALUE_MASK = 0x3FF;
 const int _LEAD_SURROGATE_MIN = 0xD800;
 const int _TAIL_SURROGATE_MIN = 0xDC00;
 
 const int _REPLACEMENT_CHARACTER = 0xFFFD;
 const int _BOM_CHARACTER = 0xFEFF;
@@ skipping 10 matching lines @@
 
 
 /**
  * Decodes UTF-8.
  *
  * The decoder handles chunked input.
  */
 // TODO(floitsch): make this class public.
 class _Utf8Decoder {
   final bool _allowMalformed;
+  final StringSink _stringSink;
   bool _isFirstCharacter = true;
   int _value = 0;
   int _expectedUnits = 0;
   int _extraUnits = 0;
 
-  _Utf8Decoder(this._allowMalformed);
+  _Utf8Decoder(this._stringSink, this._allowMalformed);
 
   bool get hasPartialInput => _expectedUnits > 0;
 
   // Limits of one through four byte encodings.
   static const List<int> _LIMITS = const <int>[
       _ONE_BYTE_LIMIT,
       _TWO_BYTE_LIMIT,
       _THREE_BYTE_LIMIT,
       _FOUR_BYTE_LIMIT ];
 
-  void close(StringSink sink) {
+  void close() {
+    flush();
+  }
+
+  /**
+   * Flushes this decoder as if closed.
+   *
+   * This method throws if the input was partial and the decoder was
+   * constructed with `allowMalformed` set to `false`.
+   */
+  void flush() {
     if (hasPartialInput) {
       if (!_allowMalformed) {
         throw new FormatException("Unfinished UTF-8 octet sequence");
       }
-      sink.writeCharCode(_REPLACEMENT_CHARACTER);
+      _stringSink.writeCharCode(_REPLACEMENT_CHARACTER);
+      _value = 0;
+      _expectedUnits = 0;
+      _extraUnits = 0;
     }
   }
 
-  void convert(List<int> codeUnits, int startIndex, int endIndex,
-               StringSink sink) {
+  void convert(List<int> codeUnits, int startIndex, int endIndex) {
     int value = _value;
     int expectedUnits = _expectedUnits;
     int extraUnits = _extraUnits;
     _value = 0;
     _expectedUnits = 0;
     _extraUnits = 0;
 
     int i = startIndex;
     loop: while (true) {
       multibyte: if (expectedUnits > 0) {
         do {
           if (i == endIndex) {
             break loop;
           }
           int unit = codeUnits[i];
           if ((unit & 0xC0) != 0x80) {
             expectedUnits = 0;
             if (!_allowMalformed) {
               throw new FormatException(
                   "Bad UTF-8 encoding 0x${unit.toRadixString(16)}");
             }
             _isFirstCharacter = false;
-            sink.writeCharCode(_REPLACEMENT_CHARACTER);
+            _stringSink.writeCharCode(_REPLACEMENT_CHARACTER);
             break multibyte;
           } else {
             value = (value << 6) | (unit & 0x3f);
             expectedUnits--;
             i++;
           }
         } while (expectedUnits > 0);
         if (value <= _LIMITS[extraUnits - 1]) {
           // Overly long encoding. The value could be encoded with a shorter
           // encoding.
           if (!_allowMalformed) {
             throw new FormatException(
                 "Overlong encoding of 0x${value.toRadixString(16)}");
           }
           expectedUnits = extraUnits = 0;
           value = _REPLACEMENT_CHARACTER;
         }
         if (value > _FOUR_BYTE_LIMIT) {
           if (!_allowMalformed) {
             throw new FormatException("Character outside valid Unicode range: "
                                       "0x${value.toRadixString(16)}");
           }
           value = _REPLACEMENT_CHARACTER;
         }
         if (!_isFirstCharacter || value != _BOM_CHARACTER) {
-          sink.writeCharCode(value);
+          _stringSink.writeCharCode(value);
         }
         _isFirstCharacter = false;
       }
 
       while (i < endIndex) {
         int unit = codeUnits[i++];
         if (unit <= _ONE_BYTE_LIMIT) {
           _isFirstCharacter = false;
-          sink.writeCharCode(unit);
+          _stringSink.writeCharCode(unit);
         } else {
           if ((unit & 0xE0) == 0xC0) {
             value = unit & 0x1F;
             expectedUnits = extraUnits = 1;
             continue loop;
           }
           if ((unit & 0xF0) == 0xE0) {
             value = unit & 0x0F;
             expectedUnits = extraUnits = 2;
             continue loop;
           }
           // 0xF5, 0xF6 ... 0xFF never appear in valid UTF-8 sequences.
           if ((unit & 0xF8) == 0xF0 && unit < 0xF5) {
             value = unit & 0x07;
             expectedUnits = extraUnits = 3;
             continue loop;
           }
           if (!_allowMalformed) {
             throw new FormatException(
                 "Bad UTF-8 encoding 0x${unit.toRadixString(16)}");
           }
           value = _REPLACEMENT_CHARACTER;
           expectedUnits = extraUnits = 0;
           _isFirstCharacter = false;
-          sink.writeCharCode(value);
+          _stringSink.writeCharCode(value);
         }
       }
       break loop;
     }
     if (expectedUnits > 0) {
       _value = value;
       _expectedUnits = expectedUnits;
       _extraUnits = extraUnits;
     }
   }
 }