Improve streaming UTF-8 decoder

sdk/lib/io/string_stream.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 // Interface for decoders decoding binary data into string data. The
 // decoder keeps track of line breaks during decoding.
 abstract class _StringDecoder {
   // Add more binary data to be decoded. The ownership of the buffer
   // is transfered to the decoder and the caller most not modify it any more.
   int write(List<int> buffer);
@@ skipping 157 matching lines @@
   Function onError;
 
   final int LF = 10;
   final int CR = 13;
 }
 
 
 // Utility class for decoding UTF-8 from data delivered as a stream of
 // bytes.
 class _UTF8Decoder extends _StringDecoderBase {
+  static const kMaxCodePoint = 0x10FFFF;
+  static const kReplacementCodePoint = 0x3f;
+
+  void _reportError(error) {
+    if (onError != null) {
+      onError(error);
+      return false;
+    } else {
+      throw error;
+    }
+  }
+
   // Process the next UTF-8 encoded character.
   bool _processNext() {
     // Peek the next byte to calculate the number of bytes required for
     // the next character.
     int value = _bufferList.peek() & 0xFF;
     if ((value & 0x80) == 0x80) {
       int additionalBytes;
       if ((value & 0xe0) == 0xc0) {  // 110xxxxx
         value = value & 0x1F;
         additionalBytes = 1;
       } else if ((value & 0xf0) == 0xe0) {  // 1110xxxx
         value = value & 0x0F;
         additionalBytes = 2;
-      } else {  // 11110xxx
+      } else if ((value & 0xf8) == 0xf0) {  // 11110xxx
         value = value & 0x07;
         additionalBytes = 3;
+      } else if ((value & 0xfc) == 0xf8) {  // 111110xx
+        value = value & 0x03;
+        additionalBytes = 4;
+      } else if ((value & 0xfe) == 0xfc) {  // 1111110x
+        value = value & 0x01;
+        additionalBytes = 5;
+      } else {
+        _reportError(new DecoderException("Illegal UTF-8"));
       }
       // Check if there are enough bytes to decode the character. Otherwise
       // return false.
       if (_bufferList.length < additionalBytes + 1) {
         return false;
       }
       // Remove the value peeked from the buffer list.
       _bufferList.next();
       for (int i = 0; i < additionalBytes; i++) {
         int byte = _bufferList.next();
+        if ((byte & 0xc0) != 0x80) {
+          _reportError(new DecoderException("Illegal UTF-8"));
+        }
         value = value << 6 | (byte & 0x3F);
       }
     } else {
       // Remove the value peeked from the buffer list.
       _bufferList.next();
     }
-    addChar(value);
+    print(value.toRadixString(16));

[Mads Ager (google), 2012/12/07 11:43:21]

hovsa. ;-)

+    if (value > kMaxCodePoint) {
+      addChar(kReplacementCodePoint);
+    } else {
+      addChar(value);
+    }
     return true;
   }
 }
 
 
 // Utility class for decoding ascii data delivered as a stream of
 // bytes.
 class _AsciiDecoder extends _StringDecoderBase {
   // Process the next ascii encoded character.
   bool _processNext() {
@@ skipping 287 matching lines @@
   bool _inputClosed = false;  // Is the underlying input stream closed?
   bool _closed = false;  // Is this stream closed.
   bool _eof = false;  // Has all data been read from the decoder?
   Timer _scheduledDataCallback;
   Timer _scheduledLineCallback;
   Timer _scheduledCloseCallback;
   Function _clientDataHandler;
   Function _clientLineHandler;
   Function _clientCloseHandler;
 }