Replace old utf8 decoder with new one.

tests/lib/convert/utf82_test.dart

Index: tests/lib/convert/utf82_test.dart
diff --git a/tests/lib/convert/utf82_test.dart b/tests/lib/convert/utf82_test.dart
new file mode 100644
index 0000000000000000000000000000000000000000..36cecfc7a50a5160569eb4c44736f402bc3371e0
--- /dev/null
+++ b/tests/lib/convert/utf82_test.dart
@@ -0,0 +1,70 @@
+// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library utf8_test;
+import "package:expect/expect.dart";
+import 'dart:convert';
+import 'dart:codec';
+
+String decode(List<int> bytes) => new Utf8Decoder().convert(bytes);
+String decodeAllowMalformed(List<int> bytes) {
+  return new Utf8Decoder(allowMalformed: true).convert(bytes);
+}
+
+String decode2(List<int> bytes) => UTF8.decode(bytes);
+String decodeAllowMalformed2(List<int> bytes) {
+  return UTF8.decode(bytes, allowMalformed: true);
+}
+
+String decode3(List<int> bytes) => new Utf8Codec().decode(bytes);
+String decodeAllowMalformed3(List<int> bytes) {
+  return new Utf8Codec(allowMalformed: true).decode(bytes);
+}
+
+String decode4(List<int> bytes) => new Utf8Codec().decoder.convert(bytes);
+String decodeAllowMalformed4(List<int> bytes) {
+  return new Utf8Codec(allowMalformed: true).decoder.convert(bytes);
+}
+
+final TESTS = [
+  // Unfinished UTF-8 sequences.
+  [ 0xc3 ],
+  [ 0xE2, 0x82 ],
+  [ 0xF0, 0xA4, 0xAD ],
+  // Overlong encoding of euro-sign.
+  [ 0xF0, 0x82, 0x82, 0xAC ]];

[Lasse Reichstein Nielsen, 2013/07/16 12:23:03]

Test with "[0x80, 0x80]" too.
And decide what happens. I'd make it one invalid sequence (which is what it
looks like if you are moving backwards - it's an uninitiated encoding, not
unterminated).

How about overlong encodings? (e.g., [0xC0, 0x80])?

And how about encoded surrogates?

[floitsch, 2013/07/16 14:25:24]

I prefer making each one a separate error.
Otherwise it is not clear what 0x80, 0x80, 0x80 should be.
There could be 1, 2, or 3 missing bytes.
done.
I would prefer not to check those. But I can if you prefer.

[Lasse Reichstein Nielsen, 2013/07/17 09:14:50]

The point is that UTF-8 can be traversed both forwards and backwards.
If you traverse it backwards, you will pass all three 0x80's before realizing
that something is missing. At that point, it would make sense to make them one
error. I'd do the same when moving forward.

+
+main() {
+  var allTests = TESTS.expand((test) {
+    // Pairs of test and expected string output when malformed strings are
+    // allowed. Replacement character: 0xFFFD
+    return [[ test, "\u{FFFD}" ],
+            [ new List.from([0x61])..addAll(test), "a\u{FFFD}" ],
+            [ new List.from([0x61])..addAll(test)..add(0x61), "a\u{FFFD}a" ],
+            [ new List.from(test)..add(0x61), "\u{FFFD}a" ],
+            [ new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}" ],
+            [ new List.from(test)..add(0x61)..addAll(test),
+              "\u{FFFD}a\u{FFFD}" ],
+            [ new List.from([0xc3, 0xa5])..addAll(test), "å\u{FFFD}" ],
+            [ new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]),
+              "å\u{FFFD}å" ],
+            [ new List.from(test)..addAll([0xc3, 0xa5]), "\u{FFFD}å" ],
+            [ new List.from(test)..addAll([0xc3, 0xa5])..addAll(test),
+              "\u{FFFD}å\u{FFFD}" ]];
+  });
+
+  for (var test in allTests) {
+    List<int> bytes = test[0];
+    Expect.throws(() => decode(bytes), (e) => e is FormatException);
+    Expect.throws(() => decode2(bytes), (e) => e is FormatException);
+    Expect.throws(() => decode3(bytes), (e) => e is FormatException);
+    Expect.throws(() => decode4(bytes), (e) => e is FormatException);
+
+    String expected = test[1];
+    Expect.equals(expected, decodeAllowMalformed(bytes));
+    Expect.equals(expected, decodeAllowMalformed2(bytes));
+    Expect.equals(expected, decodeAllowMalformed3(bytes));
+    Expect.equals(expected, decodeAllowMalformed4(bytes));
+  }
+}