| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library utf8_test; | 5 library utf8_test; |
| 6 |
| 6 import "package:expect/expect.dart"; | 7 import "package:expect/expect.dart"; |
| 7 import 'dart:convert'; | 8 import 'dart:convert'; |
| 8 | 9 |
| 9 String decode(List<int> bytes) => new Utf8Decoder().convert(bytes); | 10 String decode(List<int> bytes) => new Utf8Decoder().convert(bytes); |
| 10 String decodeAllowMalformed(List<int> bytes) { | 11 String decodeAllowMalformed(List<int> bytes) { |
| 11 return new Utf8Decoder(allowMalformed: true).convert(bytes); | 12 return new Utf8Decoder(allowMalformed: true).convert(bytes); |
| 12 } | 13 } |
| 13 | 14 |
| 14 String decode2(List<int> bytes) => UTF8.decode(bytes); | 15 String decode2(List<int> bytes) => UTF8.decode(bytes); |
| 15 String decodeAllowMalformed2(List<int> bytes) { | 16 String decodeAllowMalformed2(List<int> bytes) { |
| 16 return UTF8.decode(bytes, allowMalformed: true); | 17 return UTF8.decode(bytes, allowMalformed: true); |
| 17 } | 18 } |
| 18 | 19 |
| 19 String decode3(List<int> bytes) => new Utf8Codec().decode(bytes); | 20 String decode3(List<int> bytes) => new Utf8Codec().decode(bytes); |
| 20 String decodeAllowMalformed3(List<int> bytes) { | 21 String decodeAllowMalformed3(List<int> bytes) { |
| 21 return new Utf8Codec(allowMalformed: true).decode(bytes); | 22 return new Utf8Codec(allowMalformed: true).decode(bytes); |
| 22 } | 23 } |
| 23 | 24 |
| 24 String decode4(List<int> bytes) => new Utf8Codec().decoder.convert(bytes); | 25 String decode4(List<int> bytes) => new Utf8Codec().decoder.convert(bytes); |
| 25 String decodeAllowMalformed4(List<int> bytes) { | 26 String decodeAllowMalformed4(List<int> bytes) { |
| 26 return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); | 27 return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); |
| 27 } | 28 } |
| 28 | 29 |
| 29 final TESTS = [ | 30 final TESTS = [ |
| 30 // Unfinished UTF-8 sequences. | 31 // Unfinished UTF-8 sequences. |
| 31 [ 0xc3 ], | 32 [0xc3], |
| 32 [ 0xE2, 0x82 ], | 33 [0xE2, 0x82], |
| 33 [ 0xF0, 0xA4, 0xAD ], | 34 [0xF0, 0xA4, 0xAD], |
| 34 // Overlong encoding of euro-sign. | 35 // Overlong encoding of euro-sign. |
| 35 [ 0xF0, 0x82, 0x82, 0xAC ], | 36 [0xF0, 0x82, 0x82, 0xAC], |
| 36 // Other overlong/unfinished sequences. | 37 // Other overlong/unfinished sequences. |
| 37 [ 0xC0 ], | 38 [0xC0], |
| 38 [ 0xC1 ], | 39 [0xC1], |
| 39 [ 0xF5 ], | 40 [0xF5], |
| 40 [ 0xF6 ], | 41 [0xF6], |
| 41 [ 0xF7 ], | 42 [0xF7], |
| 42 [ 0xF8 ], | 43 [0xF8], |
| 43 [ 0xF9 ], | 44 [0xF9], |
| 44 [ 0xFA ], | 45 [0xFA], |
| 45 [ 0xFB ], | 46 [0xFB], |
| 46 [ 0xFC ], | 47 [0xFC], |
| 47 [ 0xFD ], | 48 [0xFD], |
| 48 [ 0xFE ], | 49 [0xFE], |
| 49 [ 0xFF ], | 50 [0xFF], |
| 50 [ 0xC0, 0x80 ], | 51 [0xC0, 0x80], |
| 51 [ 0xC1, 0x80 ], | 52 [0xC1, 0x80], |
| 52 // Outside valid range. | 53 // Outside valid range. |
| 53 [ 0xF4, 0xBF, 0xBF, 0xBF ], | 54 [0xF4, 0xBF, 0xBF, 0xBF], |
| 54 // Negative | 55 // Negative |
| 55 [ -0x01 ], | 56 [-0x01], |
| 56 [ -0xFF ], | 57 [-0xFF], |
| 57 [ -0x80000000 ], | 58 [-0x80000000], |
| 58 [ -0x40000000 ], | 59 [-0x40000000], |
| 59 [ -0x80000000000000000]]; | 60 [-0x80000000000000000] |
| 61 ]; |
| 60 | 62 |
| 61 final TESTS2 = [ | 63 final TESTS2 = [ |
| 62 // Test that 0xC0|1, 0x80 does not eat the next character. | 64 // Test that 0xC0|1, 0x80 does not eat the next character. |
| 63 [[ 0xC0, 0x80, 0x61 ], "Xa" ], | 65 [ |
| 64 [[ 0xC1, 0x80, 0x61 ], "Xa" ], | 66 [0xC0, 0x80, 0x61], |
| 67 "Xa" |
| 68 ], |
| 69 [ |
| 70 [0xC1, 0x80, 0x61], |
| 71 "Xa" |
| 72 ], |
| 65 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. | 73 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. |
| 66 [[ 0xF5, 0x80 ], "XX" ], | 74 [ |
| 67 [[ 0xF6, 0x80 ], "XX" ], | 75 [0xF5, 0x80], |
| 68 [[ 0xF7, 0x80 ], "XX" ], | 76 "XX" |
| 69 [[ 0xF8, 0x80 ], "XX" ], | 77 ], |
| 70 [[ 0xF9, 0x80 ], "XX" ], | 78 [ |
| 71 [[ 0xFA, 0x80 ], "XX" ], | 79 [0xF6, 0x80], |
| 72 [[ 0xFB, 0x80 ], "XX" ], | 80 "XX" |
| 73 [[ 0xFC, 0x80 ], "XX" ], | 81 ], |
| 74 [[ 0xFD, 0x80 ], "XX" ], | 82 [ |
| 75 [[ 0xFE, 0x80 ], "XX" ], | 83 [0xF7, 0x80], |
| 76 [[ 0xFF, 0x80 ], "XX" ], | 84 "XX" |
| 77 [[ 0xF5, 0x80, 0x61 ], "XXa" ], | 85 ], |
| 78 [[ 0xF6, 0x80, 0x61 ], "XXa" ], | 86 [ |
| 79 [[ 0xF7, 0x80, 0x61 ], "XXa" ], | 87 [0xF8, 0x80], |
| 80 [[ 0xF8, 0x80, 0x61 ], "XXa" ], | 88 "XX" |
| 81 [[ 0xF9, 0x80, 0x61 ], "XXa" ], | 89 ], |
| 82 [[ 0xFA, 0x80, 0x61 ], "XXa" ], | 90 [ |
| 83 [[ 0xFB, 0x80, 0x61 ], "XXa" ], | 91 [0xF9, 0x80], |
| 84 [[ 0xFC, 0x80, 0x61 ], "XXa" ], | 92 "XX" |
| 85 [[ 0xFD, 0x80, 0x61 ], "XXa" ], | 93 ], |
| 86 [[ 0xFE, 0x80, 0x61 ], "XXa" ], | 94 [ |
| 87 [[ 0xFF, 0x80, 0x61 ], "XXa" ], | 95 [0xFA, 0x80], |
| 96 "XX" |
| 97 ], |
| 98 [ |
| 99 [0xFB, 0x80], |
| 100 "XX" |
| 101 ], |
| 102 [ |
| 103 [0xFC, 0x80], |
| 104 "XX" |
| 105 ], |
| 106 [ |
| 107 [0xFD, 0x80], |
| 108 "XX" |
| 109 ], |
| 110 [ |
| 111 [0xFE, 0x80], |
| 112 "XX" |
| 113 ], |
| 114 [ |
| 115 [0xFF, 0x80], |
| 116 "XX" |
| 117 ], |
| 118 [ |
| 119 [0xF5, 0x80, 0x61], |
| 120 "XXa" |
| 121 ], |
| 122 [ |
| 123 [0xF6, 0x80, 0x61], |
| 124 "XXa" |
| 125 ], |
| 126 [ |
| 127 [0xF7, 0x80, 0x61], |
| 128 "XXa" |
| 129 ], |
| 130 [ |
| 131 [0xF8, 0x80, 0x61], |
| 132 "XXa" |
| 133 ], |
| 134 [ |
| 135 [0xF9, 0x80, 0x61], |
| 136 "XXa" |
| 137 ], |
| 138 [ |
| 139 [0xFA, 0x80, 0x61], |
| 140 "XXa" |
| 141 ], |
| 142 [ |
| 143 [0xFB, 0x80, 0x61], |
| 144 "XXa" |
| 145 ], |
| 146 [ |
| 147 [0xFC, 0x80, 0x61], |
| 148 "XXa" |
| 149 ], |
| 150 [ |
| 151 [0xFD, 0x80, 0x61], |
| 152 "XXa" |
| 153 ], |
| 154 [ |
| 155 [0xFE, 0x80, 0x61], |
| 156 "XXa" |
| 157 ], |
| 158 [ |
| 159 [0xFF, 0x80, 0x61], |
| 160 "XXa" |
| 161 ], |
| 88 // Characters outside the valid range. | 162 // Characters outside the valid range. |
| 89 [[ 0xF5, 0x80, 0x80, 0x61 ], "XXXa" ], | 163 [ |
| 90 [[ 0xF6, 0x80, 0x80, 0x61 ], "XXXa" ], | 164 [0xF5, 0x80, 0x80, 0x61], |
| 91 [[ 0xF7, 0x80, 0x80, 0x61 ], "XXXa" ], | 165 "XXXa" |
| 92 [[ 0xF8, 0x80, 0x80, 0x61 ], "XXXa" ], | 166 ], |
| 93 [[ 0xF9, 0x80, 0x80, 0x61 ], "XXXa" ], | 167 [ |
| 94 [[ 0xFA, 0x80, 0x80, 0x61 ], "XXXa" ], | 168 [0xF6, 0x80, 0x80, 0x61], |
| 95 [[ 0xFB, 0x80, 0x80, 0x61 ], "XXXa" ], | 169 "XXXa" |
| 96 [[ 0xFC, 0x80, 0x80, 0x61 ], "XXXa" ], | 170 ], |
| 97 [[ 0xFD, 0x80, 0x80, 0x61 ], "XXXa" ], | 171 [ |
| 98 [[ 0xFE, 0x80, 0x80, 0x61 ], "XXXa" ], | 172 [0xF7, 0x80, 0x80, 0x61], |
| 99 [[ 0xFF, 0x80, 0x80, 0x61 ], "XXXa" ]]; | 173 "XXXa" |
| 174 ], |
| 175 [ |
| 176 [0xF8, 0x80, 0x80, 0x61], |
| 177 "XXXa" |
| 178 ], |
| 179 [ |
| 180 [0xF9, 0x80, 0x80, 0x61], |
| 181 "XXXa" |
| 182 ], |
| 183 [ |
| 184 [0xFA, 0x80, 0x80, 0x61], |
| 185 "XXXa" |
| 186 ], |
| 187 [ |
| 188 [0xFB, 0x80, 0x80, 0x61], |
| 189 "XXXa" |
| 190 ], |
| 191 [ |
| 192 [0xFC, 0x80, 0x80, 0x61], |
| 193 "XXXa" |
| 194 ], |
| 195 [ |
| 196 [0xFD, 0x80, 0x80, 0x61], |
| 197 "XXXa" |
| 198 ], |
| 199 [ |
| 200 [0xFE, 0x80, 0x80, 0x61], |
| 201 "XXXa" |
| 202 ], |
| 203 [ |
| 204 [0xFF, 0x80, 0x80, 0x61], |
| 205 "XXXa" |
| 206 ] |
| 207 ]; |
| 100 | 208 |
| 101 main() { | 209 main() { |
| 102 var allTests = TESTS.expand((test) { | 210 var allTests = TESTS.expand((test) { |
| 103 // Pairs of test and expected string output when malformed strings are | 211 // Pairs of test and expected string output when malformed strings are |
| 104 // allowed. Replacement character: U+FFFD | 212 // allowed. Replacement character: U+FFFD |
| 105 return [[ test, "\u{FFFD}" ], | 213 return [ |
| 106 [ new List.from([0x61])..addAll(test), "a\u{FFFD}" ], | 214 [test, "\u{FFFD}"], |
| 107 [ new List.from([0x61])..addAll(test)..add(0x61), "a\u{FFFD}a" ], | 215 [ |
| 108 [ new List.from(test)..add(0x61), "\u{FFFD}a" ], | 216 new List.from([0x61])..addAll(test), |
| 109 [ new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}" ], | 217 "a\u{FFFD}" |
| 110 [ new List.from(test)..add(0x61)..addAll(test), | 218 ], |
| 111 "\u{FFFD}a\u{FFFD}" ], | 219 [ |
| 112 [ new List.from([0xc3, 0xa5])..addAll(test), "å\u{FFFD}" ], | 220 new List.from([0x61]) |
| 113 [ new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), | 221 ..addAll(test) |
| 114 "å\u{FFFD}å" ], | 222 ..add(0x61), |
| 115 [ new List.from(test)..addAll([0xc3, 0xa5]), "\u{FFFD}å" ], | 223 "a\u{FFFD}a" |
| 116 [ new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), | 224 ], |
| 117 "\u{FFFD}å\u{FFFD}" ]]; | 225 [new List.from(test)..add(0x61), "\u{FFFD}a"], |
| 226 [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], |
| 227 [ |
| 228 new List.from(test) |
| 229 ..add(0x61) |
| 230 ..addAll(test), |
| 231 "\u{FFFD}a\u{FFFD}" |
| 232 ], |
| 233 [ |
| 234 new List.from([0xc3, 0xa5])..addAll(test), |
| 235 "å\u{FFFD}" |
| 236 ], |
| 237 [ |
| 238 new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), |
| 239 "å\u{FFFD}å" |
| 240 ], |
| 241 [ |
| 242 new List.from(test)..addAll([0xc3, 0xa5]), |
| 243 "\u{FFFD}å" |
| 244 ], |
| 245 [ |
| 246 new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), |
| 247 "\u{FFFD}å\u{FFFD}" |
| 248 ] |
| 249 ]; |
| 118 }); | 250 }); |
| 119 | 251 |
| 120 var allTests2 = TESTS2.map((test) { | 252 var allTests2 = TESTS2.map((test) { |
| 121 // Pairs of test and expected string output when malformed strings are | 253 // Pairs of test and expected string output when malformed strings are |
| 122 // allowed. Replacement character: U+FFFD | 254 // allowed. Replacement character: U+FFFD |
| 123 String expected = test[1].replaceAll("X", "\u{FFFD}"); | 255 String expected = test[1].replaceAll("X", "\u{FFFD}"); |
| 124 return [test[0], expected]; | 256 return [test[0], expected]; |
| 125 }); | 257 }); |
| 126 | 258 |
| 127 for (var test in []..addAll(allTests)..addAll(allTests2)) { | 259 for (var test in []..addAll(allTests)..addAll(allTests2)) { |
| 128 List<int> bytes = test[0]; | 260 List<int> bytes = test[0]; |
| 129 Expect.throws(() => decode(bytes), (e) => e is FormatException); | 261 Expect.throws(() => decode(bytes), (e) => e is FormatException); |
| 130 Expect.throws(() => decode2(bytes), (e) => e is FormatException); | 262 Expect.throws(() => decode2(bytes), (e) => e is FormatException); |
| 131 Expect.throws(() => decode3(bytes), (e) => e is FormatException); | 263 Expect.throws(() => decode3(bytes), (e) => e is FormatException); |
| 132 Expect.throws(() => decode4(bytes), (e) => e is FormatException); | 264 Expect.throws(() => decode4(bytes), (e) => e is FormatException); |
| 133 | 265 |
| 134 String expected = test[1]; | 266 String expected = test[1]; |
| 135 Expect.equals(expected, decodeAllowMalformed(bytes)); | 267 Expect.equals(expected, decodeAllowMalformed(bytes)); |
| 136 Expect.equals(expected, decodeAllowMalformed2(bytes)); | 268 Expect.equals(expected, decodeAllowMalformed2(bytes)); |
| 137 Expect.equals(expected, decodeAllowMalformed3(bytes)); | 269 Expect.equals(expected, decodeAllowMalformed3(bytes)); |
| 138 Expect.equals(expected, decodeAllowMalformed4(bytes)); | 270 Expect.equals(expected, decodeAllowMalformed4(bytes)); |
| 139 } | 271 } |
| 140 } | 272 } |
| OLD | NEW |