OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library utf8_test; | 5 library utf8_test; |
| 6 |
6 import "package:expect/expect.dart"; | 7 import "package:expect/expect.dart"; |
7 import 'dart:convert'; | 8 import 'dart:convert'; |
8 | 9 |
9 String decode(List<int> bytes) => new Utf8Decoder().convert(bytes); | 10 String decode(List<int> bytes) => new Utf8Decoder().convert(bytes); |
10 String decodeAllowMalformed(List<int> bytes) { | 11 String decodeAllowMalformed(List<int> bytes) { |
11 return new Utf8Decoder(allowMalformed: true).convert(bytes); | 12 return new Utf8Decoder(allowMalformed: true).convert(bytes); |
12 } | 13 } |
13 | 14 |
14 String decode2(List<int> bytes) => UTF8.decode(bytes); | 15 String decode2(List<int> bytes) => UTF8.decode(bytes); |
15 String decodeAllowMalformed2(List<int> bytes) { | 16 String decodeAllowMalformed2(List<int> bytes) { |
16 return UTF8.decode(bytes, allowMalformed: true); | 17 return UTF8.decode(bytes, allowMalformed: true); |
17 } | 18 } |
18 | 19 |
19 String decode3(List<int> bytes) => new Utf8Codec().decode(bytes); | 20 String decode3(List<int> bytes) => new Utf8Codec().decode(bytes); |
20 String decodeAllowMalformed3(List<int> bytes) { | 21 String decodeAllowMalformed3(List<int> bytes) { |
21 return new Utf8Codec(allowMalformed: true).decode(bytes); | 22 return new Utf8Codec(allowMalformed: true).decode(bytes); |
22 } | 23 } |
23 | 24 |
24 String decode4(List<int> bytes) => new Utf8Codec().decoder.convert(bytes); | 25 String decode4(List<int> bytes) => new Utf8Codec().decoder.convert(bytes); |
25 String decodeAllowMalformed4(List<int> bytes) { | 26 String decodeAllowMalformed4(List<int> bytes) { |
26 return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); | 27 return new Utf8Codec(allowMalformed: true).decoder.convert(bytes); |
27 } | 28 } |
28 | 29 |
29 final TESTS = [ | 30 final TESTS = [ |
30 // Unfinished UTF-8 sequences. | 31 // Unfinished UTF-8 sequences. |
31 [ 0xc3 ], | 32 [0xc3], |
32 [ 0xE2, 0x82 ], | 33 [0xE2, 0x82], |
33 [ 0xF0, 0xA4, 0xAD ], | 34 [0xF0, 0xA4, 0xAD], |
34 // Overlong encoding of euro-sign. | 35 // Overlong encoding of euro-sign. |
35 [ 0xF0, 0x82, 0x82, 0xAC ], | 36 [0xF0, 0x82, 0x82, 0xAC], |
36 // Other overlong/unfinished sequences. | 37 // Other overlong/unfinished sequences. |
37 [ 0xC0 ], | 38 [0xC0], |
38 [ 0xC1 ], | 39 [0xC1], |
39 [ 0xF5 ], | 40 [0xF5], |
40 [ 0xF6 ], | 41 [0xF6], |
41 [ 0xF7 ], | 42 [0xF7], |
42 [ 0xF8 ], | 43 [0xF8], |
43 [ 0xF9 ], | 44 [0xF9], |
44 [ 0xFA ], | 45 [0xFA], |
45 [ 0xFB ], | 46 [0xFB], |
46 [ 0xFC ], | 47 [0xFC], |
47 [ 0xFD ], | 48 [0xFD], |
48 [ 0xFE ], | 49 [0xFE], |
49 [ 0xFF ], | 50 [0xFF], |
50 [ 0xC0, 0x80 ], | 51 [0xC0, 0x80], |
51 [ 0xC1, 0x80 ], | 52 [0xC1, 0x80], |
52 // Outside valid range. | 53 // Outside valid range. |
53 [ 0xF4, 0xBF, 0xBF, 0xBF ], | 54 [0xF4, 0xBF, 0xBF, 0xBF], |
54 // Negative | 55 // Negative |
55 [ -0x01 ], | 56 [-0x01], |
56 [ -0xFF ], | 57 [-0xFF], |
57 [ -0x80000000 ], | 58 [-0x80000000], |
58 [ -0x40000000 ], | 59 [-0x40000000], |
59 [ -0x80000000000000000]]; | 60 [-0x80000000000000000] |
| 61 ]; |
60 | 62 |
61 final TESTS2 = [ | 63 final TESTS2 = [ |
62 // Test that 0xC0|1, 0x80 does not eat the next character. | 64 // Test that 0xC0|1, 0x80 does not eat the next character. |
63 [[ 0xC0, 0x80, 0x61 ], "Xa" ], | 65 [ |
64 [[ 0xC1, 0x80, 0x61 ], "Xa" ], | 66 [0xC0, 0x80, 0x61], |
| 67 "Xa" |
| 68 ], |
| 69 [ |
| 70 [0xC1, 0x80, 0x61], |
| 71 "Xa" |
| 72 ], |
65 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. | 73 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. |
66 [[ 0xF5, 0x80 ], "XX" ], | 74 [ |
67 [[ 0xF6, 0x80 ], "XX" ], | 75 [0xF5, 0x80], |
68 [[ 0xF7, 0x80 ], "XX" ], | 76 "XX" |
69 [[ 0xF8, 0x80 ], "XX" ], | 77 ], |
70 [[ 0xF9, 0x80 ], "XX" ], | 78 [ |
71 [[ 0xFA, 0x80 ], "XX" ], | 79 [0xF6, 0x80], |
72 [[ 0xFB, 0x80 ], "XX" ], | 80 "XX" |
73 [[ 0xFC, 0x80 ], "XX" ], | 81 ], |
74 [[ 0xFD, 0x80 ], "XX" ], | 82 [ |
75 [[ 0xFE, 0x80 ], "XX" ], | 83 [0xF7, 0x80], |
76 [[ 0xFF, 0x80 ], "XX" ], | 84 "XX" |
77 [[ 0xF5, 0x80, 0x61 ], "XXa" ], | 85 ], |
78 [[ 0xF6, 0x80, 0x61 ], "XXa" ], | 86 [ |
79 [[ 0xF7, 0x80, 0x61 ], "XXa" ], | 87 [0xF8, 0x80], |
80 [[ 0xF8, 0x80, 0x61 ], "XXa" ], | 88 "XX" |
81 [[ 0xF9, 0x80, 0x61 ], "XXa" ], | 89 ], |
82 [[ 0xFA, 0x80, 0x61 ], "XXa" ], | 90 [ |
83 [[ 0xFB, 0x80, 0x61 ], "XXa" ], | 91 [0xF9, 0x80], |
84 [[ 0xFC, 0x80, 0x61 ], "XXa" ], | 92 "XX" |
85 [[ 0xFD, 0x80, 0x61 ], "XXa" ], | 93 ], |
86 [[ 0xFE, 0x80, 0x61 ], "XXa" ], | 94 [ |
87 [[ 0xFF, 0x80, 0x61 ], "XXa" ], | 95 [0xFA, 0x80], |
| 96 "XX" |
| 97 ], |
| 98 [ |
| 99 [0xFB, 0x80], |
| 100 "XX" |
| 101 ], |
| 102 [ |
| 103 [0xFC, 0x80], |
| 104 "XX" |
| 105 ], |
| 106 [ |
| 107 [0xFD, 0x80], |
| 108 "XX" |
| 109 ], |
| 110 [ |
| 111 [0xFE, 0x80], |
| 112 "XX" |
| 113 ], |
| 114 [ |
| 115 [0xFF, 0x80], |
| 116 "XX" |
| 117 ], |
| 118 [ |
| 119 [0xF5, 0x80, 0x61], |
| 120 "XXa" |
| 121 ], |
| 122 [ |
| 123 [0xF6, 0x80, 0x61], |
| 124 "XXa" |
| 125 ], |
| 126 [ |
| 127 [0xF7, 0x80, 0x61], |
| 128 "XXa" |
| 129 ], |
| 130 [ |
| 131 [0xF8, 0x80, 0x61], |
| 132 "XXa" |
| 133 ], |
| 134 [ |
| 135 [0xF9, 0x80, 0x61], |
| 136 "XXa" |
| 137 ], |
| 138 [ |
| 139 [0xFA, 0x80, 0x61], |
| 140 "XXa" |
| 141 ], |
| 142 [ |
| 143 [0xFB, 0x80, 0x61], |
| 144 "XXa" |
| 145 ], |
| 146 [ |
| 147 [0xFC, 0x80, 0x61], |
| 148 "XXa" |
| 149 ], |
| 150 [ |
| 151 [0xFD, 0x80, 0x61], |
| 152 "XXa" |
| 153 ], |
| 154 [ |
| 155 [0xFE, 0x80, 0x61], |
| 156 "XXa" |
| 157 ], |
| 158 [ |
| 159 [0xFF, 0x80, 0x61], |
| 160 "XXa" |
| 161 ], |
88 // Characters outside the valid range. | 162 // Characters outside the valid range. |
89 [[ 0xF5, 0x80, 0x80, 0x61 ], "XXXa" ], | 163 [ |
90 [[ 0xF6, 0x80, 0x80, 0x61 ], "XXXa" ], | 164 [0xF5, 0x80, 0x80, 0x61], |
91 [[ 0xF7, 0x80, 0x80, 0x61 ], "XXXa" ], | 165 "XXXa" |
92 [[ 0xF8, 0x80, 0x80, 0x61 ], "XXXa" ], | 166 ], |
93 [[ 0xF9, 0x80, 0x80, 0x61 ], "XXXa" ], | 167 [ |
94 [[ 0xFA, 0x80, 0x80, 0x61 ], "XXXa" ], | 168 [0xF6, 0x80, 0x80, 0x61], |
95 [[ 0xFB, 0x80, 0x80, 0x61 ], "XXXa" ], | 169 "XXXa" |
96 [[ 0xFC, 0x80, 0x80, 0x61 ], "XXXa" ], | 170 ], |
97 [[ 0xFD, 0x80, 0x80, 0x61 ], "XXXa" ], | 171 [ |
98 [[ 0xFE, 0x80, 0x80, 0x61 ], "XXXa" ], | 172 [0xF7, 0x80, 0x80, 0x61], |
99 [[ 0xFF, 0x80, 0x80, 0x61 ], "XXXa" ]]; | 173 "XXXa" |
| 174 ], |
| 175 [ |
| 176 [0xF8, 0x80, 0x80, 0x61], |
| 177 "XXXa" |
| 178 ], |
| 179 [ |
| 180 [0xF9, 0x80, 0x80, 0x61], |
| 181 "XXXa" |
| 182 ], |
| 183 [ |
| 184 [0xFA, 0x80, 0x80, 0x61], |
| 185 "XXXa" |
| 186 ], |
| 187 [ |
| 188 [0xFB, 0x80, 0x80, 0x61], |
| 189 "XXXa" |
| 190 ], |
| 191 [ |
| 192 [0xFC, 0x80, 0x80, 0x61], |
| 193 "XXXa" |
| 194 ], |
| 195 [ |
| 196 [0xFD, 0x80, 0x80, 0x61], |
| 197 "XXXa" |
| 198 ], |
| 199 [ |
| 200 [0xFE, 0x80, 0x80, 0x61], |
| 201 "XXXa" |
| 202 ], |
| 203 [ |
| 204 [0xFF, 0x80, 0x80, 0x61], |
| 205 "XXXa" |
| 206 ] |
| 207 ]; |
100 | 208 |
101 main() { | 209 main() { |
102 var allTests = TESTS.expand((test) { | 210 var allTests = TESTS.expand((test) { |
103 // Pairs of test and expected string output when malformed strings are | 211 // Pairs of test and expected string output when malformed strings are |
104 // allowed. Replacement character: U+FFFD | 212 // allowed. Replacement character: U+FFFD |
105 return [[ test, "\u{FFFD}" ], | 213 return [ |
106 [ new List.from([0x61])..addAll(test), "a\u{FFFD}" ], | 214 [test, "\u{FFFD}"], |
107 [ new List.from([0x61])..addAll(test)..add(0x61), "a\u{FFFD}a" ], | 215 [ |
108 [ new List.from(test)..add(0x61), "\u{FFFD}a" ], | 216 new List.from([0x61])..addAll(test), |
109 [ new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}" ], | 217 "a\u{FFFD}" |
110 [ new List.from(test)..add(0x61)..addAll(test), | 218 ], |
111 "\u{FFFD}a\u{FFFD}" ], | 219 [ |
112 [ new List.from([0xc3, 0xa5])..addAll(test), "å\u{FFFD}" ], | 220 new List.from([0x61]) |
113 [ new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), | 221 ..addAll(test) |
114 "å\u{FFFD}å" ], | 222 ..add(0x61), |
115 [ new List.from(test)..addAll([0xc3, 0xa5]), "\u{FFFD}å" ], | 223 "a\u{FFFD}a" |
116 [ new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), | 224 ], |
117 "\u{FFFD}å\u{FFFD}" ]]; | 225 [new List.from(test)..add(0x61), "\u{FFFD}a"], |
| 226 [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], |
| 227 [ |
| 228 new List.from(test) |
| 229 ..add(0x61) |
| 230 ..addAll(test), |
| 231 "\u{FFFD}a\u{FFFD}" |
| 232 ], |
| 233 [ |
| 234 new List.from([0xc3, 0xa5])..addAll(test), |
| 235 "å\u{FFFD}" |
| 236 ], |
| 237 [ |
| 238 new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), |
| 239 "å\u{FFFD}å" |
| 240 ], |
| 241 [ |
| 242 new List.from(test)..addAll([0xc3, 0xa5]), |
| 243 "\u{FFFD}å" |
| 244 ], |
| 245 [ |
| 246 new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), |
| 247 "\u{FFFD}å\u{FFFD}" |
| 248 ] |
| 249 ]; |
118 }); | 250 }); |
119 | 251 |
120 var allTests2 = TESTS2.map((test) { | 252 var allTests2 = TESTS2.map((test) { |
121 // Pairs of test and expected string output when malformed strings are | 253 // Pairs of test and expected string output when malformed strings are |
122 // allowed. Replacement character: U+FFFD | 254 // allowed. Replacement character: U+FFFD |
123 String expected = test[1].replaceAll("X", "\u{FFFD}"); | 255 String expected = test[1].replaceAll("X", "\u{FFFD}"); |
124 return [test[0], expected]; | 256 return [test[0], expected]; |
125 }); | 257 }); |
126 | 258 |
127 for (var test in []..addAll(allTests)..addAll(allTests2)) { | 259 for (var test in []..addAll(allTests)..addAll(allTests2)) { |
128 List<int> bytes = test[0]; | 260 List<int> bytes = test[0]; |
129 Expect.throws(() => decode(bytes), (e) => e is FormatException); | 261 Expect.throws(() => decode(bytes), (e) => e is FormatException); |
130 Expect.throws(() => decode2(bytes), (e) => e is FormatException); | 262 Expect.throws(() => decode2(bytes), (e) => e is FormatException); |
131 Expect.throws(() => decode3(bytes), (e) => e is FormatException); | 263 Expect.throws(() => decode3(bytes), (e) => e is FormatException); |
132 Expect.throws(() => decode4(bytes), (e) => e is FormatException); | 264 Expect.throws(() => decode4(bytes), (e) => e is FormatException); |
133 | 265 |
134 String expected = test[1]; | 266 String expected = test[1]; |
135 Expect.equals(expected, decodeAllowMalformed(bytes)); | 267 Expect.equals(expected, decodeAllowMalformed(bytes)); |
136 Expect.equals(expected, decodeAllowMalformed2(bytes)); | 268 Expect.equals(expected, decodeAllowMalformed2(bytes)); |
137 Expect.equals(expected, decodeAllowMalformed3(bytes)); | 269 Expect.equals(expected, decodeAllowMalformed3(bytes)); |
138 Expect.equals(expected, decodeAllowMalformed4(bytes)); | 270 Expect.equals(expected, decodeAllowMalformed4(bytes)); |
139 } | 271 } |
140 } | 272 } |
OLD | NEW |