OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library utf8_test; | 5 library utf8_test; |
| 6 |
6 import "package:expect/expect.dart"; | 7 import "package:expect/expect.dart"; |
7 import 'dart:convert'; | 8 import 'dart:convert'; |
8 | 9 |
9 String decode(List<int> bytes, int chunkSize) { | 10 String decode(List<int> bytes, int chunkSize) { |
10 StringBuffer buffer = new StringBuffer(); | 11 StringBuffer buffer = new StringBuffer(); |
11 var stringSink = new StringConversionSink.fromStringSink(buffer); | 12 var stringSink = new StringConversionSink.fromStringSink(buffer); |
12 var byteSink = new Utf8Decoder().startChunkedConversion(stringSink); | 13 var byteSink = new Utf8Decoder().startChunkedConversion(stringSink); |
13 int i = 0; | 14 int i = 0; |
14 while (i < bytes.length) { | 15 while (i < bytes.length) { |
15 List nextChunk = []; | 16 List nextChunk = []; |
(...skipping 24 matching lines...) Expand all Loading... |
40 } | 41 } |
41 } | 42 } |
42 byteSink.add(nextChunk); | 43 byteSink.add(nextChunk); |
43 } | 44 } |
44 byteSink.close(); | 45 byteSink.close(); |
45 return buffer.toString(); | 46 return buffer.toString(); |
46 } | 47 } |
47 | 48 |
48 final TESTS = [ | 49 final TESTS = [ |
49 // Unfinished UTF-8 sequences. | 50 // Unfinished UTF-8 sequences. |
50 [ 0xc3 ], | 51 [0xc3], |
51 [ 0xE2, 0x82 ], | 52 [0xE2, 0x82], |
52 [ 0xF0, 0xA4, 0xAD ], | 53 [0xF0, 0xA4, 0xAD], |
53 // Overlong encoding of euro-sign. | 54 // Overlong encoding of euro-sign. |
54 [ 0xF0, 0x82, 0x82, 0xAC ], | 55 [0xF0, 0x82, 0x82, 0xAC], |
55 // Other overlong/unfinished sequences. | 56 // Other overlong/unfinished sequences. |
56 [ 0xC0 ], | 57 [0xC0], |
57 [ 0xC1 ], | 58 [0xC1], |
58 [ 0xF5 ], | 59 [0xF5], |
59 [ 0xF6 ], | 60 [0xF6], |
60 [ 0xF7 ], | 61 [0xF7], |
61 [ 0xF8 ], | 62 [0xF8], |
62 [ 0xF9 ], | 63 [0xF9], |
63 [ 0xFA ], | 64 [0xFA], |
64 [ 0xFB ], | 65 [0xFB], |
65 [ 0xFC ], | 66 [0xFC], |
66 [ 0xFD ], | 67 [0xFD], |
67 [ 0xFE ], | 68 [0xFE], |
68 [ 0xFF ], | 69 [0xFF], |
69 [ 0xC0, 0x80 ], | 70 [0xC0, 0x80], |
70 [ 0xC1, 0x80 ], | 71 [0xC1, 0x80], |
71 // Outside valid range. | 72 // Outside valid range. |
72 [ 0xF4, 0xBF, 0xBF, 0xBF ]]; | 73 [0xF4, 0xBF, 0xBF, 0xBF] |
| 74 ]; |
73 | 75 |
74 final TESTS2 = [ | 76 final TESTS2 = [ |
75 // Test that 0xC0|1, 0x80 does not eat the next character. | 77 // Test that 0xC0|1, 0x80 does not eat the next character. |
76 [[ 0xC0, 0x80, 0x61 ], "Xa" ], | 78 [ |
77 [[ 0xC1, 0x80, 0x61 ], "Xa" ], | 79 [0xC0, 0x80, 0x61], |
| 80 "Xa" |
| 81 ], |
| 82 [ |
| 83 [0xC1, 0x80, 0x61], |
| 84 "Xa" |
| 85 ], |
78 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. | 86 // 0xF5 .. 0xFF never appear in valid UTF-8 sequences. |
79 [[ 0xF5, 0x80 ], "XX" ], | 87 [ |
80 [[ 0xF6, 0x80 ], "XX" ], | 88 [0xF5, 0x80], |
81 [[ 0xF7, 0x80 ], "XX" ], | 89 "XX" |
82 [[ 0xF8, 0x80 ], "XX" ], | 90 ], |
83 [[ 0xF9, 0x80 ], "XX" ], | 91 [ |
84 [[ 0xFA, 0x80 ], "XX" ], | 92 [0xF6, 0x80], |
85 [[ 0xFB, 0x80 ], "XX" ], | 93 "XX" |
86 [[ 0xFC, 0x80 ], "XX" ], | 94 ], |
87 [[ 0xFD, 0x80 ], "XX" ], | 95 [ |
88 [[ 0xFE, 0x80 ], "XX" ], | 96 [0xF7, 0x80], |
89 [[ 0xFF, 0x80 ], "XX" ], | 97 "XX" |
90 [[ 0xF5, 0x80, 0x61 ], "XXa" ], | 98 ], |
91 [[ 0xF6, 0x80, 0x61 ], "XXa" ], | 99 [ |
92 [[ 0xF7, 0x80, 0x61 ], "XXa" ], | 100 [0xF8, 0x80], |
93 [[ 0xF8, 0x80, 0x61 ], "XXa" ], | 101 "XX" |
94 [[ 0xF9, 0x80, 0x61 ], "XXa" ], | 102 ], |
95 [[ 0xFA, 0x80, 0x61 ], "XXa" ], | 103 [ |
96 [[ 0xFB, 0x80, 0x61 ], "XXa" ], | 104 [0xF9, 0x80], |
97 [[ 0xFC, 0x80, 0x61 ], "XXa" ], | 105 "XX" |
98 [[ 0xFD, 0x80, 0x61 ], "XXa" ], | 106 ], |
99 [[ 0xFE, 0x80, 0x61 ], "XXa" ], | 107 [ |
100 [[ 0xFF, 0x80, 0x61 ], "XXa" ], | 108 [0xFA, 0x80], |
| 109 "XX" |
| 110 ], |
| 111 [ |
| 112 [0xFB, 0x80], |
| 113 "XX" |
| 114 ], |
| 115 [ |
| 116 [0xFC, 0x80], |
| 117 "XX" |
| 118 ], |
| 119 [ |
| 120 [0xFD, 0x80], |
| 121 "XX" |
| 122 ], |
| 123 [ |
| 124 [0xFE, 0x80], |
| 125 "XX" |
| 126 ], |
| 127 [ |
| 128 [0xFF, 0x80], |
| 129 "XX" |
| 130 ], |
| 131 [ |
| 132 [0xF5, 0x80, 0x61], |
| 133 "XXa" |
| 134 ], |
| 135 [ |
| 136 [0xF6, 0x80, 0x61], |
| 137 "XXa" |
| 138 ], |
| 139 [ |
| 140 [0xF7, 0x80, 0x61], |
| 141 "XXa" |
| 142 ], |
| 143 [ |
| 144 [0xF8, 0x80, 0x61], |
| 145 "XXa" |
| 146 ], |
| 147 [ |
| 148 [0xF9, 0x80, 0x61], |
| 149 "XXa" |
| 150 ], |
| 151 [ |
| 152 [0xFA, 0x80, 0x61], |
| 153 "XXa" |
| 154 ], |
| 155 [ |
| 156 [0xFB, 0x80, 0x61], |
| 157 "XXa" |
| 158 ], |
| 159 [ |
| 160 [0xFC, 0x80, 0x61], |
| 161 "XXa" |
| 162 ], |
| 163 [ |
| 164 [0xFD, 0x80, 0x61], |
| 165 "XXa" |
| 166 ], |
| 167 [ |
| 168 [0xFE, 0x80, 0x61], |
| 169 "XXa" |
| 170 ], |
| 171 [ |
| 172 [0xFF, 0x80, 0x61], |
| 173 "XXa" |
| 174 ], |
101 // Characters outside the valid range. | 175 // Characters outside the valid range. |
102 [[ 0xF5, 0x80, 0x80, 0x61 ], "XXXa" ], | 176 [ |
103 [[ 0xF6, 0x80, 0x80, 0x61 ], "XXXa" ], | 177 [0xF5, 0x80, 0x80, 0x61], |
104 [[ 0xF7, 0x80, 0x80, 0x61 ], "XXXa" ], | 178 "XXXa" |
105 [[ 0xF8, 0x80, 0x80, 0x61 ], "XXXa" ], | 179 ], |
106 [[ 0xF9, 0x80, 0x80, 0x61 ], "XXXa" ], | 180 [ |
107 [[ 0xFA, 0x80, 0x80, 0x61 ], "XXXa" ], | 181 [0xF6, 0x80, 0x80, 0x61], |
108 [[ 0xFB, 0x80, 0x80, 0x61 ], "XXXa" ], | 182 "XXXa" |
109 [[ 0xFC, 0x80, 0x80, 0x61 ], "XXXa" ], | 183 ], |
110 [[ 0xFD, 0x80, 0x80, 0x61 ], "XXXa" ], | 184 [ |
111 [[ 0xFE, 0x80, 0x80, 0x61 ], "XXXa" ], | 185 [0xF7, 0x80, 0x80, 0x61], |
112 [[ 0xFF, 0x80, 0x80, 0x61 ], "XXXa" ]]; | 186 "XXXa" |
| 187 ], |
| 188 [ |
| 189 [0xF8, 0x80, 0x80, 0x61], |
| 190 "XXXa" |
| 191 ], |
| 192 [ |
| 193 [0xF9, 0x80, 0x80, 0x61], |
| 194 "XXXa" |
| 195 ], |
| 196 [ |
| 197 [0xFA, 0x80, 0x80, 0x61], |
| 198 "XXXa" |
| 199 ], |
| 200 [ |
| 201 [0xFB, 0x80, 0x80, 0x61], |
| 202 "XXXa" |
| 203 ], |
| 204 [ |
| 205 [0xFC, 0x80, 0x80, 0x61], |
| 206 "XXXa" |
| 207 ], |
| 208 [ |
| 209 [0xFD, 0x80, 0x80, 0x61], |
| 210 "XXXa" |
| 211 ], |
| 212 [ |
| 213 [0xFE, 0x80, 0x80, 0x61], |
| 214 "XXXa" |
| 215 ], |
| 216 [ |
| 217 [0xFF, 0x80, 0x80, 0x61], |
| 218 "XXXa" |
| 219 ] |
| 220 ]; |
113 | 221 |
114 main() { | 222 main() { |
115 var allTests = TESTS.expand((test) { | 223 var allTests = TESTS.expand((test) { |
116 // Pairs of test and expected string output when malformed strings are | 224 // Pairs of test and expected string output when malformed strings are |
117 // allowed. Replacement character: U+FFFD | 225 // allowed. Replacement character: U+FFFD |
118 return [[ test, "\u{FFFD}" ], | 226 return [ |
119 [ new List.from([0x61])..addAll(test), "a\u{FFFD}" ], | 227 [test, "\u{FFFD}"], |
120 [ new List.from([0x61])..addAll(test)..add(0x61), "a\u{FFFD}a" ], | 228 [ |
121 [ new List.from(test)..add(0x61), "\u{FFFD}a" ], | 229 new List.from([0x61])..addAll(test), |
122 [ new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}" ], | 230 "a\u{FFFD}" |
123 [ new List.from(test)..add(0x61)..addAll(test), | 231 ], |
124 "\u{FFFD}a\u{FFFD}" ], | 232 [ |
125 [ new List.from([0xc3, 0xa5])..addAll(test), "å\u{FFFD}" ], | 233 new List.from([0x61]) |
126 [ new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), | 234 ..addAll(test) |
127 "å\u{FFFD}å" ], | 235 ..add(0x61), |
128 [ new List.from(test)..addAll([0xc3, 0xa5]), "\u{FFFD}å" ], | 236 "a\u{FFFD}a" |
129 [ new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), | 237 ], |
130 "\u{FFFD}å\u{FFFD}" ]]; | 238 [new List.from(test)..add(0x61), "\u{FFFD}a"], |
| 239 [new List.from(test)..addAll(test), "\u{FFFD}\u{FFFD}"], |
| 240 [ |
| 241 new List.from(test) |
| 242 ..add(0x61) |
| 243 ..addAll(test), |
| 244 "\u{FFFD}a\u{FFFD}" |
| 245 ], |
| 246 [ |
| 247 new List.from([0xc3, 0xa5])..addAll(test), |
| 248 "å\u{FFFD}" |
| 249 ], |
| 250 [ |
| 251 new List.from([0xc3, 0xa5])..addAll(test)..addAll([0xc3, 0xa5]), |
| 252 "å\u{FFFD}å" |
| 253 ], |
| 254 [ |
| 255 new List.from(test)..addAll([0xc3, 0xa5]), |
| 256 "\u{FFFD}å" |
| 257 ], |
| 258 [ |
| 259 new List.from(test)..addAll([0xc3, 0xa5])..addAll(test), |
| 260 "\u{FFFD}å\u{FFFD}" |
| 261 ] |
| 262 ]; |
131 }); | 263 }); |
132 | 264 |
133 var allTests2 = TESTS2.map((test) { | 265 var allTests2 = TESTS2.map((test) { |
134 // Pairs of test and expected string output when malformed strings are | 266 // Pairs of test and expected string output when malformed strings are |
135 // allowed. Replacement character: U+FFFD | 267 // allowed. Replacement character: U+FFFD |
136 String expected = (test[1] as String).replaceAll("X", "\u{FFFD}"); | 268 String expected = (test[1] as String).replaceAll("X", "\u{FFFD}"); |
137 return [test[0], expected]; | 269 return [test[0], expected]; |
138 }); | 270 }); |
139 | 271 |
140 for (var test in []..addAll(allTests)..addAll(allTests2)) { | 272 for (var test in []..addAll(allTests)..addAll(allTests2)) { |
141 List<int> bytes = test[0]; | 273 List<int> bytes = test[0]; |
142 Expect.throws(() => decode(bytes, 1), (e) => e is FormatException); | 274 Expect.throws(() => decode(bytes, 1), (e) => e is FormatException); |
143 Expect.throws(() => decode(bytes, 2), (e) => e is FormatException); | 275 Expect.throws(() => decode(bytes, 2), (e) => e is FormatException); |
144 Expect.throws(() => decode(bytes, 3), (e) => e is FormatException); | 276 Expect.throws(() => decode(bytes, 3), (e) => e is FormatException); |
145 Expect.throws(() => decode(bytes, 4), (e) => e is FormatException); | 277 Expect.throws(() => decode(bytes, 4), (e) => e is FormatException); |
146 | 278 |
147 String expected = test[1]; | 279 String expected = test[1]; |
148 Expect.equals(expected, decodeAllowMalformed(bytes, 1)); | 280 Expect.equals(expected, decodeAllowMalformed(bytes, 1)); |
149 Expect.equals(expected, decodeAllowMalformed(bytes, 2)); | 281 Expect.equals(expected, decodeAllowMalformed(bytes, 2)); |
150 Expect.equals(expected, decodeAllowMalformed(bytes, 3)); | 282 Expect.equals(expected, decodeAllowMalformed(bytes, 3)); |
151 Expect.equals(expected, decodeAllowMalformed(bytes, 4)); | 283 Expect.equals(expected, decodeAllowMalformed(bytes, 4)); |
152 } | 284 } |
153 } | 285 } |
OLD | NEW |