| OLD | NEW |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library utf8_test; | 5 library utf8_test; |
| 6 import "package:expect/expect.dart"; | 6 import "package:expect/expect.dart"; |
| 7 import 'dart:convert'; | 7 import 'dart:convert'; |
| 8 | 8 |
| 9 List<int> encode(String str) { | 9 List<int> encode(String str) { |
| 10 List<int> bytes; | 10 List<int> bytes; |
| (...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 89 v--; | 89 v--; |
| 90 v |= v >> 1; | 90 v |= v >> 1; |
| 91 v |= v >> 2; | 91 v |= v >> 2; |
| 92 v |= v >> 4; | 92 v |= v >> 4; |
| 93 v |= v >> 8; | 93 v |= v >> 8; |
| 94 v |= v >> 16; | 94 v |= v >> 16; |
| 95 v++; | 95 v++; |
| 96 return v; | 96 return v; |
| 97 } | 97 } |
| 98 | 98 |
| 99 runTest(test) { |
| 100 List<int> bytes = test[0]; |
| 101 String string = test[1]; |
| 102 Expect.listEquals(bytes, encode(string)); |
| 103 Expect.listEquals(bytes, encode2(string)); |
| 104 Expect.listEquals(bytes, encode3(string)); |
| 105 Expect.listEquals(bytes, encode4(string)); |
| 106 Expect.listEquals(bytes, encode5(string)); |
| 107 Expect.listEquals(bytes, encode6(string)); |
| 108 Expect.listEquals(bytes, encode7(string)); |
| 109 } |
| 110 |
| 99 main() { | 111 main() { |
| 100 const LEADING_SURROGATE = 0xd801; | 112 const LEADING_SURROGATE = 0xd801; |
| 101 const TRAILING_SURROGATE = 0xdc12; | 113 const TRAILING_SURROGATE = 0xdc12; |
| 102 const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; | 114 const UTF8_ENCODING = const [0xf0, 0x90, 0x90, 0x92]; |
| 103 const UTF8_LEADING = const [0xed, 0xa0, 0x81]; | 115 const UTF8_LEADING = const [0xed, 0xa0, 0x81]; |
| 104 const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; | 116 const UTF8_TRAILING = const [0xed, 0xb0, 0x92]; |
| 105 const CHAR_A = 0x61; | 117 const CHAR_A = 0x61; |
| 106 | 118 |
| 107 // Test surrogates at all kinds of locations. | 119 // Test surrogates at all kinds of locations. |
| 108 var tests = []; | 120 var tests = []; |
| 109 List codeUnits = <int>[]; | 121 List codeUnits = <int>[]; |
| 110 for (int i = 0; i < 2049; i++) { | 122 for (int i = 0; i < 2049; i++) { |
| 111 // Invariant: codeUnits[0..i - 1] is filled with CHAR_A (character 'a'). | 123 // Invariant: codeUnits[0..i - 1] is filled with CHAR_A (character 'a'). |
| 112 codeUnits.length = i + 1; | 124 codeUnits.length = i + 1; |
| 113 codeUnits[i] = CHAR_A; | 125 codeUnits[i] = CHAR_A; |
| 114 | 126 |
| 115 // Only test for problem zones, close to powers of two. | 127 // Only test for problem zones, close to powers of two. |
| 116 if (i > 20 && _nextPowerOf2(i - 15) - i > 30) continue; | 128 if (i > 20 && _nextPowerOf2(i - 2) - i > 10) continue; |
| 117 | 129 |
| 118 codeUnits[i] = LEADING_SURROGATE; | 130 codeUnits[i] = LEADING_SURROGATE; |
| 119 var str = new String.fromCharCodes(codeUnits); | 131 var str = new String.fromCharCodes(codeUnits); |
| 120 var bytes = new List.filled(i + 3, CHAR_A); | 132 var bytes = new List.filled(i + 3, CHAR_A); |
| 121 bytes[i] = UTF8_LEADING[0]; | 133 bytes[i] = UTF8_LEADING[0]; |
| 122 bytes[i + 1] = UTF8_LEADING[1]; | 134 bytes[i + 1] = UTF8_LEADING[1]; |
| 123 bytes[i + 2] = UTF8_LEADING[2]; | 135 bytes[i + 2] = UTF8_LEADING[2]; |
| 124 tests.add([bytes, str]); | 136 runTest([bytes, str]); |
| 125 | 137 |
| 126 codeUnits[i] = TRAILING_SURROGATE; | 138 codeUnits[i] = TRAILING_SURROGATE; |
| 127 str = new String.fromCharCodes(codeUnits); | 139 str = new String.fromCharCodes(codeUnits); |
| 128 bytes = new List.filled(i + 3, CHAR_A); | 140 bytes = new List.filled(i + 3, CHAR_A); |
| 129 bytes[i] = UTF8_TRAILING[0]; | 141 bytes[i] = UTF8_TRAILING[0]; |
| 130 bytes[i + 1] = UTF8_TRAILING[1]; | 142 bytes[i + 1] = UTF8_TRAILING[1]; |
| 131 bytes[i + 2] = UTF8_TRAILING[2]; | 143 bytes[i + 2] = UTF8_TRAILING[2]; |
| 132 tests.add([bytes, str]); | 144 runTest([bytes, str]); |
| 133 | 145 |
| 134 codeUnits.length = i + 2; | 146 codeUnits.length = i + 2; |
| 135 codeUnits[i] = LEADING_SURROGATE; | 147 codeUnits[i] = LEADING_SURROGATE; |
| 136 codeUnits[i + 1] = TRAILING_SURROGATE; | 148 codeUnits[i + 1] = TRAILING_SURROGATE; |
| 137 str = new String.fromCharCodes(codeUnits); | 149 str = new String.fromCharCodes(codeUnits); |
| 138 bytes = new List.filled(i + 4, CHAR_A); | 150 bytes = new List.filled(i + 4, CHAR_A); |
| 139 bytes[i] = UTF8_ENCODING[0]; | 151 bytes[i] = UTF8_ENCODING[0]; |
| 140 bytes[i + 1] = UTF8_ENCODING[1]; | 152 bytes[i + 1] = UTF8_ENCODING[1]; |
| 141 bytes[i + 2] = UTF8_ENCODING[2]; | 153 bytes[i + 2] = UTF8_ENCODING[2]; |
| 142 bytes[i + 3] = UTF8_ENCODING[3]; | 154 bytes[i + 3] = UTF8_ENCODING[3]; |
| 143 tests.add([bytes, str]); | 155 runTest([bytes, str]); |
| 144 | 156 |
| 145 codeUnits[i] = TRAILING_SURROGATE; | 157 codeUnits[i] = TRAILING_SURROGATE; |
| 146 codeUnits[i + 1] = TRAILING_SURROGATE; | 158 codeUnits[i + 1] = TRAILING_SURROGATE; |
| 147 str = new String.fromCharCodes(codeUnits); | 159 str = new String.fromCharCodes(codeUnits); |
| 148 bytes = new List.filled(i + 6, CHAR_A); | 160 bytes = new List.filled(i + 6, CHAR_A); |
| 149 bytes[i] = UTF8_TRAILING[0]; | 161 bytes[i] = UTF8_TRAILING[0]; |
| 150 bytes[i + 1] = UTF8_TRAILING[1]; | 162 bytes[i + 1] = UTF8_TRAILING[1]; |
| 151 bytes[i + 2] = UTF8_TRAILING[2]; | 163 bytes[i + 2] = UTF8_TRAILING[2]; |
| 152 bytes[i + 3] = UTF8_TRAILING[0]; | 164 bytes[i + 3] = UTF8_TRAILING[0]; |
| 153 bytes[i + 4] = UTF8_TRAILING[1]; | 165 bytes[i + 4] = UTF8_TRAILING[1]; |
| 154 bytes[i + 5] = UTF8_TRAILING[2]; | 166 bytes[i + 5] = UTF8_TRAILING[2]; |
| 155 tests.add([bytes, str]); | 167 runTest([bytes, str]); |
| 156 | 168 |
| 157 codeUnits[i] = LEADING_SURROGATE; | 169 codeUnits[i] = LEADING_SURROGATE; |
| 158 codeUnits[i + 1] = LEADING_SURROGATE; | 170 codeUnits[i + 1] = LEADING_SURROGATE; |
| 159 str = new String.fromCharCodes(codeUnits); | 171 str = new String.fromCharCodes(codeUnits); |
| 160 bytes = new List.filled(i + 6, CHAR_A); | 172 bytes = new List.filled(i + 6, CHAR_A); |
| 161 bytes[i] = UTF8_LEADING[0]; | 173 bytes[i] = UTF8_LEADING[0]; |
| 162 bytes[i + 1] = UTF8_LEADING[1]; | 174 bytes[i + 1] = UTF8_LEADING[1]; |
| 163 bytes[i + 2] = UTF8_LEADING[2]; | 175 bytes[i + 2] = UTF8_LEADING[2]; |
| 164 bytes[i + 3] = UTF8_LEADING[0]; | 176 bytes[i + 3] = UTF8_LEADING[0]; |
| 165 bytes[i + 4] = UTF8_LEADING[1]; | 177 bytes[i + 4] = UTF8_LEADING[1]; |
| 166 bytes[i + 5] = UTF8_LEADING[2]; | 178 bytes[i + 5] = UTF8_LEADING[2]; |
| 167 tests.add([bytes, str]); | 179 runTest([bytes, str]); |
| 168 | 180 |
| 169 codeUnits[i] = TRAILING_SURROGATE; | 181 codeUnits[i] = TRAILING_SURROGATE; |
| 170 codeUnits[i + 1] = LEADING_SURROGATE; | 182 codeUnits[i + 1] = LEADING_SURROGATE; |
| 171 str = new String.fromCharCodes(codeUnits); | 183 str = new String.fromCharCodes(codeUnits); |
| 172 bytes = new List.filled(i + 6, CHAR_A); | 184 bytes = new List.filled(i + 6, CHAR_A); |
| 173 bytes[i] = UTF8_TRAILING[0]; | 185 bytes[i] = UTF8_TRAILING[0]; |
| 174 bytes[i + 1] = UTF8_TRAILING[1]; | 186 bytes[i + 1] = UTF8_TRAILING[1]; |
| 175 bytes[i + 2] = UTF8_TRAILING[2]; | 187 bytes[i + 2] = UTF8_TRAILING[2]; |
| 176 bytes[i + 3] = UTF8_LEADING[0]; | 188 bytes[i + 3] = UTF8_LEADING[0]; |
| 177 bytes[i + 4] = UTF8_LEADING[1]; | 189 bytes[i + 4] = UTF8_LEADING[1]; |
| 178 bytes[i + 5] = UTF8_LEADING[2]; | 190 bytes[i + 5] = UTF8_LEADING[2]; |
| 179 tests.add([bytes, str]); | 191 runTest([bytes, str]); |
| 180 | 192 |
| 181 codeUnits.length = i + 3; | 193 codeUnits.length = i + 3; |
| 182 codeUnits[i] = LEADING_SURROGATE; | 194 codeUnits[i] = LEADING_SURROGATE; |
| 183 codeUnits[i + 1] = TRAILING_SURROGATE; | 195 codeUnits[i + 1] = TRAILING_SURROGATE; |
| 184 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. | 196 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. |
| 185 str = new String.fromCharCodes(codeUnits); | 197 str = new String.fromCharCodes(codeUnits); |
| 186 bytes = new List.filled(i + 5, CHAR_A); | 198 bytes = new List.filled(i + 5, CHAR_A); |
| 187 bytes[i] = UTF8_ENCODING[0]; | 199 bytes[i] = UTF8_ENCODING[0]; |
| 188 bytes[i + 1] = UTF8_ENCODING[1]; | 200 bytes[i + 1] = UTF8_ENCODING[1]; |
| 189 bytes[i + 2] = UTF8_ENCODING[2]; | 201 bytes[i + 2] = UTF8_ENCODING[2]; |
| 190 bytes[i + 3] = UTF8_ENCODING[3]; | 202 bytes[i + 3] = UTF8_ENCODING[3]; |
| 191 // No need to assign the 'a' character. The whole list is already filled | 203 // No need to assign the 'a' character. The whole list is already filled |
| 192 // with it. | 204 // with it. |
| 193 tests.add([bytes, str]); | 205 runTest([bytes, str]); |
| 194 | 206 |
| 195 codeUnits[i] = TRAILING_SURROGATE; | 207 codeUnits[i] = TRAILING_SURROGATE; |
| 196 codeUnits[i + 1] = TRAILING_SURROGATE; | 208 codeUnits[i + 1] = TRAILING_SURROGATE; |
| 197 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. | 209 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. |
| 198 str = new String.fromCharCodes(codeUnits); | 210 str = new String.fromCharCodes(codeUnits); |
| 199 bytes = new List.filled(i + 7, CHAR_A); | 211 bytes = new List.filled(i + 7, CHAR_A); |
| 200 bytes[i] = UTF8_TRAILING[0]; | 212 bytes[i] = UTF8_TRAILING[0]; |
| 201 bytes[i + 1] = UTF8_TRAILING[1]; | 213 bytes[i + 1] = UTF8_TRAILING[1]; |
| 202 bytes[i + 2] = UTF8_TRAILING[2]; | 214 bytes[i + 2] = UTF8_TRAILING[2]; |
| 203 bytes[i + 3] = UTF8_TRAILING[0]; | 215 bytes[i + 3] = UTF8_TRAILING[0]; |
| 204 bytes[i + 4] = UTF8_TRAILING[1]; | 216 bytes[i + 4] = UTF8_TRAILING[1]; |
| 205 bytes[i + 5] = UTF8_TRAILING[2]; | 217 bytes[i + 5] = UTF8_TRAILING[2]; |
| 206 tests.add([bytes, str]); | 218 runTest([bytes, str]); |
| 207 | 219 |
| 208 codeUnits[i] = LEADING_SURROGATE; | 220 codeUnits[i] = LEADING_SURROGATE; |
| 209 codeUnits[i + 1] = LEADING_SURROGATE; | 221 codeUnits[i + 1] = LEADING_SURROGATE; |
| 210 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. | 222 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. |
| 211 str = new String.fromCharCodes(codeUnits); | 223 str = new String.fromCharCodes(codeUnits); |
| 212 bytes = new List.filled(i + 7, CHAR_A); | 224 bytes = new List.filled(i + 7, CHAR_A); |
| 213 bytes[i] = UTF8_LEADING[0]; | 225 bytes[i] = UTF8_LEADING[0]; |
| 214 bytes[i + 1] = UTF8_LEADING[1]; | 226 bytes[i + 1] = UTF8_LEADING[1]; |
| 215 bytes[i + 2] = UTF8_LEADING[2]; | 227 bytes[i + 2] = UTF8_LEADING[2]; |
| 216 bytes[i + 3] = UTF8_LEADING[0]; | 228 bytes[i + 3] = UTF8_LEADING[0]; |
| 217 bytes[i + 4] = UTF8_LEADING[1]; | 229 bytes[i + 4] = UTF8_LEADING[1]; |
| 218 bytes[i + 5] = UTF8_LEADING[2]; | 230 bytes[i + 5] = UTF8_LEADING[2]; |
| 219 tests.add([bytes, str]); | 231 runTest([bytes, str]); |
| 220 | 232 |
| 221 codeUnits[i] = TRAILING_SURROGATE; | 233 codeUnits[i] = TRAILING_SURROGATE; |
| 222 codeUnits[i + 1] = LEADING_SURROGATE; | 234 codeUnits[i + 1] = LEADING_SURROGATE; |
| 223 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. | 235 codeUnits[i + 2] = CHAR_A; // Add trailing 'a'. |
| 224 str = new String.fromCharCodes(codeUnits); | 236 str = new String.fromCharCodes(codeUnits); |
| 225 bytes = new List.filled(i + 7, CHAR_A); | 237 bytes = new List.filled(i + 7, CHAR_A); |
| 226 bytes[i] = UTF8_TRAILING[0]; | 238 bytes[i] = UTF8_TRAILING[0]; |
| 227 bytes[i + 1] = UTF8_TRAILING[1]; | 239 bytes[i + 1] = UTF8_TRAILING[1]; |
| 228 bytes[i + 2] = UTF8_TRAILING[2]; | 240 bytes[i + 2] = UTF8_TRAILING[2]; |
| 229 bytes[i + 3] = UTF8_LEADING[0]; | 241 bytes[i + 3] = UTF8_LEADING[0]; |
| 230 bytes[i + 4] = UTF8_LEADING[1]; | 242 bytes[i + 4] = UTF8_LEADING[1]; |
| 231 bytes[i + 5] = UTF8_LEADING[2]; | 243 bytes[i + 5] = UTF8_LEADING[2]; |
| 232 tests.add([bytes, str]); | 244 runTest([bytes, str]); |
| 233 | 245 |
| 234 // Make sure the invariant is correct. | 246 // Make sure the invariant is correct. |
| 235 codeUnits[i] = CHAR_A; | 247 codeUnits[i] = CHAR_A; |
| 236 } | 248 } |
| 237 | |
| 238 for (var test in tests) { | |
| 239 List<int> bytes = test[0]; | |
| 240 String string = test[1]; | |
| 241 Expect.listEquals(bytes, encode(string)); | |
| 242 Expect.listEquals(bytes, encode2(string)); | |
| 243 Expect.listEquals(bytes, encode3(string)); | |
| 244 Expect.listEquals(bytes, encode4(string)); | |
| 245 Expect.listEquals(bytes, encode5(string)); | |
| 246 Expect.listEquals(bytes, encode6(string)); | |
| 247 Expect.listEquals(bytes, encode7(string)); | |
| 248 } | |
| 249 } | 249 } |
| OLD | NEW |