OLD | NEW |
1 /******************************************************************** | 1 /******************************************************************** |
2 * COPYRIGHT: | 2 * COPYRIGHT: |
3 * Copyright (c) 2002-2012, International Business Machines Corporation and | 3 * Copyright (c) 2002-2014, International Business Machines Corporation and |
4 * others. All Rights Reserved. | 4 * others. All Rights Reserved. |
5 ********************************************************************/ | 5 ********************************************************************/ |
6 | 6 |
7 /** | 7 /** |
8 * UCAConformanceTest performs conformance tests defined in the data | 8 * UCAConformanceTest performs conformance tests defined in the data |
9 * files. ICU ships with stub data files, as the whole test are too | 9 * files. ICU ships with stub data files, as the whole test are too |
10 * long. To do the whole test, download the test files. | 10 * long. To do the whole test, download the test files. |
11 */ | 11 */ |
12 | 12 |
13 #include "unicode/utypes.h" | 13 #include "unicode/utypes.h" |
14 | 14 |
15 #if !UCONFIG_NO_COLLATION | 15 #if !UCONFIG_NO_COLLATION |
16 | 16 |
17 #include "ucaconf.h" | 17 #include "ucaconf.h" |
| 18 #include "unicode/sortkey.h" |
| 19 #include "unicode/tblcoll.h" |
18 #include "unicode/ustring.h" | 20 #include "unicode/ustring.h" |
19 #include "cmemory.h" | 21 #include "cmemory.h" |
20 #include "cstring.h" | 22 #include "cstring.h" |
21 #include "uparse.h" | 23 #include "uparse.h" |
22 | 24 |
23 UCAConformanceTest::UCAConformanceTest() : | 25 UCAConformanceTest::UCAConformanceTest() : |
24 rbUCA(NULL), | 26 rbUCA(NULL), |
25 testFile(NULL), | 27 testFile(NULL), |
26 status(U_ZERO_ERROR) | 28 status(U_ZERO_ERROR) |
27 { | 29 { |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
62 TESTCASE_AUTO(TestTableNonIgnorable); | 64 TESTCASE_AUTO(TestTableNonIgnorable); |
63 TESTCASE_AUTO(TestTableShifted); | 65 TESTCASE_AUTO(TestTableShifted); |
64 TESTCASE_AUTO(TestRulesNonIgnorable); | 66 TESTCASE_AUTO(TestRulesNonIgnorable); |
65 TESTCASE_AUTO(TestRulesShifted); | 67 TESTCASE_AUTO(TestRulesShifted); |
66 TESTCASE_AUTO_END; | 68 TESTCASE_AUTO_END; |
67 } | 69 } |
68 | 70 |
69 void UCAConformanceTest::initRbUCA() | 71 void UCAConformanceTest::initRbUCA() |
70 { | 72 { |
71 if(!rbUCA) { | 73 if(!rbUCA) { |
72 UnicodeString ucarules; | |
73 if (UCA) { | 74 if (UCA) { |
| 75 UnicodeString ucarules; |
74 UCA->getRules(UCOL_FULL_RULES, ucarules); | 76 UCA->getRules(UCOL_FULL_RULES, ucarules); |
75 rbUCA = new RuleBasedCollator(ucarules, status); | 77 rbUCA = new RuleBasedCollator(ucarules, status); |
76 if (U_FAILURE(status)) { | 78 if (U_FAILURE(status)) { |
77 dataerrln("Failure creating UCA rule-based collator: %s", u_erro
rName(status)); | 79 dataerrln("Failure creating UCA rule-based collator: %s", u_erro
rName(status)); |
78 return; | 80 return; |
79 } | 81 } |
80 } else { | 82 } else { |
81 dataerrln("Failure creating UCA rule-based collator: %s", u_errorNam
e(status)); | 83 dataerrln("Failure creating UCA rule-based collator: %s", u_errorNam
e(status)); |
82 return; | 84 return; |
83 } | 85 } |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
135 | 137 |
136 if (testFile == 0) { | 138 if (testFile == 0) { |
137 *(buffer+bufLen) = 0; | 139 *(buffer+bufLen) = 0; |
138 dataerrln("Could not open any of the conformance test files, tri
ed opening base %s\n", buffer); | 140 dataerrln("Could not open any of the conformance test files, tri
ed opening base %s\n", buffer); |
139 return; | 141 return; |
140 } else { | 142 } else { |
141 infoln( | 143 infoln( |
142 "INFO: Working with the stub file.\n" | 144 "INFO: Working with the stub file.\n" |
143 "If you need the full conformance test, please\n" | 145 "If you need the full conformance test, please\n" |
144 "download the appropriate data files from:\n" | 146 "download the appropriate data files from:\n" |
145 "http://source.icu-project.org/repos/icu/tools/trunk/unicode
tools/com/ibm/text/data/"); | 147 "http://unicode.org/cldr/trac/browser/trunk/common/uca"); |
146 } | 148 } |
147 } | 149 } |
148 } | 150 } |
149 } | 151 } |
150 | 152 |
151 static const uint32_t IS_SHIFTED = 1; | 153 static const uint32_t IS_SHIFTED = 1; |
152 static const uint32_t FROM_RULES = 2; | 154 static const uint32_t FROM_RULES = 2; |
153 | 155 |
154 static UBool | 156 static UBool |
155 skipLineBecauseOfBug(const UChar *s, int32_t length, uint32_t flags) { | 157 skipLineBecauseOfBug(const UChar *s, int32_t length, uint32_t flags) { |
156 // TODO: Fix ICU ticket #8052 | 158 // Add temporary exceptions here if there are ICU bugs, until we can fix the
m. |
157 if(length >= 3 && | 159 // For examples see the ICU 52 version of this file. |
158 (s[0] == 0xfb2 || s[0] == 0xfb3) && | 160 (void)s; |
159 s[1] == 0x334 && | 161 (void)length; |
160 (s[2] == 0xf73 || s[2] == 0xf75 || s[2] == 0xf81)) { | 162 (void)flags; |
161 return TRUE; | |
162 } | |
163 // TODO: Fix ICU ticket #9361 | |
164 if((flags & IS_SHIFTED) != 0 && length >= 2 && s[0] == 0xfffe) { | |
165 return TRUE; | |
166 } | |
167 // TODO: Fix tailoring builder, ICU ticket #9593. | |
168 UChar c; | |
169 if((flags & FROM_RULES) != 0 && length >= 2 && ((c = s[1]) == 0xedc || c ==
0xedd)) { | |
170 return TRUE; | |
171 } | |
172 return FALSE; | 163 return FALSE; |
173 } | 164 } |
174 | 165 |
175 static UCollationResult | 166 static UCollationResult |
176 normalizeResult(int32_t result) { | 167 normalizeResult(int32_t result) { |
177 return result<0 ? UCOL_LESS : result==0 ? UCOL_EQUAL : UCOL_GREATER; | 168 return result<0 ? UCOL_LESS : result==0 ? UCOL_EQUAL : UCOL_GREATER; |
178 } | 169 } |
179 | 170 |
180 void UCAConformanceTest::testConformance(const Collator *coll) | 171 void UCAConformanceTest::testConformance(const Collator *coll) |
181 { | 172 { |
182 if(testFile == 0) { | 173 if(testFile == 0) { |
183 return; | 174 return; |
184 } | 175 } |
185 uint32_t skipFlags = 0; | 176 uint32_t skipFlags = 0; |
186 if(coll->getAttribute(UCOL_ALTERNATE_HANDLING, status) == UCOL_SHIFTED) { | 177 if(coll->getAttribute(UCOL_ALTERNATE_HANDLING, status) == UCOL_SHIFTED) { |
187 skipFlags |= IS_SHIFTED; | 178 skipFlags |= IS_SHIFTED; |
188 } | 179 } |
189 if(coll == rbUCA) { | 180 if(coll == rbUCA) { |
190 skipFlags |= FROM_RULES; | 181 skipFlags |= FROM_RULES; |
191 } | 182 } |
192 | 183 |
| 184 logln("-prop:ucaconfnosortkeys=1 turns off getSortKey() in UCAConformanceTes
t"); |
| 185 UBool withSortKeys = getProperty("ucaconfnosortkeys") == NULL; |
| 186 |
193 int32_t line = 0; | 187 int32_t line = 0; |
194 | 188 |
195 UChar b1[1024], b2[1024]; | 189 UChar b1[1024], b2[1024]; |
196 UChar *buffer = b1, *oldB = NULL; | 190 UChar *buffer = b1, *oldB = NULL; |
197 | 191 |
198 char lineB1[1024], lineB2[1024]; | 192 char lineB1[1024], lineB2[1024]; |
199 char *lineB = lineB1, *oldLineB = lineB2; | 193 char *lineB = lineB1, *oldLineB = lineB2; |
200 | 194 |
201 uint8_t sk1[1024], sk2[1024]; | 195 uint8_t sk1[1024], sk2[1024]; |
202 uint8_t *oldSk = NULL, *newSk = sk1; | 196 uint8_t *oldSk = NULL, *newSk = sk1; |
(...skipping 16 matching lines...) Expand all Loading... |
219 (long)line, u_errorName(status), lineB); | 213 (long)line, u_errorName(status), lineB); |
220 status = U_ZERO_ERROR; | 214 status = U_ZERO_ERROR; |
221 } | 215 } |
222 buffer[buflen] = 0; | 216 buffer[buflen] = 0; |
223 | 217 |
224 if(skipLineBecauseOfBug(buffer, buflen, skipFlags)) { | 218 if(skipLineBecauseOfBug(buffer, buflen, skipFlags)) { |
225 logln("Skipping line %i because of a known bug", line); | 219 logln("Skipping line %i because of a known bug", line); |
226 continue; | 220 continue; |
227 } | 221 } |
228 | 222 |
229 int32_t resLen = coll->getSortKey(buffer, buflen, newSk, 1024); | 223 int32_t resLen = withSortKeys ? coll->getSortKey(buffer, buflen, newSk,
1024) : 0; |
230 | 224 |
231 if(oldSk != NULL) { | 225 if(oldSk != NULL) { |
232 int32_t skres = strcmp((char *)oldSk, (char *)newSk); | 226 UBool ok=TRUE; |
| 227 int32_t skres = withSortKeys ? strcmp((char *)oldSk, (char *)newSk)
: 0; |
233 int32_t cmpres = coll->compare(oldB, oldBlen, buffer, buflen, status
); | 228 int32_t cmpres = coll->compare(oldB, oldBlen, buffer, buflen, status
); |
234 int32_t cmpres2 = coll->compare(buffer, buflen, oldB, oldBlen, statu
s); | 229 int32_t cmpres2 = coll->compare(buffer, buflen, oldB, oldBlen, statu
s); |
235 | 230 |
236 if(cmpres != -cmpres2) { | 231 if(cmpres != -cmpres2) { |
237 errln("Compare result not symmetrical on line %i", line); | 232 errln("Compare result not symmetrical on line %i: " |
| 233 "previous vs. current (%d) / current vs. previous (%d)", |
| 234 line, cmpres, cmpres2); |
| 235 ok = FALSE; |
238 } | 236 } |
239 | 237 |
240 if(cmpres != normalizeResult(skres)) { | 238 // TODO: Compare with normalization turned off if the input passes t
he FCD test. |
| 239 |
| 240 if(withSortKeys && cmpres != normalizeResult(skres)) { |
241 errln("Difference between coll->compare (%d) and sortkey compare
(%d) on line %i", | 241 errln("Difference between coll->compare (%d) and sortkey compare
(%d) on line %i", |
242 cmpres, skres, line); | 242 cmpres, skres, line); |
243 errln(" Previous data line %s", oldLineB); | 243 ok = FALSE; |
244 errln(" Current data line %s", lineB); | |
245 } | 244 } |
246 | 245 |
247 int32_t res = cmpres; | 246 int32_t res = cmpres; |
248 if(res == 0 && !isAtLeastUCA62) { | 247 if(res == 0 && !isAtLeastUCA62) { |
249 // Up to UCA 6.1, the collation test files use a custom tie-brea
ker, | 248 // Up to UCA 6.1, the collation test files use a custom tie-brea
ker, |
250 // comparing the raw input strings. | 249 // comparing the raw input strings. |
251 res = u_strcmpCodePointOrder(oldB, buffer); | 250 res = u_strcmpCodePointOrder(oldB, buffer); |
252 // Starting with UCA 6.2, the collation test files use the stand
ard UCA tie-breaker, | 251 // Starting with UCA 6.2, the collation test files use the stand
ard UCA tie-breaker, |
253 // comparing the NFD versions of the input strings, | 252 // comparing the NFD versions of the input strings, |
254 // which we do via setting strength=identical. | 253 // which we do via setting strength=identical. |
255 } | 254 } |
256 if(res > 0) { | 255 if(res > 0) { |
257 errln("Line %i is not greater or equal than previous line", line
); | 256 errln("Line %i is not greater or equal than previous line", line
); |
| 257 ok = FALSE; |
| 258 } |
| 259 |
| 260 if(!ok) { |
258 errln(" Previous data line %s", oldLineB); | 261 errln(" Previous data line %s", oldLineB); |
259 errln(" Current data line %s", lineB); | 262 errln(" Current data line %s", lineB); |
260 UnicodeString oldS, newS; | 263 if(withSortKeys) { |
261 prettify(CollationKey(oldSk, oldLen), oldS); | 264 UnicodeString oldS, newS; |
262 prettify(CollationKey(newSk, resLen), newS); | 265 prettify(CollationKey(oldSk, oldLen), oldS); |
263 errln(" Previous key: "+oldS); | 266 prettify(CollationKey(newSk, resLen), newS); |
264 errln(" Current key: "+newS); | 267 errln(" Previous key: "+oldS); |
| 268 errln(" Current key: "+newS); |
| 269 } |
265 } | 270 } |
266 } | 271 } |
267 | 272 |
268 // swap buffers | 273 // swap buffers |
269 oldLineB = lineB; | 274 oldLineB = lineB; |
270 oldB = buffer; | 275 oldB = buffer; |
271 oldSk = newSk; | 276 oldSk = newSk; |
272 if(lineB == lineB1) { | 277 if(lineB == lineB1) { |
273 lineB = lineB2; | 278 lineB = lineB2; |
274 buffer = b2; | 279 buffer = b2; |
(...skipping 22 matching lines...) Expand all Loading... |
297 if (U_FAILURE(status)) { | 302 if (U_FAILURE(status)) { |
298 dataerrln("Error running UCA Conformance Test: %s", u_errorName(status))
; | 303 dataerrln("Error running UCA Conformance Test: %s", u_errorName(status))
; |
299 return; | 304 return; |
300 } | 305 } |
301 setCollShifted(UCA); | 306 setCollShifted(UCA); |
302 openTestFile("SHIFTED"); | 307 openTestFile("SHIFTED"); |
303 testConformance(UCA); | 308 testConformance(UCA); |
304 } | 309 } |
305 | 310 |
306 void UCAConformanceTest::TestRulesNonIgnorable(/* par */) { | 311 void UCAConformanceTest::TestRulesNonIgnorable(/* par */) { |
| 312 if(logKnownIssue("cldrbug:6745", "UCARules.txt has problems")) { return; } |
307 initRbUCA(); | 313 initRbUCA(); |
308 | 314 |
309 if(U_SUCCESS(status)) { | 315 if(U_SUCCESS(status)) { |
310 setCollNonIgnorable(rbUCA); | 316 setCollNonIgnorable(rbUCA); |
311 openTestFile("NON_IGNORABLE"); | 317 openTestFile("NON_IGNORABLE"); |
312 testConformance(rbUCA); | 318 testConformance(rbUCA); |
313 } | 319 } |
314 } | 320 } |
315 | 321 |
316 void UCAConformanceTest::TestRulesShifted(/* par */) { | 322 void UCAConformanceTest::TestRulesShifted(/* par */) { |
317 logln("This test is currently disabled, as it is impossible to " | 323 logln("This test is currently disabled, as it is impossible to " |
318 "wholly represent fractional UCA using tailoring rules."); | 324 "wholly represent fractional UCA using tailoring rules."); |
319 return; | 325 return; |
320 | 326 |
321 initRbUCA(); | 327 initRbUCA(); |
322 | 328 |
323 if(U_SUCCESS(status)) { | 329 if(U_SUCCESS(status)) { |
324 setCollShifted(rbUCA); | 330 setCollShifted(rbUCA); |
325 openTestFile("SHIFTED"); | 331 openTestFile("SHIFTED"); |
326 testConformance(rbUCA); | 332 testConformance(rbUCA); |
327 } | 333 } |
328 } | 334 } |
329 | 335 |
330 #endif /* #if !UCONFIG_NO_COLLATION */ | 336 #endif /* #if !UCONFIG_NO_COLLATION */ |
OLD | NEW |