OLD | NEW |
| (Empty) |
1 /* | |
2 ********************************************************************** | |
3 * Copyright (C) 2010-2014, International Business Machines | |
4 * Corporation and others. All Rights Reserved. | |
5 ********************************************************************** | |
6 * file name: dicttrieperf.cpp | |
7 * encoding: US-ASCII | |
8 * tab size: 8 (not used) | |
9 * indentation:4 | |
10 * | |
11 * created on: 2010dec09 | |
12 * created by: Markus W. Scherer | |
13 * | |
14 * Performance test program for dictionary-type tries. | |
15 * | |
16 * Usage from within <ICU build tree>/test/perf/dicttrieperf/ : | |
17 * (Linux) | |
18 * make | |
19 * export LD_LIBRARY_PATH=../../../lib:../../../stubdata:../../../tools/ctestfw | |
20 * ./dicttrieperf --sourcedir <ICU build tree>/data/out/tmp --passes 3 --iterat
ions 1000 | |
21 * or | |
22 * ./dicttrieperf -f <ICU source tree>/source/data/brkitr/thaidict.txt --passes
3 --iterations 250 | |
23 */ | |
24 | |
25 #include <stdio.h> | |
26 #include <stdlib.h> | |
27 #include "unicode/bytestrie.h" | |
28 #include "unicode/bytestriebuilder.h" | |
29 #include "unicode/localpointer.h" | |
30 #include "unicode/ucharstrie.h" | |
31 #include "unicode/ucharstriebuilder.h" | |
32 #include "unicode/uperf.h" | |
33 #include "unicode/utext.h" | |
34 #include "charstr.h" | |
35 #include "package.h" | |
36 #include "toolutil.h" | |
37 #include "ucbuf.h" // struct ULine | |
38 #include "uoptions.h" | |
39 #include "uvectr32.h" | |
40 #include "cmemory.h" // for UPRV_LENGTHOF | |
41 | |
42 // Test object. | |
43 class DictionaryTriePerfTest : public UPerfTest { | |
44 public: | |
45 DictionaryTriePerfTest(int32_t argc, const char *argv[], UErrorCode &status) | |
46 : UPerfTest(argc, argv, NULL, 0, "", status), numTextLines(0) { | |
47 if(hasFile()) { | |
48 getLines(status); | |
49 for(int32_t i=0; i<numLines; ++i) { | |
50 // Skip comment lines (start with a character below 'A'). | |
51 if(lines[i].name[0]>=0x41) { | |
52 ++numTextLines; | |
53 // Remove trailing CR LF. | |
54 int32_t len=lines[i].len; | |
55 UChar c; | |
56 while(len>0 && ((c=lines[i].name[len-1])==0xa || c==0xd)) { | |
57 --len; | |
58 } | |
59 lines[i].len=len; | |
60 } | |
61 } | |
62 } | |
63 } | |
64 | |
65 virtual UPerfFunction *runIndexedTest(int32_t index, UBool exec, const char
*&name, char *par=NULL); | |
66 | |
67 const char *getSourceDir() const { return sourceDir; } | |
68 | |
69 UBool hasFile() const { return ucharBuf!=NULL; } | |
70 const ULine *getCachedLines() const { return lines; } | |
71 int32_t getNumLines() const { return numLines; } | |
72 int32_t numTextLines; // excluding comment lines | |
73 }; | |
74 | |
75 // Performance test function object. | |
76 // Loads icudt46l.dat (or whatever its current versioned filename) | |
77 // from the -s or --sourcedir path. | |
78 class PackageLookup : public UPerfFunction { | |
79 protected: | |
80 PackageLookup(const DictionaryTriePerfTest &perf) { | |
81 IcuToolErrorCode errorCode("PackageLookup()"); | |
82 CharString filename(perf.getSourceDir(), errorCode); | |
83 int32_t filenameLength=filename.length(); | |
84 if(filenameLength>0 && filename[filenameLength-1]!=U_FILE_SEP_CHAR && | |
85 filename[filenameLength-1]!=U_FILE_ALT_SEP_CHAR)
{ | |
86 filename.append(U_FILE_SEP_CHAR, errorCode); | |
87 } | |
88 filename.append(U_ICUDATA_NAME, errorCode); | |
89 filename.append(".dat", errorCode); | |
90 pkg.readPackage(filename.data()); | |
91 } | |
92 | |
93 public: | |
94 virtual ~PackageLookup() {} | |
95 | |
96 // virtual void call(UErrorCode* pErrorCode) { ... } | |
97 | |
98 virtual long getOperationsPerIteration() { | |
99 return pkg.getItemCount(); | |
100 } | |
101 | |
102 // virtual long getEventsPerIteration(); | |
103 | |
104 protected: | |
105 Package pkg; | |
106 }; | |
107 | |
108 struct TOCEntry { | |
109 int32_t nameOffset, dataOffset; | |
110 }; | |
111 | |
112 // Similar to ICU 4.6 offsetTOCLookupFn() (in ucmndata.c). | |
113 static int32_t simpleBinarySearch(const char *s, const char *names, const TOCEnt
ry *toc, int32_t count) { | |
114 int32_t start=0; | |
115 int32_t limit=count; | |
116 int32_t lastNumber=limit; | |
117 for(;;) { | |
118 int32_t number=(start+limit)/2; | |
119 if(lastNumber==number) { // have we moved? | |
120 return -1; // not found | |
121 } | |
122 lastNumber=number; | |
123 int32_t cmp=strcmp(s, names+toc[number].nameOffset); | |
124 if(cmp<0) { | |
125 limit=number; | |
126 } else if(cmp>0) { | |
127 start=number; | |
128 } else { // found s | |
129 return number; | |
130 } | |
131 } | |
132 } | |
133 | |
134 class BinarySearchPackageLookup : public PackageLookup { | |
135 public: | |
136 BinarySearchPackageLookup(const DictionaryTriePerfTest &perf) | |
137 : PackageLookup(perf) { | |
138 IcuToolErrorCode errorCode("BinarySearchPackageLookup()"); | |
139 int32_t count=pkg.getItemCount(); | |
140 toc=new TOCEntry[count]; | |
141 for(int32_t i=0; i<count; ++i) { | |
142 toc[i].nameOffset=itemNames.length(); | |
143 toc[i].dataOffset=i; // arbitrary value, see toc comment below | |
144 // The Package class removes the "icudt46l/" prefix. | |
145 // We restore that here for a fair performance test. | |
146 const char *name=pkg.getItem(i)->name; | |
147 itemNames.append("icudt46l/", errorCode); | |
148 itemNames.append(name, strlen(name)+1, errorCode); | |
149 } | |
150 printf("size of item names: %6ld\n", (long)itemNames.length()); | |
151 printf("size of TOC: %6ld\n", (long)(count*8)); | |
152 printf("total index size: %6ld\n", (long)(itemNames.length()+count*8))
; | |
153 } | |
154 virtual ~BinarySearchPackageLookup() { | |
155 delete[] toc; | |
156 } | |
157 | |
158 virtual void call(UErrorCode * /*pErrorCode*/) { | |
159 int32_t count=pkg.getItemCount(); | |
160 const char *itemNameChars=itemNames.data(); | |
161 const char *name=itemNameChars; | |
162 for(int32_t i=0; i<count; ++i) { | |
163 if(simpleBinarySearch(name, itemNameChars, toc, count)<0) { | |
164 fprintf(stderr, "item not found: %s\n", name); | |
165 } | |
166 name=strchr(name, 0)+1; | |
167 } | |
168 } | |
169 | |
170 protected: | |
171 CharString itemNames; | |
172 // toc imitates a .dat file's array of UDataOffsetTOCEntry | |
173 // with nameOffset and dataOffset. | |
174 // We don't need the dataOffsets, but we want to imitate the real | |
175 // memory density, to measure equivalent CPU cache usage. | |
176 TOCEntry *toc; | |
177 }; | |
178 | |
179 #ifndef MIN | |
180 #define MIN(a,b) (((a)<(b)) ? (a) : (b)) | |
181 #endif | |
182 | |
183 // Compare strings where we know the shared prefix length, | |
184 // and advance the prefix length as we find that the strings share even more cha
racters. | |
185 static int32_t strcmpAfterPrefix(const char *s1, const char *s2, int32_t &prefix
Length) { | |
186 int32_t pl=prefixLength; | |
187 s1+=pl; | |
188 s2+=pl; | |
189 int32_t cmp=0; | |
190 for(;;) { | |
191 int32_t c1=(uint8_t)*s1++; | |
192 int32_t c2=(uint8_t)*s2++; | |
193 cmp=c1-c2; | |
194 if(cmp!=0 || c1==0) { // different or done | |
195 break; | |
196 } | |
197 ++pl; // increment shared same-prefix length | |
198 } | |
199 prefixLength=pl; | |
200 return cmp; | |
201 } | |
202 | |
203 static int32_t prefixBinarySearch(const char *s, const char *names, const TOCEnt
ry *toc, int32_t count) { | |
204 if(count==0) { | |
205 return -1; | |
206 } | |
207 int32_t start=0; | |
208 int32_t limit=count; | |
209 // Remember the shared prefix between s, start and limit, | |
210 // and don't compare that shared prefix again. | |
211 // The shared prefix should get longer as we narrow the [start, limit[ range
. | |
212 int32_t startPrefixLength=0; | |
213 int32_t limitPrefixLength=0; | |
214 // Prime the prefix lengths so that we don't keep prefixLength at 0 until | |
215 // both the start and limit indexes have moved. | |
216 // At the same time, we find if s is one of the start and (limit-1) names, | |
217 // and if not, exclude them from the actual binary search. | |
218 if(0==strcmpAfterPrefix(s, names+toc[0].nameOffset, startPrefixLength)) { | |
219 return 0; | |
220 } | |
221 ++start; | |
222 --limit; | |
223 if(0==strcmpAfterPrefix(s, names+toc[limit].nameOffset, limitPrefixLength))
{ | |
224 return limit; | |
225 } | |
226 while(start<limit) { | |
227 int32_t i=(start+limit)/2; | |
228 int32_t prefixLength=MIN(startPrefixLength, limitPrefixLength); | |
229 int32_t cmp=strcmpAfterPrefix(s, names+toc[i].nameOffset, prefixLength); | |
230 if(cmp<0) { | |
231 limit=i; | |
232 limitPrefixLength=prefixLength; | |
233 } else if(cmp==0) { | |
234 return i; | |
235 } else { | |
236 start=i+1; | |
237 startPrefixLength=prefixLength; | |
238 } | |
239 } | |
240 return -1; | |
241 } | |
242 | |
243 class PrefixBinarySearchPackageLookup : public BinarySearchPackageLookup { | |
244 public: | |
245 PrefixBinarySearchPackageLookup(const DictionaryTriePerfTest &perf) | |
246 : BinarySearchPackageLookup(perf) {} | |
247 | |
248 virtual void call(UErrorCode * /*pErrorCode*/) { | |
249 int32_t count=pkg.getItemCount(); | |
250 const char *itemNameChars=itemNames.data(); | |
251 const char *name=itemNameChars; | |
252 for(int32_t i=0; i<count; ++i) { | |
253 if(prefixBinarySearch(name, itemNameChars, toc, count)<0) { | |
254 fprintf(stderr, "item not found: %s\n", name); | |
255 } | |
256 name=strchr(name, 0)+1; | |
257 } | |
258 } | |
259 }; | |
260 | |
261 static int32_t bytesTrieLookup(const char *s, const char *nameTrieBytes) { | |
262 BytesTrie trie(nameTrieBytes); | |
263 if(USTRINGTRIE_HAS_VALUE(trie.next(s, -1))) { | |
264 return trie.getValue(); | |
265 } else { | |
266 return -1; | |
267 } | |
268 } | |
269 | |
270 class BytesTriePackageLookup : public PackageLookup { | |
271 public: | |
272 BytesTriePackageLookup(const DictionaryTriePerfTest &perf) | |
273 : PackageLookup(perf) { | |
274 IcuToolErrorCode errorCode("BinarySearchPackageLookup()"); | |
275 builder=new BytesTrieBuilder(errorCode); | |
276 int32_t count=pkg.getItemCount(); | |
277 for(int32_t i=0; i<count; ++i) { | |
278 // The Package class removes the "icudt46l/" prefix. | |
279 // We restore that here for a fair performance test. | |
280 // We store all full names so that we do not have to reconstruct the
m | |
281 // in the call() function. | |
282 const char *name=pkg.getItem(i)->name; | |
283 int32_t offset=itemNames.length(); | |
284 itemNames.append("icudt46l/", errorCode); | |
285 itemNames.append(name, -1, errorCode); | |
286 // As value, set the data item index. | |
287 // In a real implementation, we would use that to get the | |
288 // start and limit offset of the data item. | |
289 StringPiece fullName(itemNames.toStringPiece()); | |
290 fullName.remove_prefix(offset); | |
291 builder->add(fullName, i, errorCode); | |
292 // NUL-terminate the name for call() to find the next one. | |
293 itemNames.append(0, errorCode); | |
294 } | |
295 int32_t length=builder->buildStringPiece(USTRINGTRIE_BUILD_SMALL, errorC
ode).length(); | |
296 printf("size of BytesTrie: %6ld\n", (long)length); | |
297 // count+1: +1 for the last-item limit offset which we should have alway
s had | |
298 printf("size of dataOffsets:%6ld\n", (long)((count+1)*4)); | |
299 printf("total index size: %6ld\n", (long)(length+(count+1)*4)); | |
300 } | |
301 virtual ~BytesTriePackageLookup() { | |
302 delete builder; | |
303 } | |
304 | |
305 virtual void call(UErrorCode *pErrorCode) { | |
306 int32_t count=pkg.getItemCount(); | |
307 const char *nameTrieBytes=builder->buildStringPiece(USTRINGTRIE_BUILD_SM
ALL, *pErrorCode).data(); | |
308 const char *name=itemNames.data(); | |
309 for(int32_t i=0; i<count; ++i) { | |
310 if(bytesTrieLookup(name, nameTrieBytes)<0) { | |
311 fprintf(stderr, "item not found: %s\n", name); | |
312 } | |
313 name=strchr(name, 0)+1; | |
314 } | |
315 } | |
316 | |
317 protected: | |
318 BytesTrieBuilder *builder; | |
319 CharString itemNames; | |
320 }; | |
321 | |
322 // Performance test function object. | |
323 // Each subclass loads a dictionary text file | |
324 // from the -s or --sourcedir path plus -f or --file-name. | |
325 // For example, <ICU source dir>/source/data/brkitr/thaidict.txt. | |
326 class DictLookup : public UPerfFunction { | |
327 public: | |
328 DictLookup(const DictionaryTriePerfTest &perfTest) : perf(perfTest) {} | |
329 | |
330 virtual long getOperationsPerIteration() { | |
331 return perf.numTextLines; | |
332 } | |
333 | |
334 protected: | |
335 const DictionaryTriePerfTest &perf; | |
336 }; | |
337 | |
338 // Closely imitate CompactTrieDictionary::matches(). | |
339 // Note: CompactTrieDictionary::matches() is part of its trie implementation, | |
340 // and while it loops over the text, it knows the current state. | |
341 // By contrast, this implementation uses UCharsTrie API functions that have to | |
342 // check the trie state each time and load/store state in the object. | |
343 // (Whether it hasNext() and whether it is in the middle of a linear-match node.
) | |
344 static int32_t | |
345 ucharsTrieMatches(UCharsTrie &trie, | |
346 UText *text, int32_t textLimit, | |
347 int32_t *lengths, int &count, int limit ) { | |
348 UChar32 c=utext_next32(text); | |
349 // Notes: | |
350 // a) CompactTrieDictionary::matches() does not check for U_SENTINEL. | |
351 // b) It also ignores non-BMP code points by casting to UChar! | |
352 if(c<0) { | |
353 return 0; | |
354 } | |
355 // Should be firstForCodePoint() but CompactTrieDictionary | |
356 // handles only code units. | |
357 UStringTrieResult result=trie.first(c); | |
358 int32_t numChars=1; | |
359 count=0; | |
360 for(;;) { | |
361 if(USTRINGTRIE_HAS_VALUE(result)) { | |
362 if(count<limit) { | |
363 // lengths[count++]=(int32_t)utext_getNativeIndex(text); | |
364 lengths[count++]=numChars; // CompactTrieDictionary just counts
chars too. | |
365 } | |
366 if(result==USTRINGTRIE_FINAL_VALUE) { | |
367 break; | |
368 } | |
369 } else if(result==USTRINGTRIE_NO_MATCH) { | |
370 break; | |
371 } | |
372 if(numChars>=textLimit) { | |
373 // Note: Why do we have both a text limit and a UText that knows its
length? | |
374 break; | |
375 } | |
376 UChar32 c=utext_next32(text); | |
377 // Notes: | |
378 // a) CompactTrieDictionary::matches() does not check for U_SENTINEL. | |
379 // b) It also ignores non-BMP code points by casting to UChar! | |
380 if(c<0) { | |
381 break; | |
382 } | |
383 ++numChars; | |
384 // Should be nextForCodePoint() but CompactTrieDictionary | |
385 // handles only code units. | |
386 result=trie.next(c); | |
387 } | |
388 #if 0 | |
389 // Note: CompactTrieDictionary::matches() comments say that it leaves the UT
ext | |
390 // after the longest prefix match and returns the number of characters | |
391 // that were matched. | |
392 if(index!=lastMatch) { | |
393 utext_setNativeIndex(text, lastMatch); | |
394 } | |
395 return lastMatch-start; | |
396 // However, it does not do either of these, so I am not trying to | |
397 // imitate it (or its docs) 100%. | |
398 #endif | |
399 return numChars; | |
400 } | |
401 | |
402 class UCharsTrieDictLookup : public DictLookup { | |
403 public: | |
404 UCharsTrieDictLookup(const DictionaryTriePerfTest &perfTest) | |
405 : DictLookup(perfTest), trie(NULL) { | |
406 IcuToolErrorCode errorCode("UCharsTrieDictLookup()"); | |
407 builder=new UCharsTrieBuilder(errorCode); | |
408 const ULine *lines=perf.getCachedLines(); | |
409 int32_t numLines=perf.getNumLines(); | |
410 for(int32_t i=0; i<numLines; ++i) { | |
411 // Skip comment lines (start with a character below 'A'). | |
412 if(lines[i].name[0]<0x41) { | |
413 continue; | |
414 } | |
415 builder->add(UnicodeString(FALSE, lines[i].name, lines[i].len), 0, e
rrorCode); | |
416 } | |
417 UnicodeString trieUChars; | |
418 int32_t length=builder->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, trie
UChars, errorCode).length(); | |
419 printf("size of UCharsTrie: %6ld bytes\n", (long)length*2); | |
420 trie=builder->build(USTRINGTRIE_BUILD_SMALL, errorCode); | |
421 } | |
422 | |
423 virtual ~UCharsTrieDictLookup() { | |
424 delete builder; | |
425 delete trie; | |
426 } | |
427 | |
428 protected: | |
429 UCharsTrieBuilder *builder; | |
430 UCharsTrie *trie; | |
431 }; | |
432 | |
433 class UCharsTrieDictMatches : public UCharsTrieDictLookup { | |
434 public: | |
435 UCharsTrieDictMatches(const DictionaryTriePerfTest &perfTest) | |
436 : UCharsTrieDictLookup(perfTest) {} | |
437 | |
438 virtual void call(UErrorCode *pErrorCode) { | |
439 UText text=UTEXT_INITIALIZER; | |
440 int32_t lengths[20]; | |
441 const ULine *lines=perf.getCachedLines(); | |
442 int32_t numLines=perf.getNumLines(); | |
443 for(int32_t i=0; i<numLines; ++i) { | |
444 // Skip comment lines (start with a character below 'A'). | |
445 if(lines[i].name[0]<0x41) { | |
446 continue; | |
447 } | |
448 utext_openUChars(&text, lines[i].name, lines[i].len, pErrorCode); | |
449 int32_t count=0; | |
450 ucharsTrieMatches(*trie, &text, lines[i].len, | |
451 lengths, count, UPRV_LENGTHOF(lengths)); | |
452 if(count==0 || lengths[count-1]!=lines[i].len) { | |
453 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
454 } | |
455 } | |
456 } | |
457 }; | |
458 | |
459 class UCharsTrieDictContains : public UCharsTrieDictLookup { | |
460 public: | |
461 UCharsTrieDictContains(const DictionaryTriePerfTest &perfTest) | |
462 : UCharsTrieDictLookup(perfTest) {} | |
463 | |
464 virtual void call(UErrorCode * /*pErrorCode*/) { | |
465 const ULine *lines=perf.getCachedLines(); | |
466 int32_t numLines=perf.getNumLines(); | |
467 for(int32_t i=0; i<numLines; ++i) { | |
468 // Skip comment lines (which start with a character below 'A'). | |
469 if(lines[i].name[0]<0x41) { | |
470 continue; | |
471 } | |
472 if(!USTRINGTRIE_HAS_VALUE(trie->reset().next(lines[i].name, lines[i]
.len))) { | |
473 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
474 } | |
475 } | |
476 } | |
477 }; | |
478 | |
479 static inline int32_t thaiCharToByte(UChar32 c) { | |
480 if(0xe00<=c && c<=0xefe) { | |
481 return c&0xff; | |
482 } else if(c==0x2e) { | |
483 return 0xff; | |
484 } else { | |
485 return -1; | |
486 } | |
487 } | |
488 | |
489 static UBool thaiWordToBytes(const UChar *s, int32_t length, | |
490 CharString &str, UErrorCode &errorCode) { | |
491 for(int32_t i=0; i<length; ++i) { | |
492 UChar c=s[i]; | |
493 int32_t b=thaiCharToByte(c); | |
494 if(b>=0) { | |
495 str.append((char)b, errorCode); | |
496 } else { | |
497 fprintf(stderr, "thaiWordToBytes(): unable to encode U+%04X as a byt
e\n", c); | |
498 return FALSE; | |
499 } | |
500 } | |
501 return TRUE; | |
502 } | |
503 | |
504 class BytesTrieDictLookup : public DictLookup { | |
505 public: | |
506 BytesTrieDictLookup(const DictionaryTriePerfTest &perfTest) | |
507 : DictLookup(perfTest), trie(NULL), noDict(FALSE) { | |
508 IcuToolErrorCode errorCode("BytesTrieDictLookup()"); | |
509 builder=new BytesTrieBuilder(errorCode); | |
510 CharString str; | |
511 const ULine *lines=perf.getCachedLines(); | |
512 int32_t numLines=perf.getNumLines(); | |
513 for(int32_t i=0; i<numLines; ++i) { | |
514 // Skip comment lines (start with a character below 'A'). | |
515 if(lines[i].name[0]<0x41) { | |
516 continue; | |
517 } | |
518 if(!thaiWordToBytes(lines[i].name, lines[i].len, str.clear(), errorC
ode)) { | |
519 fprintf(stderr, "thaiWordToBytes(): failed for word %ld (0-based
)\n", (long)i); | |
520 noDict=TRUE; | |
521 break; | |
522 } | |
523 builder->add(str.toStringPiece(), 0, errorCode); | |
524 } | |
525 if(!noDict) { | |
526 int32_t length=builder->buildStringPiece(USTRINGTRIE_BUILD_SMALL, er
rorCode).length(); | |
527 printf("size of BytesTrie: %6ld bytes\n", (long)length); | |
528 trie=builder->build(USTRINGTRIE_BUILD_SMALL, errorCode); | |
529 } | |
530 } | |
531 | |
532 virtual ~BytesTrieDictLookup() { | |
533 delete builder; | |
534 delete trie; | |
535 } | |
536 | |
537 protected: | |
538 BytesTrieBuilder *builder; | |
539 BytesTrie *trie; | |
540 UBool noDict; | |
541 }; | |
542 | |
543 static int32_t | |
544 bytesTrieMatches(BytesTrie &trie, | |
545 UText *text, int32_t textLimit, | |
546 int32_t *lengths, int &count, int limit ) { | |
547 UChar32 c=utext_next32(text); | |
548 if(c<0) { | |
549 return 0; | |
550 } | |
551 UStringTrieResult result=trie.first(thaiCharToByte(c)); | |
552 int32_t numChars=1; | |
553 count=0; | |
554 for(;;) { | |
555 if(USTRINGTRIE_HAS_VALUE(result)) { | |
556 if(count<limit) { | |
557 // lengths[count++]=(int32_t)utext_getNativeIndex(text); | |
558 lengths[count++]=numChars; // CompactTrieDictionary just counts
chars too. | |
559 } | |
560 if(result==USTRINGTRIE_FINAL_VALUE) { | |
561 break; | |
562 } | |
563 } else if(result==USTRINGTRIE_NO_MATCH) { | |
564 break; | |
565 } | |
566 if(numChars>=textLimit) { | |
567 break; | |
568 } | |
569 UChar32 c=utext_next32(text); | |
570 if(c<0) { | |
571 break; | |
572 } | |
573 ++numChars; | |
574 result=trie.next(thaiCharToByte(c)); | |
575 } | |
576 return numChars; | |
577 } | |
578 | |
579 class BytesTrieDictMatches : public BytesTrieDictLookup { | |
580 public: | |
581 BytesTrieDictMatches(const DictionaryTriePerfTest &perfTest) | |
582 : BytesTrieDictLookup(perfTest) {} | |
583 | |
584 virtual void call(UErrorCode *pErrorCode) { | |
585 if(noDict) { | |
586 return; | |
587 } | |
588 UText text=UTEXT_INITIALIZER; | |
589 int32_t lengths[20]; | |
590 const ULine *lines=perf.getCachedLines(); | |
591 int32_t numLines=perf.getNumLines(); | |
592 for(int32_t i=0; i<numLines; ++i) { | |
593 // Skip comment lines (start with a character below 'A'). | |
594 if(lines[i].name[0]<0x41) { | |
595 continue; | |
596 } | |
597 utext_openUChars(&text, lines[i].name, lines[i].len, pErrorCode); | |
598 int32_t count=0; | |
599 bytesTrieMatches(*trie, &text, lines[i].len, | |
600 lengths, count, UPRV_LENGTHOF(lengths)); | |
601 if(count==0 || lengths[count-1]!=lines[i].len) { | |
602 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
603 } | |
604 } | |
605 } | |
606 }; | |
607 | |
608 class BytesTrieDictContains : public BytesTrieDictLookup { | |
609 public: | |
610 BytesTrieDictContains(const DictionaryTriePerfTest &perfTest) | |
611 : BytesTrieDictLookup(perfTest) {} | |
612 | |
613 virtual void call(UErrorCode * /*pErrorCode*/) { | |
614 if(noDict) { | |
615 return; | |
616 } | |
617 const ULine *lines=perf.getCachedLines(); | |
618 int32_t numLines=perf.getNumLines(); | |
619 for(int32_t i=0; i<numLines; ++i) { | |
620 const UChar *line=lines[i].name; | |
621 // Skip comment lines (start with a character below 'A'). | |
622 if(line[0]<0x41) { | |
623 continue; | |
624 } | |
625 UStringTrieResult result=trie->first(thaiCharToByte(line[0])); | |
626 int32_t lineLength=lines[i].len; | |
627 for(int32_t j=1; j<lineLength; ++j) { | |
628 if(!USTRINGTRIE_HAS_NEXT(result)) { | |
629 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
630 break; | |
631 } | |
632 result=trie->next(thaiCharToByte(line[j])); | |
633 } | |
634 if(!USTRINGTRIE_HAS_VALUE(result)) { | |
635 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
636 } | |
637 } | |
638 } | |
639 }; | |
640 | |
641 UPerfFunction *DictionaryTriePerfTest::runIndexedTest(int32_t index, UBool exec, | |
642 const char *&name, char *
/*par*/) { | |
643 if(hasFile()) { | |
644 switch(index) { | |
645 case 0: | |
646 name="ucharstriematches"; | |
647 if(exec) { | |
648 return new UCharsTrieDictMatches(*this); | |
649 } | |
650 break; | |
651 case 1: | |
652 name="ucharstriecontains"; | |
653 if(exec) { | |
654 return new UCharsTrieDictContains(*this); | |
655 } | |
656 break; | |
657 case 2: | |
658 name="bytestriematches"; | |
659 if(exec) { | |
660 return new BytesTrieDictMatches(*this); | |
661 } | |
662 break; | |
663 case 3: | |
664 name="bytestriecontains"; | |
665 if(exec) { | |
666 return new BytesTrieDictContains(*this); | |
667 } | |
668 break; | |
669 default: | |
670 name=""; | |
671 break; | |
672 } | |
673 } else { | |
674 if(index==0 && exec) { | |
675 puts("Running BytesTrie perf tests on the .dat package file from the
--sourcedir.\n" | |
676 "For UCharsTrie perf tests on a dictionary text file, specify t
he -f or --file-name.\n"); | |
677 } | |
678 switch(index) { | |
679 case 0: | |
680 name="simplebinarysearch"; | |
681 if(exec) { | |
682 return new BinarySearchPackageLookup(*this); | |
683 } | |
684 break; | |
685 case 1: | |
686 name="prefixbinarysearch"; | |
687 if(exec) { | |
688 return new PrefixBinarySearchPackageLookup(*this); | |
689 } | |
690 break; | |
691 case 2: | |
692 name="bytestrie"; | |
693 if(exec) { | |
694 return new BytesTriePackageLookup(*this); | |
695 } | |
696 break; | |
697 default: | |
698 name=""; | |
699 break; | |
700 } | |
701 } | |
702 return NULL; | |
703 } | |
704 | |
705 int main(int argc, const char *argv[]) { | |
706 IcuToolErrorCode errorCode("dicttrieperf main()"); | |
707 DictionaryTriePerfTest test(argc, argv, errorCode); | |
708 if(errorCode.isFailure()) { | |
709 fprintf(stderr, "DictionaryTriePerfTest() failed: %s\n", errorCode.error
Name()); | |
710 test.usage(); | |
711 return errorCode.reset(); | |
712 } | |
713 if(!test.run()) { | |
714 fprintf(stderr, "FAILED: Tests could not be run, please check the argume
nts.\n"); | |
715 return -1; | |
716 } | |
717 return 0; | |
718 } | |
OLD | NEW |