| OLD | NEW |
| (Empty) |
| 1 /* | |
| 2 ********************************************************************** | |
| 3 * Copyright (C) 2010-2014, International Business Machines | |
| 4 * Corporation and others. All Rights Reserved. | |
| 5 ********************************************************************** | |
| 6 * file name: dicttrieperf.cpp | |
| 7 * encoding: US-ASCII | |
| 8 * tab size: 8 (not used) | |
| 9 * indentation:4 | |
| 10 * | |
| 11 * created on: 2010dec09 | |
| 12 * created by: Markus W. Scherer | |
| 13 * | |
| 14 * Performance test program for dictionary-type tries. | |
| 15 * | |
| 16 * Usage from within <ICU build tree>/test/perf/dicttrieperf/ : | |
| 17 * (Linux) | |
| 18 * make | |
| 19 * export LD_LIBRARY_PATH=../../../lib:../../../stubdata:../../../tools/ctestfw | |
| 20 * ./dicttrieperf --sourcedir <ICU build tree>/data/out/tmp --passes 3 --iterat
ions 1000 | |
| 21 * or | |
| 22 * ./dicttrieperf -f <ICU source tree>/source/data/brkitr/thaidict.txt --passes
3 --iterations 250 | |
| 23 */ | |
| 24 | |
| 25 #include <stdio.h> | |
| 26 #include <stdlib.h> | |
| 27 #include "unicode/bytestrie.h" | |
| 28 #include "unicode/bytestriebuilder.h" | |
| 29 #include "unicode/localpointer.h" | |
| 30 #include "unicode/ucharstrie.h" | |
| 31 #include "unicode/ucharstriebuilder.h" | |
| 32 #include "unicode/uperf.h" | |
| 33 #include "unicode/utext.h" | |
| 34 #include "charstr.h" | |
| 35 #include "package.h" | |
| 36 #include "toolutil.h" | |
| 37 #include "ucbuf.h" // struct ULine | |
| 38 #include "uoptions.h" | |
| 39 #include "uvectr32.h" | |
| 40 #include "cmemory.h" // for UPRV_LENGTHOF | |
| 41 | |
| 42 // Test object. | |
| 43 class DictionaryTriePerfTest : public UPerfTest { | |
| 44 public: | |
| 45 DictionaryTriePerfTest(int32_t argc, const char *argv[], UErrorCode &status) | |
| 46 : UPerfTest(argc, argv, NULL, 0, "", status), numTextLines(0) { | |
| 47 if(hasFile()) { | |
| 48 getLines(status); | |
| 49 for(int32_t i=0; i<numLines; ++i) { | |
| 50 // Skip comment lines (start with a character below 'A'). | |
| 51 if(lines[i].name[0]>=0x41) { | |
| 52 ++numTextLines; | |
| 53 // Remove trailing CR LF. | |
| 54 int32_t len=lines[i].len; | |
| 55 UChar c; | |
| 56 while(len>0 && ((c=lines[i].name[len-1])==0xa || c==0xd)) { | |
| 57 --len; | |
| 58 } | |
| 59 lines[i].len=len; | |
| 60 } | |
| 61 } | |
| 62 } | |
| 63 } | |
| 64 | |
| 65 virtual UPerfFunction *runIndexedTest(int32_t index, UBool exec, const char
*&name, char *par=NULL); | |
| 66 | |
| 67 const char *getSourceDir() const { return sourceDir; } | |
| 68 | |
| 69 UBool hasFile() const { return ucharBuf!=NULL; } | |
| 70 const ULine *getCachedLines() const { return lines; } | |
| 71 int32_t getNumLines() const { return numLines; } | |
| 72 int32_t numTextLines; // excluding comment lines | |
| 73 }; | |
| 74 | |
| 75 // Performance test function object. | |
| 76 // Loads icudt46l.dat (or whatever its current versioned filename) | |
| 77 // from the -s or --sourcedir path. | |
| 78 class PackageLookup : public UPerfFunction { | |
| 79 protected: | |
| 80 PackageLookup(const DictionaryTriePerfTest &perf) { | |
| 81 IcuToolErrorCode errorCode("PackageLookup()"); | |
| 82 CharString filename(perf.getSourceDir(), errorCode); | |
| 83 int32_t filenameLength=filename.length(); | |
| 84 if(filenameLength>0 && filename[filenameLength-1]!=U_FILE_SEP_CHAR && | |
| 85 filename[filenameLength-1]!=U_FILE_ALT_SEP_CHAR)
{ | |
| 86 filename.append(U_FILE_SEP_CHAR, errorCode); | |
| 87 } | |
| 88 filename.append(U_ICUDATA_NAME, errorCode); | |
| 89 filename.append(".dat", errorCode); | |
| 90 pkg.readPackage(filename.data()); | |
| 91 } | |
| 92 | |
| 93 public: | |
| 94 virtual ~PackageLookup() {} | |
| 95 | |
| 96 // virtual void call(UErrorCode* pErrorCode) { ... } | |
| 97 | |
| 98 virtual long getOperationsPerIteration() { | |
| 99 return pkg.getItemCount(); | |
| 100 } | |
| 101 | |
| 102 // virtual long getEventsPerIteration(); | |
| 103 | |
| 104 protected: | |
| 105 Package pkg; | |
| 106 }; | |
| 107 | |
| 108 struct TOCEntry { | |
| 109 int32_t nameOffset, dataOffset; | |
| 110 }; | |
| 111 | |
| 112 // Similar to ICU 4.6 offsetTOCLookupFn() (in ucmndata.c). | |
| 113 static int32_t simpleBinarySearch(const char *s, const char *names, const TOCEnt
ry *toc, int32_t count) { | |
| 114 int32_t start=0; | |
| 115 int32_t limit=count; | |
| 116 int32_t lastNumber=limit; | |
| 117 for(;;) { | |
| 118 int32_t number=(start+limit)/2; | |
| 119 if(lastNumber==number) { // have we moved? | |
| 120 return -1; // not found | |
| 121 } | |
| 122 lastNumber=number; | |
| 123 int32_t cmp=strcmp(s, names+toc[number].nameOffset); | |
| 124 if(cmp<0) { | |
| 125 limit=number; | |
| 126 } else if(cmp>0) { | |
| 127 start=number; | |
| 128 } else { // found s | |
| 129 return number; | |
| 130 } | |
| 131 } | |
| 132 } | |
| 133 | |
| 134 class BinarySearchPackageLookup : public PackageLookup { | |
| 135 public: | |
| 136 BinarySearchPackageLookup(const DictionaryTriePerfTest &perf) | |
| 137 : PackageLookup(perf) { | |
| 138 IcuToolErrorCode errorCode("BinarySearchPackageLookup()"); | |
| 139 int32_t count=pkg.getItemCount(); | |
| 140 toc=new TOCEntry[count]; | |
| 141 for(int32_t i=0; i<count; ++i) { | |
| 142 toc[i].nameOffset=itemNames.length(); | |
| 143 toc[i].dataOffset=i; // arbitrary value, see toc comment below | |
| 144 // The Package class removes the "icudt46l/" prefix. | |
| 145 // We restore that here for a fair performance test. | |
| 146 const char *name=pkg.getItem(i)->name; | |
| 147 itemNames.append("icudt46l/", errorCode); | |
| 148 itemNames.append(name, strlen(name)+1, errorCode); | |
| 149 } | |
| 150 printf("size of item names: %6ld\n", (long)itemNames.length()); | |
| 151 printf("size of TOC: %6ld\n", (long)(count*8)); | |
| 152 printf("total index size: %6ld\n", (long)(itemNames.length()+count*8))
; | |
| 153 } | |
| 154 virtual ~BinarySearchPackageLookup() { | |
| 155 delete[] toc; | |
| 156 } | |
| 157 | |
| 158 virtual void call(UErrorCode * /*pErrorCode*/) { | |
| 159 int32_t count=pkg.getItemCount(); | |
| 160 const char *itemNameChars=itemNames.data(); | |
| 161 const char *name=itemNameChars; | |
| 162 for(int32_t i=0; i<count; ++i) { | |
| 163 if(simpleBinarySearch(name, itemNameChars, toc, count)<0) { | |
| 164 fprintf(stderr, "item not found: %s\n", name); | |
| 165 } | |
| 166 name=strchr(name, 0)+1; | |
| 167 } | |
| 168 } | |
| 169 | |
| 170 protected: | |
| 171 CharString itemNames; | |
| 172 // toc imitates a .dat file's array of UDataOffsetTOCEntry | |
| 173 // with nameOffset and dataOffset. | |
| 174 // We don't need the dataOffsets, but we want to imitate the real | |
| 175 // memory density, to measure equivalent CPU cache usage. | |
| 176 TOCEntry *toc; | |
| 177 }; | |
| 178 | |
| 179 #ifndef MIN | |
| 180 #define MIN(a,b) (((a)<(b)) ? (a) : (b)) | |
| 181 #endif | |
| 182 | |
| 183 // Compare strings where we know the shared prefix length, | |
| 184 // and advance the prefix length as we find that the strings share even more cha
racters. | |
| 185 static int32_t strcmpAfterPrefix(const char *s1, const char *s2, int32_t &prefix
Length) { | |
| 186 int32_t pl=prefixLength; | |
| 187 s1+=pl; | |
| 188 s2+=pl; | |
| 189 int32_t cmp=0; | |
| 190 for(;;) { | |
| 191 int32_t c1=(uint8_t)*s1++; | |
| 192 int32_t c2=(uint8_t)*s2++; | |
| 193 cmp=c1-c2; | |
| 194 if(cmp!=0 || c1==0) { // different or done | |
| 195 break; | |
| 196 } | |
| 197 ++pl; // increment shared same-prefix length | |
| 198 } | |
| 199 prefixLength=pl; | |
| 200 return cmp; | |
| 201 } | |
| 202 | |
| 203 static int32_t prefixBinarySearch(const char *s, const char *names, const TOCEnt
ry *toc, int32_t count) { | |
| 204 if(count==0) { | |
| 205 return -1; | |
| 206 } | |
| 207 int32_t start=0; | |
| 208 int32_t limit=count; | |
| 209 // Remember the shared prefix between s, start and limit, | |
| 210 // and don't compare that shared prefix again. | |
| 211 // The shared prefix should get longer as we narrow the [start, limit[ range
. | |
| 212 int32_t startPrefixLength=0; | |
| 213 int32_t limitPrefixLength=0; | |
| 214 // Prime the prefix lengths so that we don't keep prefixLength at 0 until | |
| 215 // both the start and limit indexes have moved. | |
| 216 // At the same time, we find if s is one of the start and (limit-1) names, | |
| 217 // and if not, exclude them from the actual binary search. | |
| 218 if(0==strcmpAfterPrefix(s, names+toc[0].nameOffset, startPrefixLength)) { | |
| 219 return 0; | |
| 220 } | |
| 221 ++start; | |
| 222 --limit; | |
| 223 if(0==strcmpAfterPrefix(s, names+toc[limit].nameOffset, limitPrefixLength))
{ | |
| 224 return limit; | |
| 225 } | |
| 226 while(start<limit) { | |
| 227 int32_t i=(start+limit)/2; | |
| 228 int32_t prefixLength=MIN(startPrefixLength, limitPrefixLength); | |
| 229 int32_t cmp=strcmpAfterPrefix(s, names+toc[i].nameOffset, prefixLength); | |
| 230 if(cmp<0) { | |
| 231 limit=i; | |
| 232 limitPrefixLength=prefixLength; | |
| 233 } else if(cmp==0) { | |
| 234 return i; | |
| 235 } else { | |
| 236 start=i+1; | |
| 237 startPrefixLength=prefixLength; | |
| 238 } | |
| 239 } | |
| 240 return -1; | |
| 241 } | |
| 242 | |
| 243 class PrefixBinarySearchPackageLookup : public BinarySearchPackageLookup { | |
| 244 public: | |
| 245 PrefixBinarySearchPackageLookup(const DictionaryTriePerfTest &perf) | |
| 246 : BinarySearchPackageLookup(perf) {} | |
| 247 | |
| 248 virtual void call(UErrorCode * /*pErrorCode*/) { | |
| 249 int32_t count=pkg.getItemCount(); | |
| 250 const char *itemNameChars=itemNames.data(); | |
| 251 const char *name=itemNameChars; | |
| 252 for(int32_t i=0; i<count; ++i) { | |
| 253 if(prefixBinarySearch(name, itemNameChars, toc, count)<0) { | |
| 254 fprintf(stderr, "item not found: %s\n", name); | |
| 255 } | |
| 256 name=strchr(name, 0)+1; | |
| 257 } | |
| 258 } | |
| 259 }; | |
| 260 | |
| 261 static int32_t bytesTrieLookup(const char *s, const char *nameTrieBytes) { | |
| 262 BytesTrie trie(nameTrieBytes); | |
| 263 if(USTRINGTRIE_HAS_VALUE(trie.next(s, -1))) { | |
| 264 return trie.getValue(); | |
| 265 } else { | |
| 266 return -1; | |
| 267 } | |
| 268 } | |
| 269 | |
| 270 class BytesTriePackageLookup : public PackageLookup { | |
| 271 public: | |
| 272 BytesTriePackageLookup(const DictionaryTriePerfTest &perf) | |
| 273 : PackageLookup(perf) { | |
| 274 IcuToolErrorCode errorCode("BinarySearchPackageLookup()"); | |
| 275 builder=new BytesTrieBuilder(errorCode); | |
| 276 int32_t count=pkg.getItemCount(); | |
| 277 for(int32_t i=0; i<count; ++i) { | |
| 278 // The Package class removes the "icudt46l/" prefix. | |
| 279 // We restore that here for a fair performance test. | |
| 280 // We store all full names so that we do not have to reconstruct the
m | |
| 281 // in the call() function. | |
| 282 const char *name=pkg.getItem(i)->name; | |
| 283 int32_t offset=itemNames.length(); | |
| 284 itemNames.append("icudt46l/", errorCode); | |
| 285 itemNames.append(name, -1, errorCode); | |
| 286 // As value, set the data item index. | |
| 287 // In a real implementation, we would use that to get the | |
| 288 // start and limit offset of the data item. | |
| 289 StringPiece fullName(itemNames.toStringPiece()); | |
| 290 fullName.remove_prefix(offset); | |
| 291 builder->add(fullName, i, errorCode); | |
| 292 // NUL-terminate the name for call() to find the next one. | |
| 293 itemNames.append(0, errorCode); | |
| 294 } | |
| 295 int32_t length=builder->buildStringPiece(USTRINGTRIE_BUILD_SMALL, errorC
ode).length(); | |
| 296 printf("size of BytesTrie: %6ld\n", (long)length); | |
| 297 // count+1: +1 for the last-item limit offset which we should have alway
s had | |
| 298 printf("size of dataOffsets:%6ld\n", (long)((count+1)*4)); | |
| 299 printf("total index size: %6ld\n", (long)(length+(count+1)*4)); | |
| 300 } | |
| 301 virtual ~BytesTriePackageLookup() { | |
| 302 delete builder; | |
| 303 } | |
| 304 | |
| 305 virtual void call(UErrorCode *pErrorCode) { | |
| 306 int32_t count=pkg.getItemCount(); | |
| 307 const char *nameTrieBytes=builder->buildStringPiece(USTRINGTRIE_BUILD_SM
ALL, *pErrorCode).data(); | |
| 308 const char *name=itemNames.data(); | |
| 309 for(int32_t i=0; i<count; ++i) { | |
| 310 if(bytesTrieLookup(name, nameTrieBytes)<0) { | |
| 311 fprintf(stderr, "item not found: %s\n", name); | |
| 312 } | |
| 313 name=strchr(name, 0)+1; | |
| 314 } | |
| 315 } | |
| 316 | |
| 317 protected: | |
| 318 BytesTrieBuilder *builder; | |
| 319 CharString itemNames; | |
| 320 }; | |
| 321 | |
| 322 // Performance test function object. | |
| 323 // Each subclass loads a dictionary text file | |
| 324 // from the -s or --sourcedir path plus -f or --file-name. | |
| 325 // For example, <ICU source dir>/source/data/brkitr/thaidict.txt. | |
| 326 class DictLookup : public UPerfFunction { | |
| 327 public: | |
| 328 DictLookup(const DictionaryTriePerfTest &perfTest) : perf(perfTest) {} | |
| 329 | |
| 330 virtual long getOperationsPerIteration() { | |
| 331 return perf.numTextLines; | |
| 332 } | |
| 333 | |
| 334 protected: | |
| 335 const DictionaryTriePerfTest &perf; | |
| 336 }; | |
| 337 | |
| 338 // Closely imitate CompactTrieDictionary::matches(). | |
| 339 // Note: CompactTrieDictionary::matches() is part of its trie implementation, | |
| 340 // and while it loops over the text, it knows the current state. | |
| 341 // By contrast, this implementation uses UCharsTrie API functions that have to | |
| 342 // check the trie state each time and load/store state in the object. | |
| 343 // (Whether it hasNext() and whether it is in the middle of a linear-match node.
) | |
| 344 static int32_t | |
| 345 ucharsTrieMatches(UCharsTrie &trie, | |
| 346 UText *text, int32_t textLimit, | |
| 347 int32_t *lengths, int &count, int limit ) { | |
| 348 UChar32 c=utext_next32(text); | |
| 349 // Notes: | |
| 350 // a) CompactTrieDictionary::matches() does not check for U_SENTINEL. | |
| 351 // b) It also ignores non-BMP code points by casting to UChar! | |
| 352 if(c<0) { | |
| 353 return 0; | |
| 354 } | |
| 355 // Should be firstForCodePoint() but CompactTrieDictionary | |
| 356 // handles only code units. | |
| 357 UStringTrieResult result=trie.first(c); | |
| 358 int32_t numChars=1; | |
| 359 count=0; | |
| 360 for(;;) { | |
| 361 if(USTRINGTRIE_HAS_VALUE(result)) { | |
| 362 if(count<limit) { | |
| 363 // lengths[count++]=(int32_t)utext_getNativeIndex(text); | |
| 364 lengths[count++]=numChars; // CompactTrieDictionary just counts
chars too. | |
| 365 } | |
| 366 if(result==USTRINGTRIE_FINAL_VALUE) { | |
| 367 break; | |
| 368 } | |
| 369 } else if(result==USTRINGTRIE_NO_MATCH) { | |
| 370 break; | |
| 371 } | |
| 372 if(numChars>=textLimit) { | |
| 373 // Note: Why do we have both a text limit and a UText that knows its
length? | |
| 374 break; | |
| 375 } | |
| 376 UChar32 c=utext_next32(text); | |
| 377 // Notes: | |
| 378 // a) CompactTrieDictionary::matches() does not check for U_SENTINEL. | |
| 379 // b) It also ignores non-BMP code points by casting to UChar! | |
| 380 if(c<0) { | |
| 381 break; | |
| 382 } | |
| 383 ++numChars; | |
| 384 // Should be nextForCodePoint() but CompactTrieDictionary | |
| 385 // handles only code units. | |
| 386 result=trie.next(c); | |
| 387 } | |
| 388 #if 0 | |
| 389 // Note: CompactTrieDictionary::matches() comments say that it leaves the UT
ext | |
| 390 // after the longest prefix match and returns the number of characters | |
| 391 // that were matched. | |
| 392 if(index!=lastMatch) { | |
| 393 utext_setNativeIndex(text, lastMatch); | |
| 394 } | |
| 395 return lastMatch-start; | |
| 396 // However, it does not do either of these, so I am not trying to | |
| 397 // imitate it (or its docs) 100%. | |
| 398 #endif | |
| 399 return numChars; | |
| 400 } | |
| 401 | |
| 402 class UCharsTrieDictLookup : public DictLookup { | |
| 403 public: | |
| 404 UCharsTrieDictLookup(const DictionaryTriePerfTest &perfTest) | |
| 405 : DictLookup(perfTest), trie(NULL) { | |
| 406 IcuToolErrorCode errorCode("UCharsTrieDictLookup()"); | |
| 407 builder=new UCharsTrieBuilder(errorCode); | |
| 408 const ULine *lines=perf.getCachedLines(); | |
| 409 int32_t numLines=perf.getNumLines(); | |
| 410 for(int32_t i=0; i<numLines; ++i) { | |
| 411 // Skip comment lines (start with a character below 'A'). | |
| 412 if(lines[i].name[0]<0x41) { | |
| 413 continue; | |
| 414 } | |
| 415 builder->add(UnicodeString(FALSE, lines[i].name, lines[i].len), 0, e
rrorCode); | |
| 416 } | |
| 417 UnicodeString trieUChars; | |
| 418 int32_t length=builder->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, trie
UChars, errorCode).length(); | |
| 419 printf("size of UCharsTrie: %6ld bytes\n", (long)length*2); | |
| 420 trie=builder->build(USTRINGTRIE_BUILD_SMALL, errorCode); | |
| 421 } | |
| 422 | |
| 423 virtual ~UCharsTrieDictLookup() { | |
| 424 delete builder; | |
| 425 delete trie; | |
| 426 } | |
| 427 | |
| 428 protected: | |
| 429 UCharsTrieBuilder *builder; | |
| 430 UCharsTrie *trie; | |
| 431 }; | |
| 432 | |
| 433 class UCharsTrieDictMatches : public UCharsTrieDictLookup { | |
| 434 public: | |
| 435 UCharsTrieDictMatches(const DictionaryTriePerfTest &perfTest) | |
| 436 : UCharsTrieDictLookup(perfTest) {} | |
| 437 | |
| 438 virtual void call(UErrorCode *pErrorCode) { | |
| 439 UText text=UTEXT_INITIALIZER; | |
| 440 int32_t lengths[20]; | |
| 441 const ULine *lines=perf.getCachedLines(); | |
| 442 int32_t numLines=perf.getNumLines(); | |
| 443 for(int32_t i=0; i<numLines; ++i) { | |
| 444 // Skip comment lines (start with a character below 'A'). | |
| 445 if(lines[i].name[0]<0x41) { | |
| 446 continue; | |
| 447 } | |
| 448 utext_openUChars(&text, lines[i].name, lines[i].len, pErrorCode); | |
| 449 int32_t count=0; | |
| 450 ucharsTrieMatches(*trie, &text, lines[i].len, | |
| 451 lengths, count, UPRV_LENGTHOF(lengths)); | |
| 452 if(count==0 || lengths[count-1]!=lines[i].len) { | |
| 453 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
| 454 } | |
| 455 } | |
| 456 } | |
| 457 }; | |
| 458 | |
| 459 class UCharsTrieDictContains : public UCharsTrieDictLookup { | |
| 460 public: | |
| 461 UCharsTrieDictContains(const DictionaryTriePerfTest &perfTest) | |
| 462 : UCharsTrieDictLookup(perfTest) {} | |
| 463 | |
| 464 virtual void call(UErrorCode * /*pErrorCode*/) { | |
| 465 const ULine *lines=perf.getCachedLines(); | |
| 466 int32_t numLines=perf.getNumLines(); | |
| 467 for(int32_t i=0; i<numLines; ++i) { | |
| 468 // Skip comment lines (which start with a character below 'A'). | |
| 469 if(lines[i].name[0]<0x41) { | |
| 470 continue; | |
| 471 } | |
| 472 if(!USTRINGTRIE_HAS_VALUE(trie->reset().next(lines[i].name, lines[i]
.len))) { | |
| 473 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
| 474 } | |
| 475 } | |
| 476 } | |
| 477 }; | |
| 478 | |
| 479 static inline int32_t thaiCharToByte(UChar32 c) { | |
| 480 if(0xe00<=c && c<=0xefe) { | |
| 481 return c&0xff; | |
| 482 } else if(c==0x2e) { | |
| 483 return 0xff; | |
| 484 } else { | |
| 485 return -1; | |
| 486 } | |
| 487 } | |
| 488 | |
| 489 static UBool thaiWordToBytes(const UChar *s, int32_t length, | |
| 490 CharString &str, UErrorCode &errorCode) { | |
| 491 for(int32_t i=0; i<length; ++i) { | |
| 492 UChar c=s[i]; | |
| 493 int32_t b=thaiCharToByte(c); | |
| 494 if(b>=0) { | |
| 495 str.append((char)b, errorCode); | |
| 496 } else { | |
| 497 fprintf(stderr, "thaiWordToBytes(): unable to encode U+%04X as a byt
e\n", c); | |
| 498 return FALSE; | |
| 499 } | |
| 500 } | |
| 501 return TRUE; | |
| 502 } | |
| 503 | |
| 504 class BytesTrieDictLookup : public DictLookup { | |
| 505 public: | |
| 506 BytesTrieDictLookup(const DictionaryTriePerfTest &perfTest) | |
| 507 : DictLookup(perfTest), trie(NULL), noDict(FALSE) { | |
| 508 IcuToolErrorCode errorCode("BytesTrieDictLookup()"); | |
| 509 builder=new BytesTrieBuilder(errorCode); | |
| 510 CharString str; | |
| 511 const ULine *lines=perf.getCachedLines(); | |
| 512 int32_t numLines=perf.getNumLines(); | |
| 513 for(int32_t i=0; i<numLines; ++i) { | |
| 514 // Skip comment lines (start with a character below 'A'). | |
| 515 if(lines[i].name[0]<0x41) { | |
| 516 continue; | |
| 517 } | |
| 518 if(!thaiWordToBytes(lines[i].name, lines[i].len, str.clear(), errorC
ode)) { | |
| 519 fprintf(stderr, "thaiWordToBytes(): failed for word %ld (0-based
)\n", (long)i); | |
| 520 noDict=TRUE; | |
| 521 break; | |
| 522 } | |
| 523 builder->add(str.toStringPiece(), 0, errorCode); | |
| 524 } | |
| 525 if(!noDict) { | |
| 526 int32_t length=builder->buildStringPiece(USTRINGTRIE_BUILD_SMALL, er
rorCode).length(); | |
| 527 printf("size of BytesTrie: %6ld bytes\n", (long)length); | |
| 528 trie=builder->build(USTRINGTRIE_BUILD_SMALL, errorCode); | |
| 529 } | |
| 530 } | |
| 531 | |
| 532 virtual ~BytesTrieDictLookup() { | |
| 533 delete builder; | |
| 534 delete trie; | |
| 535 } | |
| 536 | |
| 537 protected: | |
| 538 BytesTrieBuilder *builder; | |
| 539 BytesTrie *trie; | |
| 540 UBool noDict; | |
| 541 }; | |
| 542 | |
| 543 static int32_t | |
| 544 bytesTrieMatches(BytesTrie &trie, | |
| 545 UText *text, int32_t textLimit, | |
| 546 int32_t *lengths, int &count, int limit ) { | |
| 547 UChar32 c=utext_next32(text); | |
| 548 if(c<0) { | |
| 549 return 0; | |
| 550 } | |
| 551 UStringTrieResult result=trie.first(thaiCharToByte(c)); | |
| 552 int32_t numChars=1; | |
| 553 count=0; | |
| 554 for(;;) { | |
| 555 if(USTRINGTRIE_HAS_VALUE(result)) { | |
| 556 if(count<limit) { | |
| 557 // lengths[count++]=(int32_t)utext_getNativeIndex(text); | |
| 558 lengths[count++]=numChars; // CompactTrieDictionary just counts
chars too. | |
| 559 } | |
| 560 if(result==USTRINGTRIE_FINAL_VALUE) { | |
| 561 break; | |
| 562 } | |
| 563 } else if(result==USTRINGTRIE_NO_MATCH) { | |
| 564 break; | |
| 565 } | |
| 566 if(numChars>=textLimit) { | |
| 567 break; | |
| 568 } | |
| 569 UChar32 c=utext_next32(text); | |
| 570 if(c<0) { | |
| 571 break; | |
| 572 } | |
| 573 ++numChars; | |
| 574 result=trie.next(thaiCharToByte(c)); | |
| 575 } | |
| 576 return numChars; | |
| 577 } | |
| 578 | |
| 579 class BytesTrieDictMatches : public BytesTrieDictLookup { | |
| 580 public: | |
| 581 BytesTrieDictMatches(const DictionaryTriePerfTest &perfTest) | |
| 582 : BytesTrieDictLookup(perfTest) {} | |
| 583 | |
| 584 virtual void call(UErrorCode *pErrorCode) { | |
| 585 if(noDict) { | |
| 586 return; | |
| 587 } | |
| 588 UText text=UTEXT_INITIALIZER; | |
| 589 int32_t lengths[20]; | |
| 590 const ULine *lines=perf.getCachedLines(); | |
| 591 int32_t numLines=perf.getNumLines(); | |
| 592 for(int32_t i=0; i<numLines; ++i) { | |
| 593 // Skip comment lines (start with a character below 'A'). | |
| 594 if(lines[i].name[0]<0x41) { | |
| 595 continue; | |
| 596 } | |
| 597 utext_openUChars(&text, lines[i].name, lines[i].len, pErrorCode); | |
| 598 int32_t count=0; | |
| 599 bytesTrieMatches(*trie, &text, lines[i].len, | |
| 600 lengths, count, UPRV_LENGTHOF(lengths)); | |
| 601 if(count==0 || lengths[count-1]!=lines[i].len) { | |
| 602 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
| 603 } | |
| 604 } | |
| 605 } | |
| 606 }; | |
| 607 | |
| 608 class BytesTrieDictContains : public BytesTrieDictLookup { | |
| 609 public: | |
| 610 BytesTrieDictContains(const DictionaryTriePerfTest &perfTest) | |
| 611 : BytesTrieDictLookup(perfTest) {} | |
| 612 | |
| 613 virtual void call(UErrorCode * /*pErrorCode*/) { | |
| 614 if(noDict) { | |
| 615 return; | |
| 616 } | |
| 617 const ULine *lines=perf.getCachedLines(); | |
| 618 int32_t numLines=perf.getNumLines(); | |
| 619 for(int32_t i=0; i<numLines; ++i) { | |
| 620 const UChar *line=lines[i].name; | |
| 621 // Skip comment lines (start with a character below 'A'). | |
| 622 if(line[0]<0x41) { | |
| 623 continue; | |
| 624 } | |
| 625 UStringTrieResult result=trie->first(thaiCharToByte(line[0])); | |
| 626 int32_t lineLength=lines[i].len; | |
| 627 for(int32_t j=1; j<lineLength; ++j) { | |
| 628 if(!USTRINGTRIE_HAS_NEXT(result)) { | |
| 629 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
| 630 break; | |
| 631 } | |
| 632 result=trie->next(thaiCharToByte(line[j])); | |
| 633 } | |
| 634 if(!USTRINGTRIE_HAS_VALUE(result)) { | |
| 635 fprintf(stderr, "word %ld (0-based) not found\n", (long)i); | |
| 636 } | |
| 637 } | |
| 638 } | |
| 639 }; | |
| 640 | |
| 641 UPerfFunction *DictionaryTriePerfTest::runIndexedTest(int32_t index, UBool exec, | |
| 642 const char *&name, char *
/*par*/) { | |
| 643 if(hasFile()) { | |
| 644 switch(index) { | |
| 645 case 0: | |
| 646 name="ucharstriematches"; | |
| 647 if(exec) { | |
| 648 return new UCharsTrieDictMatches(*this); | |
| 649 } | |
| 650 break; | |
| 651 case 1: | |
| 652 name="ucharstriecontains"; | |
| 653 if(exec) { | |
| 654 return new UCharsTrieDictContains(*this); | |
| 655 } | |
| 656 break; | |
| 657 case 2: | |
| 658 name="bytestriematches"; | |
| 659 if(exec) { | |
| 660 return new BytesTrieDictMatches(*this); | |
| 661 } | |
| 662 break; | |
| 663 case 3: | |
| 664 name="bytestriecontains"; | |
| 665 if(exec) { | |
| 666 return new BytesTrieDictContains(*this); | |
| 667 } | |
| 668 break; | |
| 669 default: | |
| 670 name=""; | |
| 671 break; | |
| 672 } | |
| 673 } else { | |
| 674 if(index==0 && exec) { | |
| 675 puts("Running BytesTrie perf tests on the .dat package file from the
--sourcedir.\n" | |
| 676 "For UCharsTrie perf tests on a dictionary text file, specify t
he -f or --file-name.\n"); | |
| 677 } | |
| 678 switch(index) { | |
| 679 case 0: | |
| 680 name="simplebinarysearch"; | |
| 681 if(exec) { | |
| 682 return new BinarySearchPackageLookup(*this); | |
| 683 } | |
| 684 break; | |
| 685 case 1: | |
| 686 name="prefixbinarysearch"; | |
| 687 if(exec) { | |
| 688 return new PrefixBinarySearchPackageLookup(*this); | |
| 689 } | |
| 690 break; | |
| 691 case 2: | |
| 692 name="bytestrie"; | |
| 693 if(exec) { | |
| 694 return new BytesTriePackageLookup(*this); | |
| 695 } | |
| 696 break; | |
| 697 default: | |
| 698 name=""; | |
| 699 break; | |
| 700 } | |
| 701 } | |
| 702 return NULL; | |
| 703 } | |
| 704 | |
| 705 int main(int argc, const char *argv[]) { | |
| 706 IcuToolErrorCode errorCode("dicttrieperf main()"); | |
| 707 DictionaryTriePerfTest test(argc, argv, errorCode); | |
| 708 if(errorCode.isFailure()) { | |
| 709 fprintf(stderr, "DictionaryTriePerfTest() failed: %s\n", errorCode.error
Name()); | |
| 710 test.usage(); | |
| 711 return errorCode.reset(); | |
| 712 } | |
| 713 if(!test.run()) { | |
| 714 fprintf(stderr, "FAILED: Tests could not be run, please check the argume
nts.\n"); | |
| 715 return -1; | |
| 716 } | |
| 717 return 0; | |
| 718 } | |
| OLD | NEW |