| OLD | NEW |
|---|
| (Empty) |
| 1 /* | |
| 2 ******************************************************************************* | |
| 3 * Copyright (C) 2014, International Business Machines Corporation and | |
| 4 * others. All Rights Reserved. | |
| 5 ******************************************************************************* | |
| 6 */ | |
| 7 | |
| 8 #include "unicode/utypes.h" | |
| 9 #if !UCONFIG_NO_BREAK_ITERATION && !UCONFIG_NO_FILTERED_BREAK_ITERATION | |
| 10 | |
| 11 #include "cmemory.h" | |
| 12 | |
| 13 #include "unicode/filteredbrk.h" | |
| 14 #include "unicode/ucharstriebuilder.h" | |
| 15 #include "unicode/ures.h" | |
| 16 | |
| 17 #include "uresimp.h" // ures_getByKeyWithFallback | |
| 18 #include "ubrkimpl.h" // U_ICUDATA_BRKITR | |
| 19 #include "uvector.h" | |
| 20 #include "cmemory.h" | |
| 21 | |
| 22 U_NAMESPACE_BEGIN | |
| 23 | |
| 24 #ifndef FB_DEBUG | |
| 25 #define FB_DEBUG 0 | |
| 26 #endif | |
| 27 | |
| 28 #if FB_DEBUG | |
| 29 #include <stdio.h> | |
| 30 static void _fb_trace(const char *m, const UnicodeString *s, UBool b, int32_t d,
const char *f, int l) { | |
| 31 char buf[2048]; | |
| 32 if(s) { | |
| 33 s->extract(0,s->length(),buf,2048); | |
| 34 } else { | |
| 35 strcpy(buf,"NULL"); | |
| 36 } | |
| 37 fprintf(stderr,"%s:%d: %s. s='%s'(%p), b=%c, d=%d\n", | |
| 38 f, l, m, buf, (const void*)s, b?'T':'F',(int)d); | |
| 39 } | |
| 40 | |
| 41 #define FB_TRACE(m,s,b,d) _fb_trace(m,s,b,d,__FILE__,__LINE__) | |
| 42 #else | |
| 43 #define FB_TRACE(m,s,b,d) | |
| 44 #endif | |
| 45 | |
| 46 static int8_t U_CALLCONV compareUnicodeString(UElement t1, UElement t2) { | |
| 47 const UnicodeString &a = *(const UnicodeString*)t1.pointer; | |
| 48 const UnicodeString &b = *(const UnicodeString*)t2.pointer; | |
| 49 return a.compare(b); | |
| 50 } | |
| 51 | |
| 52 /** | |
| 53 * A UVector which implements a set of strings. | |
| 54 */ | |
| 55 class U_I18N_API UStringSet : public UVector { | |
| 56 public: | |
| 57 UStringSet(UErrorCode &status) : UVector(uprv_deleteUObject, | |
| 58 uhash_compareUnicodeString, | |
| 59 1, | |
| 60 status) {} | |
| 61 virtual ~UStringSet(); | |
| 62 /** | |
| 63 * Is this UnicodeSet contained? | |
| 64 */ | |
| 65 inline UBool contains(const UnicodeString& s) { | |
| 66 return contains((void*) &s); | |
| 67 } | |
| 68 using UVector::contains; | |
| 69 /** | |
| 70 * Return the ith UnicodeString alias | |
| 71 */ | |
| 72 inline const UnicodeString* getStringAt(int32_t i) const { | |
| 73 return (const UnicodeString*)elementAt(i); | |
| 74 } | |
| 75 /** | |
| 76 * Adopt the UnicodeString if not already contained. | |
| 77 * Caller no longer owns the pointer in any case. | |
| 78 * @return true if adopted successfully, false otherwise (error, or else dupli
cate) | |
| 79 */ | |
| 80 inline UBool adopt(UnicodeString *str, UErrorCode &status) { | |
| 81 if(U_FAILURE(status) || contains(*str)) { | |
| 82 delete str; | |
| 83 return false; | |
| 84 } else { | |
| 85 sortedInsert(str, compareUnicodeString, status); | |
| 86 if(U_FAILURE(status)) { | |
| 87 delete str; | |
| 88 return false; | |
| 89 } | |
| 90 return true; | |
| 91 } | |
| 92 } | |
| 93 /** | |
| 94 * Add by value. | |
| 95 * @return true if successfully adopted. | |
| 96 */ | |
| 97 inline UBool add(const UnicodeString& str, UErrorCode &status) { | |
| 98 if(U_FAILURE(status)) return false; | |
| 99 UnicodeString *t = new UnicodeString(str); | |
| 100 if(t==NULL) { | |
| 101 status = U_MEMORY_ALLOCATION_ERROR; return false; | |
| 102 } | |
| 103 return adopt(t, status); | |
| 104 } | |
| 105 /** | |
| 106 * Remove this string. | |
| 107 * @return true if successfully removed, false otherwise (error, or else it wa
sn't there) | |
| 108 */ | |
| 109 inline UBool remove(const UnicodeString &s, UErrorCode &status) { | |
| 110 if(U_FAILURE(status)) return false; | |
| 111 return removeElement((void*) &s); | |
| 112 } | |
| 113 }; | |
| 114 | |
| 115 /** | |
| 116 * Virtual, won't be inlined | |
| 117 */ | |
| 118 UStringSet::~UStringSet() {} | |
| 119 | |
| 120 | |
| 121 static const int32_t kPARTIAL = (1<<0); //< partial - need to run through forwar
d trie | |
| 122 static const int32_t kMATCH = (1<<1); //< exact match - skip this one. | |
| 123 static const int32_t kSuppressInReverse = (1<<0); | |
| 124 static const int32_t kAddToForward = (1<<1); | |
| 125 static const UChar kFULLSTOP = 0x002E; // '.' | |
| 126 | |
| 127 class SimpleFilteredSentenceBreakIterator : public BreakIterator { | |
| 128 public: | |
| 129 SimpleFilteredSentenceBreakIterator(BreakIterator *adopt, UCharsTrie *forwards
, UCharsTrie *backwards, UErrorCode &status); | |
| 130 SimpleFilteredSentenceBreakIterator(const SimpleFilteredSentenceBreakIterator&
other); | |
| 131 virtual ~SimpleFilteredSentenceBreakIterator(); | |
| 132 private: | |
| 133 LocalPointer<BreakIterator> fDelegate; | |
| 134 LocalUTextPointer fText; | |
| 135 LocalPointer<UCharsTrie> fBackwardsTrie; // i.e. ".srM" for Mrs. | |
| 136 LocalPointer<UCharsTrie> fForwardsPartialTrie; // Has ".a" for "a.M." | |
| 137 | |
| 138 /* -- subclass interface -- */ | |
| 139 public: | |
| 140 /* -- cloning and other subclass stuff -- */ | |
| 141 virtual BreakIterator * createBufferClone(void * /*stackBuffer*/, | |
| 142 int32_t &/*BufferSize*/, | |
| 143 UErrorCode &status) { | |
| 144 // for now - always deep clone | |
| 145 status = U_SAFECLONE_ALLOCATED_WARNING; | |
| 146 return clone(); | |
| 147 } | |
| 148 virtual BreakIterator* clone(void) const { return new SimpleFilteredSentenceBr
eakIterator(*this); } | |
| 149 virtual UClassID getDynamicClassID(void) const { return NULL; } | |
| 150 virtual UBool operator==(const BreakIterator& o) const { if(this==&o) return t
rue; return false; } | |
| 151 | |
| 152 /* -- text modifying -- */ | |
| 153 virtual void setText(UText *text, UErrorCode &status) { fDelegate->setText(tex
t,status); } | |
| 154 virtual BreakIterator &refreshInputText(UText *input, UErrorCode &status) { fD
elegate->refreshInputText(input,status); return *this; } | |
| 155 virtual void adoptText(CharacterIterator* it) { fDelegate->adoptText(it); } | |
| 156 virtual void setText(const UnicodeString &text) { fDelegate->setText(text); } | |
| 157 | |
| 158 /* -- other functions that are just delegated -- */ | |
| 159 virtual UText *getUText(UText *fillIn, UErrorCode &status) const { return fDel
egate->getUText(fillIn,status); } | |
| 160 virtual CharacterIterator& getText(void) const { return fDelegate->getText();
} | |
| 161 | |
| 162 /* -- ITERATION -- */ | |
| 163 virtual int32_t first(void) { return fDelegate->first(); } | |
| 164 virtual int32_t preceding(int32_t /*offset*/) { /* TODO: not implemented */ re
turn UBRK_DONE; } | |
| 165 virtual int32_t previous(void) { /* TODO: not implemented */ return UBRK_DONE;
} | |
| 166 virtual UBool isBoundary(int32_t offset) { return fDelegate->isBoundary(offset
); } | |
| 167 virtual int32_t current(void) const { return fDelegate->current(); } | |
| 168 | |
| 169 virtual int32_t next(void); | |
| 170 | |
| 171 virtual int32_t next(int32_t /*n*/) { /* TODO: not implemented */ return UBRK_
DONE; } | |
| 172 virtual int32_t following(int32_t /*offset*/) { /* TODO: not implemented */ re
turn UBRK_DONE; } | |
| 173 virtual int32_t last(void) { return fDelegate->last(); } | |
| 174 | |
| 175 }; | |
| 176 | |
| 177 SimpleFilteredSentenceBreakIterator::SimpleFilteredSentenceBreakIterator(const S
impleFilteredSentenceBreakIterator& other) | |
| 178 : BreakIterator(other), fDelegate(other.fDelegate->clone()) | |
| 179 { | |
| 180 /* | |
| 181 TODO: not able to clone Tries. Should be a refcounted hidden master instead. | |
| 182 if(other.fBackwardsTrie.isValid()) { | |
| 183 fBackwardsTrie.adoptInstead(other.fBackwardsTrie->clone()); | |
| 184 } | |
| 185 if(other.fForwardsPartialTrie.isValid()) { | |
| 186 fForwardsPartialTrie.adoptInstead(other.fForwardsPartialTrie->clone()); | |
| 187 } | |
| 188 */ | |
| 189 } | |
| 190 | |
| 191 | |
| 192 SimpleFilteredSentenceBreakIterator::SimpleFilteredSentenceBreakIterator(BreakIt
erator *adopt, UCharsTrie *forwards, UCharsTrie *backwards, UErrorCode &status)
: | |
| 193 BreakIterator(adopt->getLocale(ULOC_VALID_LOCALE,status),adopt->getLocale(ULOC
_ACTUAL_LOCALE,status)), | |
| 194 fDelegate(adopt), | |
| 195 fBackwardsTrie(backwards), | |
| 196 fForwardsPartialTrie(forwards) | |
| 197 { | |
| 198 // all set.. | |
| 199 } | |
| 200 | |
| 201 SimpleFilteredSentenceBreakIterator::~SimpleFilteredSentenceBreakIterator() {} | |
| 202 | |
| 203 int32_t SimpleFilteredSentenceBreakIterator::next() { | |
| 204 int32_t n = fDelegate->next(); | |
| 205 if(n == UBRK_DONE || // at end or | |
| 206 fBackwardsTrie.isNull()) { // .. no backwards table loaded == no exceptions | |
| 207 return n; | |
| 208 } | |
| 209 // OK, do we need to break here? | |
| 210 UErrorCode status = U_ZERO_ERROR; | |
| 211 // refresh text | |
| 212 fText.adoptInstead(fDelegate->getUText(fText.orphan(), status)); | |
| 213 //if(debug2) u_printf("str, native len=%d\n", utext_nativeLength(fText.getAlia
s())); | |
| 214 do { // outer loop runs once per underlying break (from fDelegate). | |
| 215 // loops while 'n' points to an exception. | |
| 216 utext_setNativeIndex(fText.getAlias(), n); // from n.. | |
| 217 fBackwardsTrie->reset(); | |
| 218 UChar32 uch; | |
| 219 //if(debug2) u_printf(" n@ %d\n", n); | |
| 220 // Assume a space is following the '.' (so we handle the case: "Mr. /Brown
") | |
| 221 if((uch=utext_previous32(fText.getAlias()))==(UChar32)0x0020) { // TODO: sk
ip a class of chars here?? | |
| 222 // TODO only do this the 1st time? | |
| 223 //if(debug2) u_printf("skipping prev: |%C| \n", (UChar)uch); | |
| 224 } else { | |
| 225 //if(debug2) u_printf("not skipping prev: |%C| \n", (UChar)uch); | |
| 226 uch = utext_next32(fText.getAlias()); | |
| 227 //if(debug2) u_printf(" -> : |%C| \n", (UChar)uch); | |
| 228 } | |
| 229 UStringTrieResult r = USTRINGTRIE_INTERMEDIATE_VALUE; | |
| 230 | |
| 231 int32_t bestPosn = -1; | |
| 232 int32_t bestValue = -1; | |
| 233 | |
| 234 while((uch=utext_previous32(fText.getAlias()))!=U_SENTINEL && // more to
consume backwards and.. | |
| 235 USTRINGTRIE_HAS_NEXT(r=fBackwardsTrie->nextForCodePoint(uch))) {// mor
e in the trie | |
| 236 if(USTRINGTRIE_HAS_VALUE(r)) { // remember the best match so far | |
| 237 bestPosn = utext_getNativeIndex(fText.getAlias()); | |
| 238 bestValue = fBackwardsTrie->getValue(); | |
| 239 } | |
| 240 //if(debug2) u_printf("rev< /%C/ cont?%d @%d\n", (UChar)uch, r, utext_getN
ativeIndex(fText.getAlias())); | |
| 241 } | |
| 242 | |
| 243 if(USTRINGTRIE_MATCHES(r)) { // exact match? | |
| 244 //if(debug2) u_printf("rev<?/%C/?end of seq.. r=%d, bestPosn=%d, bestValue
=%d\n", (UChar)uch, r, bestPosn, bestValue); | |
| 245 bestValue = fBackwardsTrie->getValue(); | |
| 246 bestPosn = utext_getNativeIndex(fText.getAlias()); | |
| 247 //if(debug2) u_printf("rev<+/%C/+end of seq.. r=%d, bestPosn=%d, bestValue
=%d\n", (UChar)uch, r, bestPosn, bestValue); | |
| 248 } | |
| 249 | |
| 250 if(bestPosn>=0) { | |
| 251 //if(debug2) u_printf("rev< /%C/ end of seq.. r=%d, bestPosn=%d, bestValue
=%d\n", (UChar)uch, r, bestPosn, bestValue); | |
| 252 | |
| 253 //if(USTRINGTRIE_MATCHES(r)) { // matched - so, now what? | |
| 254 //int32_t bestValue = fBackwardsTrie->getValue(); | |
| 255 ////if(debug2) u_printf("rev< /%C/ matched, skip..%d bestValue=%d\n", (UC
har)uch, r, bestValue); | |
| 256 | |
| 257 if(bestValue == kMATCH) { // exact match! | |
| 258 //if(debug2) u_printf(" exact backward match\n"); | |
| 259 n = fDelegate->next(); // skip this one. Find the next lowerlevel break. | |
| 260 if(n==UBRK_DONE) return n; | |
| 261 continue; // See if the next is another exception. | |
| 262 } else if(bestValue == kPARTIAL | |
| 263 && fForwardsPartialTrie.isValid()) { // make sure there's a forw
ard trie | |
| 264 //if(debug2) u_printf(" partial backward match\n"); | |
| 265 // We matched the "Ph." in "Ph.D." - now we need to run everything throu
gh the forwards trie | |
| 266 // to see if it matches something going forward. | |
| 267 fForwardsPartialTrie->reset(); | |
| 268 UStringTrieResult rfwd = USTRINGTRIE_INTERMEDIATE_VALUE; | |
| 269 utext_setNativeIndex(fText.getAlias(), bestPosn); // hope that's close .
. | |
| 270 //if(debug2) u_printf("Retrying at %d\n", bestPosn); | |
| 271 while((uch=utext_next32(fText.getAlias()))!=U_SENTINEL && | |
| 272 USTRINGTRIE_HAS_NEXT(rfwd=fForwardsPartialTrie->nextForCodePoint(u
ch))) { | |
| 273 //if(debug2) u_printf("fwd> /%C/ cont?%d @%d\n", (UChar)uch, rfwd, ute
xt_getNativeIndex(fText.getAlias())); | |
| 274 } | |
| 275 if(USTRINGTRIE_MATCHES(rfwd)) { | |
| 276 //if(debug2) u_printf("fwd> /%C/ == forward match!\n", (UChar)uch); | |
| 277 // only full matches here, nothing to check | |
| 278 // skip the next: | |
| 279 n = fDelegate->next(); | |
| 280 if(n==UBRK_DONE) return n; | |
| 281 continue; | |
| 282 } else { | |
| 283 //if(debug2) u_printf("fwd> /%C/ no match.\n", (UChar)uch); | |
| 284 // no match (no exception) -return the 'underlying' break | |
| 285 return n; | |
| 286 } | |
| 287 } else { | |
| 288 return n; // internal error and/or no forwards trie | |
| 289 } | |
| 290 } else { | |
| 291 //if(debug2) u_printf("rev< /%C/ .. no match..%d\n", (UChar)uch, r); // n
o best match | |
| 292 return n; // No match - so exit. Not an exception. | |
| 293 } | |
| 294 } while(n != UBRK_DONE); | |
| 295 return n; | |
| 296 } | |
| 297 | |
| 298 /** | |
| 299 * Concrete implementation of builder class. | |
| 300 */ | |
| 301 class U_I18N_API SimpleFilteredBreakIteratorBuilder : public FilteredBreakIterat
orBuilder { | |
| 302 public: | |
| 303 virtual ~SimpleFilteredBreakIteratorBuilder(); | |
| 304 SimpleFilteredBreakIteratorBuilder(const Locale &fromLocale, UErrorCode &statu
s); | |
| 305 SimpleFilteredBreakIteratorBuilder(UErrorCode &status); | |
| 306 virtual UBool suppressBreakAfter(const UnicodeString& exception, UErrorCode& s
tatus); | |
| 307 virtual UBool unsuppressBreakAfter(const UnicodeString& exception, UErrorCode&
status); | |
| 308 virtual BreakIterator *build(BreakIterator* adoptBreakIterator, UErrorCode& st
atus); | |
| 309 private: | |
| 310 UStringSet fSet; | |
| 311 }; | |
| 312 | |
| 313 SimpleFilteredBreakIteratorBuilder::~SimpleFilteredBreakIteratorBuilder() | |
| 314 { | |
| 315 } | |
| 316 | |
| 317 SimpleFilteredBreakIteratorBuilder::SimpleFilteredBreakIteratorBuilder(UErrorCod
e &status) | |
| 318 : fSet(status) | |
| 319 { | |
| 320 } | |
| 321 | |
| 322 SimpleFilteredBreakIteratorBuilder::SimpleFilteredBreakIteratorBuilder(const Loc
ale &fromLocale, UErrorCode &status) | |
| 323 : fSet(status) | |
| 324 { | |
| 325 if(U_SUCCESS(status)) { | |
| 326 LocalUResourceBundlePointer b(ures_open(U_ICUDATA_BRKITR, fromLocale.getBase
Name(), &status)); | |
| 327 LocalUResourceBundlePointer exceptions(ures_getByKeyWithFallback(b.getAlias(
), "exceptions", NULL, &status)); | |
| 328 LocalUResourceBundlePointer breaks(ures_getByKeyWithFallback(exceptions.getA
lias(), "SentenceBreak", NULL, &status)); | |
| 329 if(U_FAILURE(status)) return; // leaves the builder empty, if you try to use
it. | |
| 330 | |
| 331 LocalUResourceBundlePointer strs; | |
| 332 UErrorCode subStatus = status; | |
| 333 do { | |
| 334 strs.adoptInstead(ures_getNextResource(breaks.getAlias(), strs.orphan(), &
subStatus)); | |
| 335 if(strs.isValid() && U_SUCCESS(subStatus)) { | |
| 336 UnicodeString str(ures_getUnicodeString(strs.getAlias(), &status)); | |
| 337 suppressBreakAfter(str, status); // load the string | |
| 338 } | |
| 339 } while (strs.isValid() && U_SUCCESS(subStatus)); | |
| 340 if(U_FAILURE(subStatus)&&subStatus!=U_INDEX_OUTOFBOUNDS_ERROR&&U_SUCCESS(sta
tus)) { | |
| 341 status = subStatus; | |
| 342 } | |
| 343 } | |
| 344 } | |
| 345 | |
| 346 UBool | |
| 347 SimpleFilteredBreakIteratorBuilder::suppressBreakAfter(const UnicodeString& exce
ption, UErrorCode& status) | |
| 348 { | |
| 349 UBool r = fSet.add(exception, status); | |
| 350 FB_TRACE("suppressBreakAfter",&exception,r,0); | |
| 351 return r; | |
| 352 } | |
| 353 | |
| 354 UBool | |
| 355 SimpleFilteredBreakIteratorBuilder::unsuppressBreakAfter(const UnicodeString& ex
ception, UErrorCode& status) | |
| 356 { | |
| 357 UBool r = fSet.remove(exception, status); | |
| 358 FB_TRACE("unsuppressBreakAfter",&exception,r,0); | |
| 359 return r; | |
| 360 } | |
| 361 | |
| 362 /** | |
| 363 * Jitterbug 2974: MSVC has a bug whereby new X[0] behaves badly. | |
| 364 * Work around this. | |
| 365 * | |
| 366 * Note: "new UnicodeString[subCount]" ends up calling global operator new | |
| 367 * on MSVC2012 for some reason. | |
| 368 */ | |
| 369 static inline UnicodeString* newUnicodeStringArray(size_t count) { | |
| 370 return new UnicodeString[count ? count : 1]; | |
| 371 } | |
| 372 | |
| 373 BreakIterator * | |
| 374 SimpleFilteredBreakIteratorBuilder::build(BreakIterator* adoptBreakIterator, UEr
rorCode& status) { | |
| 375 LocalPointer<BreakIterator> adopt(adoptBreakIterator); | |
| 376 | |
| 377 if(U_FAILURE(status)) { | |
| 378 return NULL; | |
| 379 } | |
| 380 | |
| 381 LocalPointer<UCharsTrieBuilder> builder(new UCharsTrieBuilder(status)); | |
| 382 LocalPointer<UCharsTrieBuilder> builder2(new UCharsTrieBuilder(status)); | |
| 383 | |
| 384 int32_t revCount = 0; | |
| 385 int32_t fwdCount = 0; | |
| 386 | |
| 387 int32_t subCount = fSet.size(); | |
| 388 | |
| 389 UnicodeString *ustrs_ptr = newUnicodeStringArray(subCount); | |
| 390 | |
| 391 LocalArray<UnicodeString> ustrs(ustrs_ptr); | |
| 392 | |
| 393 LocalMemory<int> partials; | |
| 394 partials.allocateInsteadAndReset(subCount); | |
| 395 | |
| 396 LocalPointer<UCharsTrie> backwardsTrie; // i.e. ".srM" for Mrs. | |
| 397 LocalPointer<UCharsTrie> forwardsPartialTrie; // Has ".a" for "a.M." | |
| 398 | |
| 399 int n=0; | |
| 400 for ( int32_t i = 0; | |
| 401 i<fSet.size(); | |
| 402 i++) { | |
| 403 const UnicodeString *abbr = fSet.getStringAt(i); | |
| 404 if(abbr) { | |
| 405 FB_TRACE("build",abbr,TRUE,i); | |
| 406 ustrs[n] = *abbr; // copy by value | |
| 407 FB_TRACE("ustrs[n]",&ustrs[n],TRUE,i); | |
| 408 } else { | |
| 409 FB_TRACE("build",abbr,FALSE,i); | |
| 410 status = U_MEMORY_ALLOCATION_ERROR; | |
| 411 return NULL; | |
| 412 } | |
| 413 partials[n] = 0; // default: not partial | |
| 414 n++; | |
| 415 } | |
| 416 // first pass - find partials. | |
| 417 for(int i=0;i<subCount;i++) { | |
| 418 int nn = ustrs[i].indexOf(kFULLSTOP); // TODO: non-'.' abbreviations | |
| 419 if(nn>-1 && (nn+1)!=ustrs[i].length()) { | |
| 420 FB_TRACE("partial",&ustrs[i],FALSE,i); | |
| 421 // is partial. | |
| 422 // is it unique? | |
| 423 int sameAs = -1; | |
| 424 for(int j=0;j<subCount;j++) { | |
| 425 if(j==i) continue; | |
| 426 if(ustrs[i].compare(0,nn+1,ustrs[j],0,nn+1)==0) { | |
| 427 FB_TRACE("prefix",&ustrs[j],FALSE,nn+1); | |
| 428 //UBool otherIsPartial = ((nn+1)!=ustrs[j].length()); // true if ustr
s[j] doesn't end at nn | |
| 429 if(partials[j]==0) { // hasn't been processed yet | |
| 430 partials[j] = kSuppressInReverse | kAddToForward; | |
| 431 FB_TRACE("suppressing",&ustrs[j],FALSE,j); | |
| 432 } else if(partials[j] & kSuppressInReverse) { | |
| 433 sameAs = j; // the other entry is already in the reverse table. | |
| 434 } | |
| 435 } | |
| 436 } | |
| 437 FB_TRACE("for partial same-",&ustrs[i],FALSE,sameAs); | |
| 438 FB_TRACE(" == partial #",&ustrs[i],FALSE,partials[i]); | |
| 439 UnicodeString prefix(ustrs[i], 0, nn+1); | |
| 440 if(sameAs == -1 && partials[i] == 0) { | |
| 441 // first one - add the prefix to the reverse table. | |
| 442 prefix.reverse(); | |
| 443 builder->add(prefix, kPARTIAL, status); | |
| 444 revCount++; | |
| 445 FB_TRACE("Added partial",&prefix,FALSE, i); | |
| 446 FB_TRACE(u_errorName(status),&ustrs[i],FALSE,i); | |
| 447 partials[i] = kSuppressInReverse | kAddToForward; | |
| 448 } else { | |
| 449 FB_TRACE("NOT adding partial",&prefix,FALSE, i); | |
| 450 FB_TRACE(u_errorName(status),&ustrs[i],FALSE,i); | |
| 451 } | |
| 452 } | |
| 453 } | |
| 454 for(int i=0;i<subCount;i++) { | |
| 455 if(partials[i]==0) { | |
| 456 ustrs[i].reverse(); | |
| 457 builder->add(ustrs[i], kMATCH, status); | |
| 458 revCount++; | |
| 459 FB_TRACE(u_errorName(status), &ustrs[i], FALSE, i); | |
| 460 } else { | |
| 461 FB_TRACE("Adding fwd",&ustrs[i], FALSE, i); | |
| 462 | |
| 463 // an optimization would be to only add the portion after the '.' | |
| 464 // for example, for "Ph.D." we store ".hP" in the reverse table. We could
just store "D." in the forward, | |
| 465 // instead of "Ph.D." since we already know the "Ph." part is a match. | |
| 466 // would need the trie to be able to hold 0-length strings, though. | |
| 467 builder2->add(ustrs[i], kMATCH, status); // forward | |
| 468 fwdCount++; | |
| 469 //ustrs[i].reverse(); | |
| 470 ////if(debug2) u_printf("SUPPRESS- not Added(%d): /%S/ status=%s\n",parti
als[i], ustrs[i].getTerminatedBuffer(), u_errorName(status)); | |
| 471 } | |
| 472 } | |
| 473 FB_TRACE("AbbrCount",NULL,FALSE, subCount); | |
| 474 | |
| 475 if(revCount>0) { | |
| 476 backwardsTrie.adoptInstead(builder->build(USTRINGTRIE_BUILD_FAST, status)); | |
| 477 if(U_FAILURE(status)) { | |
| 478 FB_TRACE(u_errorName(status),NULL,FALSE, -1); | |
| 479 return NULL; | |
| 480 } | |
| 481 } | |
| 482 | |
| 483 if(fwdCount>0) { | |
| 484 forwardsPartialTrie.adoptInstead(builder2->build(USTRINGTRIE_BUILD_FAST, sta
tus)); | |
| 485 if(U_FAILURE(status)) { | |
| 486 FB_TRACE(u_errorName(status),NULL,FALSE, -1); | |
| 487 return NULL; | |
| 488 } | |
| 489 } | |
| 490 | |
| 491 return new SimpleFilteredSentenceBreakIterator(adopt.orphan(), forwardsPartial
Trie.orphan(), backwardsTrie.orphan(), status); | |
| 492 } | |
| 493 | |
| 494 | |
| 495 // ----------- Base class implementation | |
| 496 | |
| 497 FilteredBreakIteratorBuilder::FilteredBreakIteratorBuilder() { | |
| 498 } | |
| 499 | |
| 500 FilteredBreakIteratorBuilder::~FilteredBreakIteratorBuilder() { | |
| 501 } | |
| 502 | |
| 503 FilteredBreakIteratorBuilder * | |
| 504 FilteredBreakIteratorBuilder::createInstance(const Locale& where, UErrorCode& st
atus) { | |
| 505 if(U_FAILURE(status)) return NULL; | |
| 506 LocalPointer<FilteredBreakIteratorBuilder> ret(new SimpleFilteredBreakIterator
Builder(where, status)); | |
| 507 if(U_SUCCESS(status) && !ret.isValid()) status = U_MEMORY_ALLOCATION_ERROR; | |
| 508 return ret.orphan(); | |
| 509 } | |
| 510 | |
| 511 FilteredBreakIteratorBuilder * | |
| 512 FilteredBreakIteratorBuilder::createInstance(UErrorCode& status) { | |
| 513 if(U_FAILURE(status)) return NULL; | |
| 514 LocalPointer<FilteredBreakIteratorBuilder> ret(new SimpleFilteredBreakIterator
Builder(status)); | |
| 515 if(U_SUCCESS(status) && !ret.isValid()) status = U_MEMORY_ALLOCATION_ERROR; | |
| 516 return ret.orphan(); | |
| 517 } | |
| 518 | |
| 519 U_NAMESPACE_END | |
| 520 | |
| 521 #endif //#if !UCONFIG_NO_BREAK_ITERATION && U_HAVE_STD_STRING && !UCONFIG_NO_FIL
TERED_BREAK_ITERATION | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1621843002:
ICU 56 update step 1 (Closed)
Base URL: https://chromium.googlesource.com/chromium/deps/icu.git@561