OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 // Implements a custom word iterator used for our spellchecker. | 5 // Implements a custom word iterator used for our spellchecker. |
6 | 6 |
7 #include "chrome/renderer/spellchecker/spellcheck_worditerator.h" | 7 #include "chrome/renderer/spellchecker/spellcheck_worditerator.h" |
8 | 8 |
9 #include <map> | 9 #include <map> |
10 #include <string> | 10 #include <string> |
(...skipping 14 matching lines...) Expand all Loading... |
25 : script_code_(USCRIPT_LATIN) { | 25 : script_code_(USCRIPT_LATIN) { |
26 } | 26 } |
27 | 27 |
28 SpellcheckCharAttribute::~SpellcheckCharAttribute() { | 28 SpellcheckCharAttribute::~SpellcheckCharAttribute() { |
29 } | 29 } |
30 | 30 |
31 void SpellcheckCharAttribute::SetDefaultLanguage(const std::string& language) { | 31 void SpellcheckCharAttribute::SetDefaultLanguage(const std::string& language) { |
32 CreateRuleSets(language); | 32 CreateRuleSets(language); |
33 } | 33 } |
34 | 34 |
35 string16 SpellcheckCharAttribute::GetRuleSet(bool allow_contraction) const { | 35 base::string16 SpellcheckCharAttribute::GetRuleSet( |
| 36 bool allow_contraction) const { |
36 return allow_contraction ? | 37 return allow_contraction ? |
37 ruleset_allow_contraction_ : ruleset_disallow_contraction_; | 38 ruleset_allow_contraction_ : ruleset_disallow_contraction_; |
38 } | 39 } |
39 | 40 |
40 void SpellcheckCharAttribute::CreateRuleSets(const std::string& language) { | 41 void SpellcheckCharAttribute::CreateRuleSets(const std::string& language) { |
41 // The template for our custom rule sets, which is based on the word-break | 42 // The template for our custom rule sets, which is based on the word-break |
42 // rules of ICU 4.0: | 43 // rules of ICU 4.0: |
43 // <http://source.icu-project.org/repos/icu/icu/tags/release-4-0/source/data/b
rkitr/word.txt>. | 44 // <http://source.icu-project.org/repos/icu/icu/tags/release-4-0/source/data/b
rkitr/word.txt>. |
44 // The major differences from the original one are listed below: | 45 // The major differences from the original one are listed below: |
45 // * It discards comments in the original rules. | 46 // * It discards comments in the original rules. |
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
182 kAllowContraction)); | 183 kAllowContraction)); |
183 ruleset_disallow_contraction_ = ASCIIToUTF16( | 184 ruleset_disallow_contraction_ = ASCIIToUTF16( |
184 base::StringPrintf(kRuleTemplate, | 185 base::StringPrintf(kRuleTemplate, |
185 aletter, | 186 aletter, |
186 aletter_extra, | 187 aletter_extra, |
187 midletter_extra, | 188 midletter_extra, |
188 aletter_plus, | 189 aletter_plus, |
189 kDisallowContraction)); | 190 kDisallowContraction)); |
190 } | 191 } |
191 | 192 |
192 bool SpellcheckCharAttribute::OutputChar(UChar c, string16* output) const { | 193 bool SpellcheckCharAttribute::OutputChar(UChar c, |
| 194 base::string16* output) const { |
193 // Call the language-specific function if necessary. | 195 // Call the language-specific function if necessary. |
194 // Otherwise, we call the default one. | 196 // Otherwise, we call the default one. |
195 switch (script_code_) { | 197 switch (script_code_) { |
196 case USCRIPT_ARABIC: | 198 case USCRIPT_ARABIC: |
197 return OutputArabic(c, output); | 199 return OutputArabic(c, output); |
198 | 200 |
199 case USCRIPT_HANGUL: | 201 case USCRIPT_HANGUL: |
200 return OutputHangul(c, output); | 202 return OutputHangul(c, output); |
201 | 203 |
202 case USCRIPT_HEBREW: | 204 case USCRIPT_HEBREW: |
203 return OutputHebrew(c, output); | 205 return OutputHebrew(c, output); |
204 | 206 |
205 default: | 207 default: |
206 return OutputDefault(c, output); | 208 return OutputDefault(c, output); |
207 } | 209 } |
208 } | 210 } |
209 | 211 |
210 bool SpellcheckCharAttribute::OutputArabic(UChar c, string16* output) const { | 212 bool SpellcheckCharAttribute::OutputArabic(UChar c, |
| 213 base::string16* output) const { |
211 // Discard characters not from Arabic alphabets. We also discard vowel marks | 214 // Discard characters not from Arabic alphabets. We also discard vowel marks |
212 // of Arabic (Damma, Fatha, Kasra, etc.) to prevent our Arabic dictionary from | 215 // of Arabic (Damma, Fatha, Kasra, etc.) to prevent our Arabic dictionary from |
213 // marking an Arabic word including vowel marks as misspelled. (We need to | 216 // marking an Arabic word including vowel marks as misspelled. (We need to |
214 // check these vowel marks manually and filter them out since their script | 217 // check these vowel marks manually and filter them out since their script |
215 // codes are USCRIPT_ARABIC.) | 218 // codes are USCRIPT_ARABIC.) |
216 if (0x0621 <= c && c <= 0x064D) | 219 if (0x0621 <= c && c <= 0x064D) |
217 output->push_back(c); | 220 output->push_back(c); |
218 return true; | 221 return true; |
219 } | 222 } |
220 | 223 |
221 bool SpellcheckCharAttribute::OutputHangul(UChar c, string16* output) const { | 224 bool SpellcheckCharAttribute::OutputHangul(UChar c, |
| 225 base::string16* output) const { |
222 // Decompose a Hangul character to a Hangul vowel and consonants used by our | 226 // Decompose a Hangul character to a Hangul vowel and consonants used by our |
223 // spellchecker. A Hangul character of Unicode is a ligature consisting of a | 227 // spellchecker. A Hangul character of Unicode is a ligature consisting of a |
224 // Hangul vowel and consonants, e.g. U+AC01 "Gag" consists of U+1100 "G", | 228 // Hangul vowel and consonants, e.g. U+AC01 "Gag" consists of U+1100 "G", |
225 // U+1161 "a", and U+11A8 "g". That is, we can treat each Hangul character as | 229 // U+1161 "a", and U+11A8 "g". That is, we can treat each Hangul character as |
226 // a point of a cubic linear space consisting of (first consonant, vowel, last | 230 // a point of a cubic linear space consisting of (first consonant, vowel, last |
227 // consonant). Therefore, we can compose a Hangul character from a vowel and | 231 // consonant). Therefore, we can compose a Hangul character from a vowel and |
228 // two consonants with linear composition: | 232 // two consonants with linear composition: |
229 // character = 0xAC00 + | 233 // character = 0xAC00 + |
230 // (first consonant - 0x1100) * 28 * 21 + | 234 // (first consonant - 0x1100) * 28 * 21 + |
231 // (vowel - 0x1161) * 28 + | 235 // (vowel - 0x1161) * 28 + |
(...skipping 26 matching lines...) Expand all Loading... |
258 int l = kLBase + index / kNCount; | 262 int l = kLBase + index / kNCount; |
259 int v = kVBase + (index % kNCount) / kTCount; | 263 int v = kVBase + (index % kNCount) / kTCount; |
260 int t = kTBase + index % kTCount; | 264 int t = kTBase + index % kTCount; |
261 output->push_back(l); | 265 output->push_back(l); |
262 output->push_back(v); | 266 output->push_back(v); |
263 if (t != kTBase) | 267 if (t != kTBase) |
264 output->push_back(t); | 268 output->push_back(t); |
265 return true; | 269 return true; |
266 } | 270 } |
267 | 271 |
268 bool SpellcheckCharAttribute::OutputHebrew(UChar c, string16* output) const { | 272 bool SpellcheckCharAttribute::OutputHebrew(UChar c, |
| 273 base::string16* output) const { |
269 // Discard characters except Hebrew alphabets. We also discard Hebrew niqquds | 274 // Discard characters except Hebrew alphabets. We also discard Hebrew niqquds |
270 // to prevent our Hebrew dictionary from marking a Hebrew word including | 275 // to prevent our Hebrew dictionary from marking a Hebrew word including |
271 // niqquds as misspelled. (Same as Arabic vowel marks, we need to check | 276 // niqquds as misspelled. (Same as Arabic vowel marks, we need to check |
272 // niqquds manually and filter them out since their script codes are | 277 // niqquds manually and filter them out since their script codes are |
273 // USCRIPT_HEBREW.) | 278 // USCRIPT_HEBREW.) |
274 // Pass through ASCII single/double quotation marks and Hebrew Geresh and | 279 // Pass through ASCII single/double quotation marks and Hebrew Geresh and |
275 // Gershayim. | 280 // Gershayim. |
276 if ((0x05D0 <= c && c <= 0x05EA) || c == 0x22 || c == 0x27 || | 281 if ((0x05D0 <= c && c <= 0x05EA) || c == 0x22 || c == 0x27 || |
277 c == 0x05F4 || c == 0x05F3) | 282 c == 0x05F4 || c == 0x05F3) |
278 output->push_back(c); | 283 output->push_back(c); |
279 return true; | 284 return true; |
280 } | 285 } |
281 | 286 |
282 bool SpellcheckCharAttribute::OutputDefault(UChar c, string16* output) const { | 287 bool SpellcheckCharAttribute::OutputDefault(UChar c, |
| 288 base::string16* output) const { |
283 // Check the script code of this character and output only if it is the one | 289 // Check the script code of this character and output only if it is the one |
284 // used by the spellchecker language. | 290 // used by the spellchecker language. |
285 UErrorCode status = U_ZERO_ERROR; | 291 UErrorCode status = U_ZERO_ERROR; |
286 UScriptCode script_code = uscript_getScript(c, &status); | 292 UScriptCode script_code = uscript_getScript(c, &status); |
287 if (script_code == script_code_ || script_code == USCRIPT_COMMON) | 293 if (script_code == script_code_ || script_code == USCRIPT_COMMON) |
288 output->push_back(c); | 294 output->push_back(c); |
289 return true; | 295 return true; |
290 } | 296 } |
291 | 297 |
292 // SpellcheckWordIterator implementation: | 298 // SpellcheckWordIterator implementation: |
(...skipping 11 matching lines...) Expand all Loading... |
304 } | 310 } |
305 | 311 |
306 bool SpellcheckWordIterator::Initialize( | 312 bool SpellcheckWordIterator::Initialize( |
307 const SpellcheckCharAttribute* attribute, | 313 const SpellcheckCharAttribute* attribute, |
308 bool allow_contraction) { | 314 bool allow_contraction) { |
309 // Create a custom ICU break iterator with empty text used in this object. (We | 315 // Create a custom ICU break iterator with empty text used in this object. (We |
310 // allow setting text later so we can re-use this iterator.) | 316 // allow setting text later so we can re-use this iterator.) |
311 DCHECK(attribute); | 317 DCHECK(attribute); |
312 UErrorCode open_status = U_ZERO_ERROR; | 318 UErrorCode open_status = U_ZERO_ERROR; |
313 UParseError parse_status; | 319 UParseError parse_status; |
314 string16 rule(attribute->GetRuleSet(allow_contraction)); | 320 base::string16 rule(attribute->GetRuleSet(allow_contraction)); |
315 | 321 |
316 // If there is no rule set, the attributes were invalid. | 322 // If there is no rule set, the attributes were invalid. |
317 if (rule.empty()) | 323 if (rule.empty()) |
318 return false; | 324 return false; |
319 | 325 |
320 iterator_ = ubrk_openRules(rule.c_str(), rule.length(), NULL, 0, | 326 iterator_ = ubrk_openRules(rule.c_str(), rule.length(), NULL, 0, |
321 &parse_status, &open_status); | 327 &parse_status, &open_status); |
322 if (U_FAILURE(open_status)) | 328 if (U_FAILURE(open_status)) |
323 return false; | 329 return false; |
324 | 330 |
(...skipping 22 matching lines...) Expand all Loading... |
347 // only of Chinese characters while the spellchecker language is English.) | 353 // only of Chinese characters while the spellchecker language is English.) |
348 position_ = ubrk_first(iterator_); | 354 position_ = ubrk_first(iterator_); |
349 if (position_ == UBRK_DONE) | 355 if (position_ == UBRK_DONE) |
350 return false; | 356 return false; |
351 | 357 |
352 text_ = text; | 358 text_ = text; |
353 length_ = static_cast<int>(length); | 359 length_ = static_cast<int>(length); |
354 return true; | 360 return true; |
355 } | 361 } |
356 | 362 |
357 bool SpellcheckWordIterator::GetNextWord(string16* word_string, | 363 bool SpellcheckWordIterator::GetNextWord(base::string16* word_string, |
358 int* word_start, | 364 int* word_start, |
359 int* word_length) { | 365 int* word_length) { |
360 DCHECK(!!text_ && length_ > 0); | 366 DCHECK(!!text_ && length_ > 0); |
361 | 367 |
362 word_string->clear(); | 368 word_string->clear(); |
363 *word_start = 0; | 369 *word_start = 0; |
364 *word_length = 0; | 370 *word_length = 0; |
365 | 371 |
366 if (!text_ || position_ == UBRK_DONE) | 372 if (!text_ || position_ == UBRK_DONE) |
367 return false; | 373 return false; |
(...skipping 26 matching lines...) Expand all Loading... |
394 | 400 |
395 void SpellcheckWordIterator::Reset() { | 401 void SpellcheckWordIterator::Reset() { |
396 if (iterator_) { | 402 if (iterator_) { |
397 ubrk_close(iterator_); | 403 ubrk_close(iterator_); |
398 iterator_ = NULL; | 404 iterator_ = NULL; |
399 } | 405 } |
400 } | 406 } |
401 | 407 |
402 bool SpellcheckWordIterator::Normalize(int input_start, | 408 bool SpellcheckWordIterator::Normalize(int input_start, |
403 int input_length, | 409 int input_length, |
404 string16* output_string) const { | 410 base::string16* output_string) const { |
405 // We use NFKC (Normalization Form, Compatible decomposition, followed by | 411 // We use NFKC (Normalization Form, Compatible decomposition, followed by |
406 // canonical Composition) defined in Unicode Standard Annex #15 to normalize | 412 // canonical Composition) defined in Unicode Standard Annex #15 to normalize |
407 // this token because it it the most suitable normalization algorithm for our | 413 // this token because it it the most suitable normalization algorithm for our |
408 // spellchecker. Nevertheless, it is not a perfect algorithm for our | 414 // spellchecker. Nevertheless, it is not a perfect algorithm for our |
409 // spellchecker and we need manual normalization as well. The normalized | 415 // spellchecker and we need manual normalization as well. The normalized |
410 // text does not have to be NUL-terminated since its characters are copied to | 416 // text does not have to be NUL-terminated since its characters are copied to |
411 // string16, which adds a NUL character when we need. | 417 // string16, which adds a NUL character when we need. |
412 icu::UnicodeString input(FALSE, &text_[input_start], input_length); | 418 icu::UnicodeString input(FALSE, &text_[input_start], input_length); |
413 UErrorCode status = U_ZERO_ERROR; | 419 UErrorCode status = U_ZERO_ERROR; |
414 icu::UnicodeString output; | 420 icu::UnicodeString output; |
415 icu::Normalizer::normalize(input, UNORM_NFKC, 0, output, status); | 421 icu::Normalizer::normalize(input, UNORM_NFKC, 0, output, status); |
416 if (status != U_ZERO_ERROR && status != U_STRING_NOT_TERMINATED_WARNING) | 422 if (status != U_ZERO_ERROR && status != U_STRING_NOT_TERMINATED_WARNING) |
417 return false; | 423 return false; |
418 | 424 |
419 // Copy the normalized text to the output. | 425 // Copy the normalized text to the output. |
420 icu::StringCharacterIterator it(output); | 426 icu::StringCharacterIterator it(output); |
421 for (UChar c = it.first(); c != icu::CharacterIterator::DONE; c = it.next()) | 427 for (UChar c = it.first(); c != icu::CharacterIterator::DONE; c = it.next()) |
422 attribute_->OutputChar(c, output_string); | 428 attribute_->OutputChar(c, output_string); |
423 | 429 |
424 return !output_string->empty(); | 430 return !output_string->empty(); |
425 } | 431 } |
OLD | NEW |