Customize Hebrew spellcheck word break iterator...

chrome/renderer/spellchecker/spellcheck_worditerator.cc

 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 // Implements a custom word iterator used for our spellchecker.
 
 #include "chrome/renderer/spellchecker/spellcheck_worditerator.h"
 
 #include <map>
 #include <string>
@@ skipping 42 matching lines @@
   // This template only changes the forward-iteration rules. So, calling
   // ubrk_prev() returns the same results as the original template.
   static const char kRuleTemplate[] =
       "!!chain;"
       "$CR           = [\\p{Word_Break = CR}];"
       "$LF           = [\\p{Word_Break = LF}];"
       "$Newline      = [\\p{Word_Break = Newline}];"
       "$Extend       = [\\p{Word_Break = Extend}];"
       "$Format       = [\\p{Word_Break = Format}];"
       "$Katakana     = [\\p{Word_Break = Katakana}];"
+      // Not all the characters in a given script are ALetter.
+      // For instance, U+05F4 is MidLetter. So, this may be
+      // better, but it leads to an empty set error in Thai.
+      // "$ALetter   = [[\\p{script=%s}] & [\\p{Word_Break = ALetter}]];"
       "$ALetter      = [\\p{script=%s}];"
       "$MidNumLet    = [\\p{Word_Break = MidNumLet}];"
-      "$MidLetter    = [\\p{Word_Break = MidLetter}];"
+      "$MidLetter    = [\\p{Word_Break = MidLetter}%s];"
       "$MidNum       = [\\p{Word_Break = MidNum}];"
       "$Numeric      = [\\p{Word_Break = Numeric}];"
       "$ExtendNumLet = [\\p{Word_Break = ExtendNumLet}];"
 
       "$Control        = [\\p{Grapheme_Cluster_Break = Control}]; "
-      "%s"
+      "%s"  // ALetterPlus
 
       "$KatakanaEx     = $Katakana     ($Extend |  $Format)*;"
       "$ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;"
       "$MidNumLetEx    = $MidNumLet    ($Extend |  $Format)*;"
       "$MidLetterEx    = $MidLetter    ($Extend |  $Format)*;"
       "$MidNumEx       = $MidNum       ($Extend |  $Format)*;"
       "$NumericEx      = $Numeric      ($Extend |  $Format)*;"
       "$ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;"
 
       "$Hiragana       = [\\p{script=Hiragana}];"
       "$Ideographic    = [\\p{Ideographic}];"
       "$HiraganaEx     = $Hiragana     ($Extend |  $Format)*;"
       "$IdeographicEx  = $Ideographic  ($Extend |  $Format)*;"
 
       "!!forward;"
       "$CR $LF;"
       "[^$CR $LF $Newline]? ($Extend |  $Format)+;"
       "$ALetterEx {200};"
       "$ALetterEx $ALetterEx {200};"
-      "%s"
+      "%s"  // (Allow|Disallow) Contraction
 
       "!!reverse;"
       "$BackALetterEx     = ($Format | $Extend)* $ALetterPlus;"
       "$BackMidNumLetEx   = ($Format | $Extend)* $MidNumLet;"
       "$BackNumericEx     = ($Format | $Extend)* $Numeric;"
       "$BackMidNumEx      = ($Format | $Extend)* $MidNum;"
       "$BackMidLetterEx   = ($Format | $Extend)* $MidLetter;"
       "$BackKatakanaEx    = ($Format | $Extend)* $Katakana;"
       "$BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;"
       "$LF $CR;"
@@ skipping 41 matching lines @@
     aletter = "Latin";
 
   const char kWithDictionary[] =
       "$dictionary   = [:LineBreak = Complex_Context:];"
       "$ALetterPlus  = [$ALetter [$dictionary-$Extend-$Control]];";
   const char kWithoutDictionary[] = "$ALetterPlus  = $ALetter;";
   const char* aletter_plus = kWithoutDictionary;
   if (script_code_ == USCRIPT_HANGUL || script_code_ == USCRIPT_THAI)
     aletter_plus = kWithDictionary;
 
+  const char kMidLetterExtra[] = "";
+  // For Hebrew, treat single/double quoation marks as MidLetter.
+  const char kMidLetterExtraHebrew[] = "\"'";
+  const char* midletter_extra = kMidLetterExtra;
+  if (script_code_ == USCRIPT_HEBREW)
+    midletter_extra = kMidLetterExtraHebrew;
+
   // Create two custom rule-sets: one allows contraction and the other does not.
   // We save these strings in UTF-16 so we can use it without conversions. (ICU
   // needs UTF-16 strings.)
   const char kAllowContraction[] =
       "$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};";
   const char kDisallowContraction[] = "";
 
   ruleset_allow_contraction_ = ASCIIToUTF16(StringPrintf(kRuleTemplate,
-      aletter, aletter_plus, kAllowContraction));
+      aletter, midletter_extra, aletter_plus, kAllowContraction));
   ruleset_disallow_contraction_ = ASCIIToUTF16(StringPrintf(kRuleTemplate,
-      aletter, aletter_plus, kDisallowContraction));
+      aletter, midletter_extra, aletter_plus, kDisallowContraction));
 }
 
 bool SpellcheckCharAttribute::OutputChar(UChar c, string16* output) const {
   // Call the language-specific function if necessary.
   // Otherwise, we call the default one.
   switch (script_code_) {
     case USCRIPT_ARABIC:
       return OutputArabic(c, output);
 
     case USCRIPT_HANGUL:
@@ skipping 64 matching lines @@
     output->push_back(t);
   return true;
 }
 
 bool SpellcheckCharAttribute::OutputHebrew(UChar c, string16* output) const {
   // Discard characters except Hebrew alphabets. We also discard Hebrew niqquds
   // to prevent our Hebrew dictionary from marking a Hebrew word including
   // niqquds as misspelled. (Same as Arabic vowel marks, we need to check
   // niqquds manually and filter them out since their script codes are
   // USCRIPT_HEBREW.)
-  if (0x05D0 <= c && c <= 0x05EA)
+  // Pass through ASCII single/double quotation marks and Hebrew Geresh and
+  // Gershayim.
+  if ((0x05D0 <= c && c <= 0x05EA) || c == 0x22 || c == 0x27 ||
+      c == 0x05F4 || c == 0x05F3)
     output->push_back(c);
   return true;
 }
 
 bool SpellcheckCharAttribute::OutputDefault(UChar c, string16* output) const {
   // Check the script code of this character and output only if it is the one
   // used by the spellchecker language.
   UErrorCode status = U_ZERO_ERROR;
   UScriptCode script_code = uscript_getScript(c, &status);
   if (script_code == script_code_ || script_code == USCRIPT_COMMON)
@@ skipping 100 matching lines @@
   if (status != U_ZERO_ERROR && status != U_STRING_NOT_TERMINATED_WARNING)
     return false;
 
   // Copy the normalized text to the output.
   icu::StringCharacterIterator it(output);
   for (UChar c = it.first(); c != icu::CharacterIterator::DONE; c = it.next())
     attribute_->OutputChar(c, output_string);
 
   return !output_string->empty();
 }