| Index: patches/brkitr.patch
|
| ===================================================================
|
| --- patches/brkitr.patch (revision 268566)
|
| +++ patches/brkitr.patch (working copy)
|
| @@ -1,14 +1,12 @@
|
| ---- source/data/brkitr/word.txt 2010-09-17 18:22:35.000000000 -0700
|
| -+++ source/data/brkitr/word.txt 2011-01-25 18:14:09.445378000 -0800
|
| -@@ -29,29 +29,49 @@
|
| - $Newline = [\p{Word_Break = Newline}];
|
| - $Extend = [\p{Word_Break = Extend}];
|
| - $Format = [\p{Word_Break = Format}];
|
| -+$Hiragana = [:Hiragana:];
|
| - $Katakana = [\p{Word_Break = Katakana}];
|
| -+$Han = [:Han:];
|
| - $ALetter = [\p{Word_Break = ALetter}];
|
| --$MidNumLet = [\p{Word_Break = MidNumLet}];
|
| +Index: source/data/brkitr/word.txt
|
| +===================================================================
|
| +--- source/data/brkitr/word.txt (revision 259715)
|
| ++++ source/data/brkitr/word.txt (working copy)
|
| +@@ -35,10 +35,16 @@
|
| + $ALetter = [\p{Word_Break = ALetter}];
|
| + $Single_Quote = [\p{Word_Break = Single_Quote}];
|
| + $Double_Quote = [\p{Word_Break = Double_Quote}];
|
| +-$MidNumLet = [\p{Word_Break = MidNumLet}];
|
| +# Remove two full stop characters from $MidNumLet and add them to $MidNum
|
| +# to break a hostname into its components at the cost of breaking
|
| +# 'e.g.' and 'i.e.' as well.
|
| @@ -16,198 +14,53 @@
|
| +# Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected
|
| +# while rules 6/7 are reverted to the old behavior we want.
|
| +$MidNumLet = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]];
|
| - $MidLetter = [\p{Word_Break = MidLetter}];
|
| --$MidNum = [\p{Word_Break = MidNum}];
|
| --$Numeric = [\p{Word_Break = Numeric}];
|
| + $MidLetter = [\p{Word_Break = MidLetter}];
|
| +-$MidNum = [\p{Word_Break = MidNum}];
|
| +-$Numeric = [\p{Word_Break = Numeric}];
|
| +$MidNum = [\p{Word_Break = MidNum}[\u002E \uFF0E]];
|
| +$Numeric = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits
|
| - $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
|
| + $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
|
|
|
| -+# Extra sets not to break 'HebrewLetter U+0022 HebrewLetter'.
|
| -+$HebrewLet = [\p{Word_Break = ALetter} & \p{Script = Hebrew} - [\u05F3]];
|
| -+# U+05F3 is ALetter and U+05F4 is MidLetter so that they're covered by
|
| -+# the current rule 6/7.
|
| -+$HebrewMidLet = [\u0022];
|
| -
|
| - # Dictionary character set, for triggering language-based break engines. Currently
|
| --# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
|
| --# 5.0 or later as the definition of Complex_Context was corrected to include all
|
| -+# limited to LineBreak=Complex_Context and CJK. Note that this set only works
|
| -+# in Unicode 5.0 or later as the definition of Complex_Context was corrected to include all
|
| - # characters requiring dictionary break.
|
| -
|
| --$dictionary = [:LineBreak = Complex_Context:];
|
| - $Control = [\p{Grapheme_Cluster_Break = Control}];
|
| --$ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]]; # Note: default ALetter does not
|
| -- # include the dictionary characters.
|
| -+$HangulSyllable = [\uac00-\ud7a3];
|
| -+$ComplexContext = [:LineBreak = Complex_Context:];
|
| -+$KanaKanji = [$Han $Hiragana $Katakana];
|
| -+$dictionaryCJK = [$KanaKanji $HangulSyllable];
|
| -+$dictionary = [$ComplexContext $dictionaryCJK];
|
| -+
|
| -+# leave CJK scripts out of ALetterPlus
|
| -+$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
|
| -+
|
| -
|
| - #
|
| - # Rules 4 Ignore Format and Extend characters,
|
| - # except when they appear at the beginning of a region of text.
|
| - #
|
| -+# TODO: check if handling of katakana in dictionary makes rules incorrect/void.
|
| - $KatakanaEx = $Katakana ($Extend | $Format)*;
|
| - $ALetterEx = $ALetterPlus ($Extend | $Format)*;
|
| - $MidNumLetEx = $MidNumLet ($Extend | $Format)*;
|
| -@@ -59,8 +79,8 @@
|
| - $MidNumEx = $MidNum ($Extend | $Format)*;
|
| - $NumericEx = $Numeric ($Extend | $Format)*;
|
| - $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;
|
| -+$HebrewLetEx = $HebrewLet ($Extend | $Format)*;
|
| -
|
| --$Hiragana = [\p{script=Hiragana}];
|
| - $Ideographic = [\p{Ideographic}];
|
| - $HiraganaEx = $Hiragana ($Extend | $Format)*;
|
| - $IdeographicEx = $Ideographic ($Extend | $Format)*;
|
| -@@ -79,12 +99,14 @@
|
| - # begins with a group of Format chars, or with a "word" consisting of a single
|
| - # char that is not in any of the listed word break categories followed by
|
| - # format char(s).
|
| --[^$CR $LF $Newline]? ($Extend | $Format)+;
|
| -+ # format char(s), or is not a CJK dictionary character.
|
| -+[^$CR $LF $Newline $dictionaryCJK]? ($Extend | $Format)+;
|
| -
|
| - $NumericEx {100};
|
| - $ALetterEx {200};
|
| --$KatakanaEx {300}; # note: these status values override those from rule 5
|
| --$HiraganaEx {300}; # by virtual of being numerically larger.
|
| -+$HangulSyllable {200};
|
| -+$KatakanaEx {400}; #originally 300
|
| -+$HiraganaEx {400}; #originally 300
|
| - $IdeographicEx {400}; #
|
| -
|
| - #
|
| -@@ -96,6 +118,9 @@
|
| - # rule 6 and 7
|
| - $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};
|
| -
|
| -+# Chrome addition
|
| -+$HebrewLetEx $HebrewMidLet $HebrewLetEx {200};
|
| -+
|
| - # rule 8
|
| -
|
| - $NumericEx $NumericEx {100};
|
| -@@ -114,19 +139,25 @@
|
| -
|
| - # rule 13
|
| -
|
| --$KatakanaEx $KatakanaEx {300};
|
| -+# To be consistent with '$KanaKanji $KanaKanji', changed
|
| -+# from 300 to 400.
|
| -+# See also TestRuleStatus in intltest/rbbiapts.cpp
|
| -+$KatakanaEx $KatakanaEx {400};
|
| -
|
| - # rule 13a/b
|
| -
|
| - $ALetterEx $ExtendNumLetEx {200}; # (13a)
|
| - $NumericEx $ExtendNumLetEx {100}; # (13a)
|
| --$KatakanaEx $ExtendNumLetEx {300}; # (13a)
|
| -+$KatakanaEx $ExtendNumLetEx {400}; # (13a)
|
| - $ExtendNumLetEx $ExtendNumLetEx {200}; # (13a)
|
| -
|
| - $ExtendNumLetEx $ALetterEx {200}; # (13b)
|
| - $ExtendNumLetEx $NumericEx {100}; # (13b)
|
| --$ExtendNumLetEx $KatakanaEx {300}; # (13b)
|
| --
|
| -+$ExtendNumLetEx $KatakanaEx {400}; # (13b)
|
| -+
|
| -+# special handling for CJK characters: chain for later dictionary segmentation
|
| -+$HangulSyllable $HangulSyllable {200};
|
| -+$KanaKanji $KanaKanji {400}; #different rule status if both kanji and kana found
|
| -
|
| -
|
| - ## -------------------------------------------------
|
| -@@ -139,13 +170,16 @@
|
| - $BackMidNumEx = ($Format | $Extend)* $MidNum;
|
| - $BackMidLetterEx = ($Format | $Extend)* $MidLetter;
|
| - $BackKatakanaEx = ($Format | $Extend)* $Katakana;
|
| -+$BackHiraganaEx = ($Extend | $Format)* $Hiragana;
|
| - $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;
|
| -+$BackHebrewLetEx = ($Format | $Extend)* $HebrewLet;
|
| -+
|
| -
|
| - # rule 3
|
| - $LF $CR;
|
| -
|
| - # rule 4
|
| --($Format | $Extend)* [^$CR $LF $Newline]?;
|
| -+($Format | $Extend)* [^$CR $LF $Newline $dictionaryCJK]?;
|
| -
|
| - # rule 5
|
| -
|
| -@@ -155,6 +189,8 @@
|
| -
|
| - $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;
|
| -
|
| -+# Chrome addition
|
| -+$BackHebrewLetEx $HebrewMidLet $BackHebrewLetEx;
|
| -
|
| - # rule 8
|
| -
|
| -@@ -181,6 +217,10 @@
|
| - $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
|
| - ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
|
| -
|
| -+# special handling for CJK characters: chain for later dictionary segmentation
|
| -+$HangulSyllable $HangulSyllable;
|
| -+$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
|
| -+
|
| - ## -------------------------------------------------
|
| -
|
| - !!safe_reverse;
|
| ---- source/data/brkitr/line.txt 2010-09-17 18:22:35.000000000 -0700
|
| -+++ source/data/brkitr/line.txt 2011-07-22 13:46:19.923562000 -0700
|
| -@@ -11,6 +11,9 @@
|
| - # TODO: Rule LB 8 remains as it was in Unicode 5.2
|
| + $Han = [:Han:];
|
| +Index: source/data/brkitr/line.txt
|
| +===================================================================
|
| +--- source/data/brkitr/line.txt (revision 259715)
|
| ++++ source/data/brkitr/line.txt (working copy)
|
| +@@ -12,9 +12,8 @@
|
| # This is only because of a limitation of ICU break engine implementation,
|
| # not because the older behavior is desirable.
|
| -+#
|
| -+# CHROME: Hebrew tailoring was incorporatd as well as some
|
| -+# other minor changes (CL, OP, ID).
|
|
|
| +-#
|
| +-# Character Classes defined by TR 14.
|
| +-#
|
| ++# CHROME: 1. Use line_ja.txt to apply small kana rules in all locales.
|
| ++# 2. Adjust CL, OP, and IS to handle 'comma-variants' consistently.
|
| +
|
| + !!chain;
|
| + !!LBCMNoChain;
|
| +@@ -57,14 +56,14 @@
|
| #
|
| - # Character Classes defined by TR 14.
|
| -@@ -55,15 +58,22 @@
|
| - #
|
| - # See rule LB 19 for an example.
|
| - #
|
| -+$SmallHira = [\u3041 \u3043 \u3045 \u3047 \u3049 \u3063 \u3083 \u3085 \u3087 \u308E \u3095 \u3096];
|
| -+$SmallKata = [\u30A1 \u30A3 \u30A5 \u30A7 \u30A9 \u30C3 \u30E3 \u30E5 \u30E7 \u30EE \u30F5 \u30F6];
|
| -+$SmallKataExt = [\u31F0 \u31F1 \u31F2 \u31F3 \u31F4 \u31F5 \u31F6 \u31F7 \u31F8 \u31F9 \u31FA \u31FB \u31FC \u31FD \u31FE \u31FF];
|
| -+$SmallKanaAndProlongedMark = [[$SmallHira] [$SmallKata] [$SmallKataExt] [\u30FC]];
|
| -+
|
|
|
| $AI = [:LineBreak = Ambiguous:];
|
| -$AL = [:LineBreak = Alphabetic:];
|
| --$BA = [:LineBreak = Break_After:];
|
| -+$AL = [[:LineBreak = Alphabetic:] - [[:Hebrew:] & [:Letter:]] - [\u23B4\u23B5]];
|
| -+$HL = [[:Hebrew:] & [:Letter:]];
|
| -+$BA = [[:LineBreak = Break_After:] - [\u2010]];
|
| -+$HH = [\u2010];
|
| ++$AL = [[:LineBreak = Alphabetic:] - [\u23B4\u23B5]];
|
| + $BA = [:LineBreak = Break_After:];
|
| $BB = [:LineBreak = Break_Before:];
|
| $BK = [:LineBreak = Mandatory_Break:];
|
| $B2 = [:LineBreak = Break_Both:];
|
| $CB = [:LineBreak = Contingent_Break:];
|
| + $CJ = [:LineBreak = Conditional_Japanese_Starter:];
|
| -$CL = [:LineBreak = Close_Punctuation:];
|
| +$CL = [[:LineBreak = Close_Punctuation:] [\uFE51\uFE10\u23B5]];
|
| $CM = [:LineBreak = Combining_Mark:];
|
| $CP = [:LineBreak = Close_Parenthesis:];
|
| $CR = [:LineBreak = Carriage_Return:];
|
| -@@ -72,17 +82,17 @@
|
| +@@ -74,16 +73,16 @@
|
| $HY = [:LineBreak = Hyphen:];
|
| $H2 = [:LineBreak = H2:];
|
| $H3 = [:LineBreak = H3:];
|
| -$ID = [:LineBreak = Ideographic:];
|
| -+$ID = [[[:LineBreak = Ideographic:] - [\uFE51]] [$SmallKanaAndProlongedMark]];
|
| ++$ID = [[[:LineBreak = Ideographic:] $CJ] - [\uFE51]];
|
| $IN = [:LineBreak = Inseperable:];
|
| -$IS = [:LineBreak = Infix_Numeric:];
|
| +$IS = [[:LineBreak = Infix_Numeric:] - [\uFE10]];
|
| @@ -216,158 +69,10 @@
|
| $JT = [:LineBreak = JT:];
|
| $LF = [:LineBreak = Line_Feed:];
|
| $NL = [:LineBreak = Next_Line:];
|
| --$NS = [:LineBreak = Nonstarter:];
|
| -+$NS = [[:LineBreak = Nonstarter:] - [$SmallKanaAndProlongedMark]];
|
| +-$NS = [[:LineBreak = Nonstarter:] $CJ];
|
| ++$NS = [:LineBreak = Nonstarter:];
|
| $NU = [:LineBreak = Numeric:];
|
| -$OP = [:LineBreak = Open_Punctuation:];
|
| +$OP = [[:LineBreak = Open_Punctuation:] \u23B4];
|
| $PO = [:LineBreak = Postfix_Numeric:];
|
| $PR = [:LineBreak = Prefix_Numeric:];
|
| - $QU = [:LineBreak = Quotation:];
|
| -@@ -108,13 +118,15 @@
|
| - # XX (Unknown, unassigned)
|
| - # as $AL (Alphabetic)
|
| - #
|
| --$ALPlus = [$AL $AI $SA $SG $XX];
|
| -+$ALPlus = [$AL $HL $AI $SA $SG $XX];
|
| -
|
| - #
|
| - # Combining Marks. X $CM* behaves as if it were X. Rule LB6.
|
| - #
|
| - $ALcm = $ALPlus $CM*;
|
| -+$HLcm = $HL $CM*;
|
| - $BAcm = $BA $CM*;
|
| -+$HHcm = $HH $CM*;
|
| - $BBcm = $BB $CM*;
|
| - $B2cm = $B2 $CM*;
|
| - $CLcm = $CL $CM*;
|
| -@@ -148,6 +160,7 @@
|
| - #
|
| - $ALPlus $CM+;
|
| - $BA $CM+;
|
| -+$HH $CM+;
|
| - $BB $CM+;
|
| - $B2 $CM+;
|
| - $CL $CM+;
|
| -@@ -190,7 +203,7 @@
|
| - # so for this one case we need to manually list out longer sequences.
|
| - #
|
| - $AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
|
| --$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];
|
| -+$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus];
|
| - $AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
|
| -
|
| -
|
| -@@ -252,7 +265,7 @@
|
| - # LB 12a Do not break before NBSP and related characters ...
|
| - # [^SP BA HY] x GL
|
| - #
|
| --[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
|
| -+[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm;
|
| - $CM+ GLcm;
|
| -
|
| -
|
| -@@ -325,7 +338,7 @@
|
| - # LB 21 x (BA | HY | NS)
|
| - # BB x
|
| - #
|
| --$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
|
| -+$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm);
|
| -
|
| - $BBcm [^$CB]; # $BB x
|
| - $BBcm $LB20NonBreaks $CM*;
|
| -@@ -381,6 +394,8 @@
|
| - $CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
|
| - $CPcm ($ALcm | $NUcm);
|
| -
|
| -+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
|
| -+$HLcm ($HY | $HH) $ALcm;
|
| -
|
| - #
|
| - # Reverse Rules.
|
| -@@ -391,6 +406,7 @@
|
| -
|
| - $CM+ $ALPlus;
|
| - $CM+ $BA;
|
| -+$CM+ $HH;
|
| - $CM+ $BB;
|
| - $CM+ $B2;
|
| - $CM+ $CL;
|
| -@@ -479,7 +495,7 @@
|
| - # LB 12a
|
| - # [^SP BA HY] x GL
|
| - #
|
| --$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
|
| -+$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
|
| -
|
| - # LB 12
|
| - # GL x
|
| -@@ -549,7 +565,7 @@
|
| - #
|
| -
|
| - # LB 21
|
| --$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
|
| -+$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
|
| -
|
| - $CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
|
| - [^$CB] $CM* $BB; #
|
| -@@ -597,6 +613,8 @@
|
| - $CM* $OP $CM* ($ALPlus | $NU);
|
| - $CM* ($ALPlus | $NU) $CM* $CP;
|
| -
|
| -+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
|
| -+$CM* $ALPlus ($HY | $HH) $CM* $HL;
|
| -
|
| - ## -------------------------------------------------
|
| -
|
| ---- source/data/brkitr/brklocal.mk 1969-12-31 16:00:00.000000000 -0800
|
| -+++ source/data/brkitr/brklocal.mk 2011-01-25 17:54:01.149770000 -0800
|
| -@@ -0,0 +1,48 @@
|
| -+# * Copyright (C) 1998-2010, International Business Machines
|
| -+# * Corporation and others. All Rights Reserved.
|
| -+BRK_RES_CLDR_VERSION = 1.9
|
| -+# A list of txt's to build
|
| -+# Note:
|
| -+#
|
| -+# If you are thinking of modifying this file, READ THIS.
|
| -+#
|
| -+# Instead of changing this file [unless you want to check it back in],
|
| -+# you should consider creating a 'brklocal.mk' file in this same directory.
|
| -+# Then, you can have your local changes remain even if you upgrade or
|
| -+# reconfigure ICU.
|
| -+#
|
| -+# Example 'brklocal.mk' files:
|
| -+#
|
| -+# * To add an additional locale to the list:
|
| -+# _____________________________________________________
|
| -+# | BRK_RES_SOURCE_LOCAL = myLocale.txt ...
|
| -+#
|
| -+# * To REPLACE the default list and only build with a few
|
| -+# locales:
|
| -+# _____________________________________________________
|
| -+# | BRK_RES_SOURCE = ar.txt ar_AE.txt en.txt de.txt zh.txt
|
| -+#
|
| -+#
|
| -+# Generated by LDML2ICUConverter, from LDML source files.
|
| -+
|
| -+# Aliases without a corresponding xx.xml file (see icu-config.xml & build.xml)
|
| -+BRK_RES_SYNTHETIC_ALIAS =
|
| -+
|
| -+
|
| -+# All aliases (to not be included under 'installed'), but not including root.
|
| -+BRK_RES_ALIAS_SOURCE = $(BRK_RES_SYNTHETIC_ALIAS)
|
| -+
|
| -+
|
| -+# List of compact trie dictionary files (ctd).
|
| -+BRK_CTD_SOURCE = thaidict.txt cjdict.txt
|
| -+
|
| -+
|
| -+# List of break iterator files (brk).
|
| -+# Chrome change: remove word_ja.txt and line_he.txt
|
| -+BRK_SOURCE = sent_el.txt word_POSIX.txt line_fi.txt char.txt word.txt line.txt sent.txt title.txt char_th.txt
|
| -+
|
| -+
|
| -+# Ordinary resources
|
| -+# Chrome change: remove ja.txt and he.txt
|
| -+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt\
|
| -+ fi.txt th.txt
|
|
|
Chromium Code Reviews