ICU 56 update step 1

source/data/brkitr/line_loose_cj.txt

-# Copyright (c) 2002-2013  International Business Machines Corporation and
+# Copyright (c) 2002-2015  International Business Machines Corporation and
 # others. All Rights Reserved.
 #
-#  file:  line_ja.txt
+#  file:  line_loose_cj.txt
 #
 #         Line Breaking Rules
 #         Implement default line breaking as defined by 
-#         Unicode Standard Annex #14 Revision 29 for Unicode 6.2
+#         Unicode Standard Annex #14 Revision 34 for Unicode 8.0
 #         http://www.unicode.org/reports/tr14/
+#         tailored as noted in 2nd paragraph below..
 #
 #         TODO:  Rule LB 8 remains as it was in Unicode 5.2
 #         This is only because of a limitation of ICU break engine implementation,
 #         not because the older behavior is desirable.
+#
+#         This tailors the line break behavior to correspond to CSS
+#         line-break=loose (BCP47 -u-lb-loose) as defined for Chinese & Japanese.
+#         It sets characters of class CJ to behave like ID.
+#         In addition, it allows breaks:
+#         * before hyphens 2010 & 2013 (both BA) and 301C, 30A0 (both NS)
+#         * before iteration marks 3005, 303B, 309D, 309E, 30FD, 30FE (all NS)
+#         * between characters of LineBreak class IN such as 2026
+#         * before some centered punct 203C, 2047, 2048, 2049, 30FB, FF1A, FF1B,
+#           FF65 (all NS) and FF01, FF1F (both EX).
+#         * before suffix characters with LineBreak class PO and EastAsianWidth A,F,W;
+#           this includes: 00B0 2030 2032 2033 2035 2103 2109 FE6A FF05 FFE0
+#         * after prefix characters with LineBreak class PR and EastAsianWidth A,F,W;
+#           this includes: 00A4 00B1 20AC 2116 FE69 FF04 FFE1 FFE5 FFE6
+
 
 #
 #  Character Classes defined by TR 14.
 #
 
 !!chain;
 !!LBCMNoChain;
 
 
 !!lookAheadHardBreak;
@@ skipping 27 matching lines @@
 #           !!lookAheadHardBreak forces the run time state machine to
 #           stop immediately when a look ahead rule ( '/' operator) matches,
 #           and set the match position to that of the look-ahead operator,
 #           no matter what other rules may be in play at the time.
 #
 #           See rule LB 19 for an example.
 #
 
 $AI = [:LineBreak =  Ambiguous:];
 $AL = [:LineBreak =  Alphabetic:];
-$BA = [:LineBreak =  Break_After:];
+$BAX = [\u2010 \u2013];
+$BA = [[:LineBreak =  Break_After:] - $BAX];
 $BB = [:LineBreak =  Break_Before:];
 $BK = [:LineBreak =  Mandatory_Break:];
 $B2 = [:LineBreak =  Break_Both:];
 $CB = [:LineBreak =  Contingent_Break:];
 $CJ = [:LineBreak =  Conditional_Japanese_Starter:];
 $CL = [:LineBreak =  Close_Punctuation:];
 $CM = [:LineBreak =  Combining_Mark:];
 $CP = [:LineBreak =  Close_Parenthesis:];
 $CR = [:LineBreak =  Carriage_Return:];
-$EX = [:LineBreak =  Exclamation:];
+$EXX = [\uFF01 \uFF1F];
+$EX = [[:LineBreak =  Exclamation:] - $EXX];
 $GL = [:LineBreak =  Glue:];
 $HL = [:LineBreak =  Hebrew_Letter:];
 $HY = [:LineBreak =  Hyphen:];
 $H2 = [:LineBreak =  H2:];
 $H3 = [:LineBreak =  H3:];
 $ID = [[:LineBreak =  Ideographic:] $CJ];
 $IN = [:LineBreak =  Inseperable:];
 $IS = [:LineBreak =  Infix_Numeric:];
 $JL = [:LineBreak =  JL:];
 $JV = [:LineBreak =  JV:];
 $JT = [:LineBreak =  JT:];
 $LF = [:LineBreak =  Line_Feed:];
 $NL = [:LineBreak =  Next_Line:];
-$NS = [:LineBreak =  Nonstarter:];
+$NSX = [\u301C \u30A0 \u3005 \u303B \u309D \u309E \u30FD \u30FE \u203C \u2047 \u2048 \u2049 \u30FB \uFF1A \uFF1B \uFF65];
+$NS = [[:LineBreak =  Nonstarter:] - $NSX];
 $NU = [:LineBreak =  Numeric:];
 $OP = [:LineBreak =  Open_Punctuation:];
-$PO = [:LineBreak =  Postfix_Numeric:];
-$PR = [:LineBreak =  Prefix_Numeric:];
+$POX = [\u00B0 \u2030 \u2032 \u2033 \u2035 \u2103 \u2109 \uFE6A \uFF05 \uFFE0];
+$PO = [[:LineBreak =  Postfix_Numeric:] - $POX];
+$PRX = [\u00A4 \u00B1 \u20AC \u2116 \uFE69 \uFF04 \uFFE1 \uFFE5 \uFFE6];
+$PR = [[:LineBreak =  Prefix_Numeric:] - $PRX];
 $QU = [:LineBreak =  Quotation:];
 $RI = [:LineBreak =  Regional_Indicator:];
 $SA = [:LineBreak =  Complex_Context:];
 $SG = [:LineBreak =  Surrogate:];
 $SP = [:LineBreak =  Space:];
 $SY = [:LineBreak =  Break_Symbols:];
 $WJ = [:LineBreak =  Word_Joiner:];
 $XX = [:LineBreak =  Unknown:];
 $ZW = [:LineBreak =  ZWSpace:];
 
@@ skipping 11 matching lines @@
 #                               XX  (Unknown, unassigned)
 #                         as $AL  (Alphabetic)
 #
 $ALPlus = [$AL $AI $SA $SG $XX];
 
 #
 #  Combining Marks.   X $CM*  behaves as if it were X.  Rule LB6.
 #
 $ALcm = $ALPlus $CM*;
 $BAcm = $BA $CM*;
+$BAXcm = $BAX $CM*;
 $BBcm = $BB $CM*;
 $B2cm = $B2 $CM*;
 $CLcm = $CL $CM*;
 $CPcm = $CP $CM*;
 $EXcm = $EX $CM*;
+$EXXcm = $EXX $CM*;
 $GLcm = $GL $CM*;
 $HLcm = $HL $CM*;
 $HYcm = $HY $CM*;
 $H2cm = $H2 $CM*;
 $H3cm = $H3 $CM*;
 $IDcm = $ID $CM*;
 $INcm = $IN $CM*;
 $IScm = $IS $CM*;
 $JLcm = $JL $CM*;
 $JVcm = $JV $CM*;
 $JTcm = $JT $CM*;
 $NScm = $NS $CM*;
+$NSXcm = $NSX $CM*;
 $NUcm = $NU $CM*;
 $OPcm = $OP $CM*;
 $POcm = $PO $CM*;
+$POXcm = $POX $CM*;
 $PRcm = $PR $CM*;
+$PRXcm = $PRX $CM*;
 $QUcm = $QU $CM*;
 $RIcm = $RI $CM*;
 $SYcm = $SY $CM*;
 $WJcm = $WJ $CM*;
 
 ## -------------------------------------------------
 
 !!forward;
 
 #
 #  Each class of character can stand by itself as an unbroken token, with trailing combining stuff
 #
 $ALPlus $CM+;
 $BA $CM+;
+$BAX $CM+;
 $BB $CM+;
 $B2 $CM+;
 $CL $CM+;
 $CP $CM+;
 $EX $CM+;
+$EXX $CM+;
 $GL $CM+;
 $HL $CM+;
 $HY $CM+;
 $H2 $CM+;
 $H3 $CM+;
 $ID $CM+;
 $IN $CM+;
 $IS $CM+;
 $JL $CM+;
 $JV $CM+;
 $JT $CM+;
 $NS $CM+;
+$NSX $CM+;
 $NU $CM+;
 $OP $CM+;
 $PO $CM+;
+$POX $CM+;
 $PR $CM+;
+$PRX $CM+;
 $QU $CM+;
 $RI $CM+;
 $SY $CM+;
 $WJ $CM+;
 
 #
 # CAN_CM  is the set of characters that may combine with CM combining chars.
 #         Note that Linebreak UAX 14's concept of a combining char and the rules
 #         for what they can combine with are _very_ different from the rest of Unicode.
 #
@@ skipping 65 matching lines @@
 # LB 12  Do not break after NBSP and related characters.
 #         GL  x
 #
 $GLcm $CAN_CM $CM*;
 $GLcm $CANT_CM;
  
 #
 # LB 12a  Do not break before NBSP and related characters ...
 #            [^SP BA HY] x GL
 #
-[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
+[[$LB8NonBreaks] - [$SP $BA $BAX $HY]] $CM* $GLcm;
 $CM+ GLcm;
 
 
 
 #
 # LB 13   Don't break before ']' or '!' or ';' or '/', even after spaces.
 #
+# Do not include $EXX here
 $LB8NonBreaks $CL;
 $CAN_CM $CM*  $CL;
 $CM+          $CL;              # by rule 10, stand-alone CM behaves as AL
 
 $LB8NonBreaks $CP;
 $CAN_CM $CM*  $CP;
 $CM+          $CP;              # by rule 10, stand-alone CM behaves as AL
 
 $LB8NonBreaks $EX;
 $CAN_CM $CM*  $EX;
@@ skipping 13 matching lines @@
 #
 $OPcm $SP* $CAN_CM $CM*;
 $OPcm $SP* $CANT_CM;
 
 $OPcm $SP+ $CM+ $AL_FOLLOW?;    # by rule 10, stand-alone CM behaves as AL
 
 # LB 15
 $QUcm $SP* $OPcm;
 
 # LB 16
+# Do not break between closing punctuation and $NS, even with intervening spaces
+# But DO allow a break between closing punctuation and $NSX, don't include it here
 ($CLcm | $CPcm) $SP* $NScm;
 
 # LB 17
 $B2cm $SP* $B2cm;
 
 #
 # LB 18  Break after spaces.
 #
 $LB18NonBreaks = [$LB8NonBreaks - [$SP]];
 $LB18Breaks    = [$LB8Breaks $SP];
@@ skipping 12 matching lines @@
 
 # LB 20
 #        <break>  $CB
 #        $CB   <break>
 
 $LB20NonBreaks = [$LB18NonBreaks - $CB];
 
 # LB 21        x   (BA | HY | NS)
 #           BB x
 #
+# DO allow breaks here before $BAXcm and $NSXcm, so don't include them
 $LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm); 
 
 $BBcm [^$CB];                                  #  $BB  x
 $BBcm $LB20NonBreaks $CM*;
 
 # LB 21a Don't break after Hebrew + Hyphen
 #   HL (HY | BA) x
 #  
-$HLcm ($HYcm | $BAcm) [^$CB]?;
+$HLcm ($HYcm | $BAcm | $BAXcm) [^$CB]?;
 
 # LB 21b (forward) Don't break between SY and HL
 # (break between HL and SY already disallowed by LB 13 above)
 $SYcm $HLcm;
 
 # LB 22
 ($ALcm | $HLcm) $INcm;
 $CM+     $INcm;     #  by rule 10, any otherwise unattached CM behaves as AL
+$EXcm    $INcm;
 $IDcm    $INcm;
-$INcm    $INcm;
+# $INcm  $INcm; # delete this rule for CSS loose
 $NUcm    $INcm;
 
 
-# $LB 23
+# LB 23
+# Do not include $POX here
 $IDcm  $POcm;
 $ALcm  $NUcm;       # includes $LB19
 $HLcm  $NUcm;
 $CM+   $NUcm;       # Rule 10, any otherwise unattached CM behaves as AL
 $NUcm  $ALcm;
 $NUcm  $HLcm;
 
 #
 # LB 24
 #
+# Do not include $PRX here
 $PRcm $IDcm;
 $PRcm ($ALcm | $HLcm);
-$POcm ($ALcm | $HLcm);
+($POcm | $POXcm) ($ALcm | $HLcm);
 
 #
 # LB 25   Numbers.
 #
-($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
+# Here do not include $PRX at the beginning or $POX at the end
+($PRcm | $POcm | $POXcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $PRXcm | $POcm)?;
 
 # LB 26  Do not break a Korean syllable
 #
 $JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
 ($JVcm | $H2cm) ($JVcm | $JTcm);
 ($JTcm | $H3cm) $JTcm;
 
 # LB 27  Treat korean Syllable Block the same as ID  (don't break it)
+# Do not include $POX or $PRX here
 ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
 ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
 $PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
 
 
 # LB 28   Do not break between alphabetics
 #
 ($ALcm | $HLcm) ($ALcm | $HLcm);
 $CM+ ($ALcm | $HLcm);      # The $CM+ is from rule 10, an unattached CM is treated as AL
 
@@ skipping 10 matching lines @@
 
 #
 #  Reverse Rules.
 #
 ## -------------------------------------------------
 
 !!reverse;
 
 $CM+ $ALPlus;
 $CM+ $BA;
+$CM+ $BAX;
 $CM+ $BB;
 $CM+ $B2;
 $CM+ $CL;
 $CM+ $CP;
 $CM+ $EX;
+$CM+ $EXX;
 $CM+ $GL;
 $CM+ $HL;
 $CM+ $HY;
 $CM+ $H2;
 $CM+ $H3;
 $CM+ $ID;
 $CM+ $IN;
 $CM+ $IS;
 $CM+ $JL;
 $CM+ $JV;
 $CM+ $JT;
 $CM+ $NS;
+$CM+ $NSX;
 $CM+ $NU;
 $CM+ $OP;
 $CM+ $PO;
+$CM+ $POX;
 $CM+ $PR;
+$CM+ $PRX;
 $CM+ $QU;
 $CM+ $RI;
 $CM+ $SY;
 $CM+ $WJ;
 $CM+;
 
 
 #
 #  Sequences of the form  (shown forwards)
 #      [CANT_CM]  <break>  [CM]  [whatever]
@@ skipping 11 matching lines @@
                                                #                a rule compiler bug which complains about
                                                #                empty sets otherwise.
           
 #
 #  Sequences of the form  (shown forwards)
 #      [CANT_CM]  <break> [CM]  <break>  [PR]
 #  The CM needs to behave as an AL
 #  This rule is concerned about getting the second of the two <breaks> in place.
 #
 
-[$PR   ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
+[$PR $PRX  ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
 
 
 
 # LB 4, 5, 5
 
 $LB4Breaks [$LB4NonBreaks-$CM];
 $LB4Breaks $CM+ $CAN_CM;
 $LF $CR;
 
 
@@ skipping 17 matching lines @@
 # LB 11
 $CM* $WJ $CM* $CAN_CM;
 $CM* $WJ      [$LB8NonBreaks-$CM];
 
      $CANT_CM $CM* $WJ;
 $CM* $CAN_CM  $CM* $WJ;
 
 # LB 12a
 #      [^SP BA HY] x GL
 #
-$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
+$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];
 
 # LB 12
 #     GL  x
 #
 $CANT_CM $CM* $GL;
 $CM* $CAN_CM $CM* $GL;
 
 
 # LB 13
+# Do not include $EXX here
 $CL $CM+ $CAN_CM;
 $CP $CM+ $CAN_CM;
 $EX $CM+ $CAN_CM;
 $IS $CM+ $CAN_CM;
 $SY $CM+ $CAN_CM;
 
 $CL [$LB8NonBreaks-$CM];
 $CP [$LB8NonBreaks-$CM];
 $EX [$LB8NonBreaks-$CM];
 $IS [$LB8NonBreaks-$CM];
@@ skipping 15 matching lines @@
      $AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
 $CM* $AL_FOLLOW_CM   $CM+ $SP+ $CM* $OP;
 $SY $CM $SP+ $OP;   # TODO:  Experiment.  Remove.
 
 
 
 # LB 15
 $CM* $OP $SP* $CM* $QU;
 
 # LB 16
+# Don't include $NSX here
 $CM* $NS $SP* $CM* ($CL | $CP);
 
 # LB 17
 $CM* $B2 $SP* $CM* $B2;
 
 # LB 18  break after spaces
 #        Nothing explicit needed here.
 
 
 #
 # LB 19
 #
 $CM* $QU $CM* $CAN_CM;                                #   . x QU
 $CM* $QU      $LB18NonBreaks;
 
 
 $CM* $CAN_CM  $CM* $QU;                               #   QU x .
      $CANT_CM $CM* $QU;
      
 #
 #  LB 20  Break before and after CB.
 #         nothing needed here.
 #
 
 # LB 21
+# Don't include $BAX or $NSX here
 $CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM];     #  . x (BA | HY | NS)
 
 $CM* [$LB20NonBreaks-$CM] $CM* $BB;                   #  BB x .
 [^$CB] $CM* $BB;                                      # 
 
 # LB21a
-[^$CB] $CM* ($HY | $BA) $CM* $HL;
+[^$CB]? $CM* ($HY | $BA | $BAX) $CM* $HL;
 
 # LB21b (reverse)
 $CM* $HL $CM* $SY;
 
 # LB 22
 $CM* $IN $CM* ($ALPlus | $HL);
+$CM* $IN $CM* $EX;
 $CM* $IN $CM* $ID;
-$CM* $IN $CM* $IN;
+# $CM* $IN $CM* $IN; # delete this rule for CSS loose
 $CM* $IN $CM* $NU;
 
 # LB 23
+# Do not include $POX here
 $CM* $PO $CM* $ID;
 $CM* $NU $CM* ($ALPlus | $HL);
 $CM* ($ALPlus | $HL) $CM* $NU;
 
 # LB 24
+# Do not include $PRX here
 $CM* $ID $CM* $PR;
 $CM* ($ALPlus | $HL) $CM* $PR;
-$CM* ($ALPlus | $HL) $CM* $PO;
+$CM* ($ALPlus | $HL) $CM* ($PO | $POX);
 
 
 # LB 25
-($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
+# Here do not include $POX at the beginning or $PRX at the end
+($CM* ($PR | $PRX | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO | $POX))?;
 
 # LB 26
 $CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
 $CM* ($JT | $JV) $CM* ($H2 | $JV);
 $CM* $JT $CM* ($H3 | $JT);
 
 # LB 27
+# Do not include $POX or $PRX here
 $CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
 $CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
 $CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
 
 # LB 28
 $CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
 
 
 # LB 29
 $CM* ($ALPlus | $HL) $CM* $IS;
@@ skipping 19 matching lines @@
 # LB 15
 $SP+ $CM* $QU;
 
 # LB 16
 $SP+ $CM* ($CL | $CP);
 
 # LB 17
 $SP+ $CM* $B2;
 
 # LB 21
-$CM* ($HY | $BA) $CM* $HL;
+$CM* ($HY | $BA | $BAX) $CM* $HL;
 
 # LB 25
 ($CM* ($IS | $SY))+ $CM* $NU;
 ($CL | $CP) $CM* ($NU | $IS | $SY);
 
 # For dictionary-based break
 $dictionary $dictionary;
 
 ## -------------------------------------------------
 
 !!safe_forward;
 
 # Skip forward over all character classes that are involved in
 #   rules containing patterns with possibly more than one char
 #   of context.
 #
 #  It might be slightly more efficient to have specific rules
 #  instead of one generic one, but only if we could
 #  turn off rule chaining.  We don't want to move more
 #  than necessary.
 #
-[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
+[$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $BAX $dictionary];
 $dictionary $dictionary;