organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-13 08:53:20 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 2

ICU-9379 Draft new linebreak files & related generated data per cldrbug 4931

Browse source 
X-SVN-Rev: 37037
This commit is contained in:
Peter Edberg 2015-02-18 08:37:16 +00:00
parent d8b9e7ecaa
commit 3565561b05
20 changed files with 4321 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.gitattributes vendored
View file

@ -53,6 +53,14 @@ icu4c/source/aclocal.m4 -text
icu4c/source/allinone/icucheck.bat -text
icu4c/source/common/common.vcxproj -text
icu4c/source/common/common.vcxproj.filters -text
icu4c/source/data/brkitr/line_loose.txt -text
icu4c/source/data/brkitr/line_loose_cj.txt -text
icu4c/source/data/brkitr/line_loose_fi.txt -text
icu4c/source/data/brkitr/line_normal.txt -text
icu4c/source/data/brkitr/line_normal_cj.txt -text
icu4c/source/data/brkitr/line_normal_fi.txt -text
icu4c/source/data/brkitr/zh.txt -text
icu4c/source/data/brkitr/zh_Hant.txt -text
icu4c/source/data/curr/pool.res -text
icu4c/source/data/in/coll/ucadata-implicithan.icu -text
icu4c/source/data/in/coll/ucadata-unihan.icu -text
@ -71,6 +79,8 @@ icu4c/source/data/makedata.vcxproj -text
icu4c/source/data/makedata.vcxproj.filters -text
icu4c/source/data/region/pool.res -text
icu4c/source/data/unit/pool.res -text
icu4c/source/data/xml/brkitr/zh.xml -text
icu4c/source/data/xml/brkitr/zh_Hant.xml -text
icu4c/source/data/zone/pool.res -text
icu4c/source/extra/uconv/uconv.vcxproj -text
icu4c/source/extra/uconv/uconv.vcxproj.filters -text

7
icu4c/source/data/brkitr/brkfiles.mk
View file

@ -1,4 +1,4 @@
# * Copyright (C) 1998-2014, International Business Machines
# * Copyright (C) 1998-2015, International Business Machines
# * Corporation and others. All Rights Reserved.
BRK_RES_CLDR_VERSION = %version%
# A list of txt's to build
@ -39,12 +39,13 @@ BRK_DICT_SOURCE = burmesedict.txt cjdict.txt khmerdict.txt laodict.txt\
# List of break iterator files (brk).
BRK_SOURCE = char.txt line.txt line_fi.txt line_ja.txt\
BRK_SOURCE = char.txt line.txt line_fi.txt line_loose.txt\
line_loose_cj.txt line_loose_fi.txt line_normal.txt line_normal_cj.txt line_normal_fi.txt\
sent.txt sent_el.txt title.txt word.txt word_POSIX.txt
# Ordinary resources
BRK_RES_SOURCE = de.txt el.txt en.txt en_US.txt\
en_US_POSIX.txt es.txt fi.txt fr.txt it.txt\
ja.txt pt.txt ru.txt
ja.txt pt.txt ru.txt zh.txt zh_Hant.txt

5
icu4c/source/data/brkitr/fi.txt
View file

@ -1,6 +1,6 @@
// ***************************************************************************
// *
// * Copyright (C) 2014 International Business Machines
// * Copyright (C) 2015 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: org.unicode.cldr.icu.NewLdml2IcuConverter
// * Source File: <path>/common/segments/fi.xml ../../xml/brkitr/fi.xml
@ -10,5 +10,8 @@ fi{
Version{"2.0.82.42"}
boundaries{
line:process(dependency){"line_fi.brk"}
line_loose:process(dependency){"line_loose_fi.brk"}
line_normal:process(dependency){"line_normal_fi.brk"}
line_strict:process(dependency){"line_fi.brk"}
}
}

7
icu4c/source/data/brkitr/ja.txt
View file

@ -1,6 +1,6 @@
// ***************************************************************************
// *
// * Copyright (C) 2014 International Business Machines
// * Copyright (C) 2015 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: org.unicode.cldr.icu.NewLdml2IcuConverter
// * Source File: <path>/common/segments/ja.xml ../../xml/brkitr/ja.xml
@ -9,6 +9,9 @@
ja{
Version{"2.0.82.42"}
boundaries{
line:process(dependency){"line_ja.brk"}
line:process(dependency){"line_normal.brk"}
line_loose:process(dependency){"line_loose_cj.brk"}
line_normal:process(dependency){"line_normal_cj.brk"}
line_strict:process(dependency){"line.brk"}
}
}

5
icu4c/source/data/brkitr/line.txt
View file

@ -1,4 +1,4 @@
# Copyright (c) 2002-2013 International Business Machines Corporation and
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line.txt
@ -11,6 +11,9 @@
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This corresponds to CSS line-break=strict (BCP47 -u-lb-strict).
# It sets characters of class CJ to behave like NS.
#
# Character Classes defined by TR 14.

7
icu4c/source/data/brkitr/line_fi.txt
View file

@ -1,4 +1,4 @@
# Copyright (c) 2002-2013 International Business Machines Corporation and
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_fi.txt
@ -7,10 +7,15 @@
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior for Finnish, while otherwise behaving
# per UAX 14 which corresponds to CSS line-break=strict (BCP47 -u-lb-strict).
# It sets characters of class CJ to behave like NS.
#
# Character Classes defined by TR 14.

692
icu4c/source/data/brkitr/line_loose.txt Normal file
View file

@ -0,0 +1,692 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_loose.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior to correspond to CSS
# line-break=loose (BCP47 -u-lb-loose) as defined for languages other than
# Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
# In addition, it allows breaks:
# * before iteration marks 3005, 303B, 309D, 309E, 30FD, 30FE (all NS)
# * between characters of LineBreak class IN
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BA = [:LineBreak = Break_After:];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EX = [:LineBreak = Exclamation:];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NSX = [\u3005 \u303B \u309D \u309E \u30FD \u30FE];
$NS = [[:LineBreak = Nonstarter:] - $NSX];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NSXcm = $NSX $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$PRcm = $PR $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NSX $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$PR $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $NSX $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
# Do not break between closing punctuation and $NS, even with intervening spaces
# But DO allow a break between closing punctuation and $NSX, don't include it here
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 21 x (BA | HY | NS)
# BB x
#
# DO allow breaks here before NSXcm, so don't include it
$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
# $INcm $INcm; # delete this rule for CSS loose
$NUcm $INcm;
# $LB 23
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
$POcm ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NSX;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $PR;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
# Don't include $NSX here
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 21
# Don't include $NSX here
$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
# $CM* $IN $CM* $IN; # delete this rule for CSS loose
$CM* $IN $CM* $NU;
# LB 23
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PO;
# LB 25
($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
$dictionary $dictionary;

725
icu4c/source/data/brkitr/line_loose_cj.txt Normal file
View file

@ -0,0 +1,725 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_loose_cj.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior to correspond to CSS
# line-break=loose (BCP47 -u-lb-loose) as defined for Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
# In addition, it allows breaks:
# * before hyphens 2010 & 2013 (both BA) and 301C, 30A0 (both NS)
# * before iteration marks 3005, 303B, 309D, 309E, 30FD, 30FE (all NS)
# * between characters of LineBreak class IN such as 2026
# * before some centered punct 203C, 2047, 2048, 2049, 30FB, FF1A, FF1B,
# FF65 (all NS) and FF01, FF1F (both EX).
# * before suffix characters with LineBreak class PO and EastAsianWidth A,F,W;
# this includes: 00B0 2030 2032 2033 2035 2103 2109 FE6A FF05 FFE0
# * after prefix characters with LineBreak class PR and EastAsianWidth A,F,W;
# this includes: 00A4 00B1 20AC 2116 FE69 FF04 FFE1 FFE5 FFE6
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BAX = [\u2010 \u2013];
$BA = [[:LineBreak = Break_After:] - $BAX];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EXX = [\uFF01 \uFF1F];
$EX = [[:LineBreak = Exclamation:] - $EXX];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NSX = [\u301C \u30A0 \u3005 \u303B \u309D \u309E \u30FD \u30FE \u203C \u2047 \u2048 \u2049 \u30FB \uFF1A \uFF1B \uFF65];
$NS = [[:LineBreak = Nonstarter:] - $NSX];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$POX = [\u00B0 \u2030 \u2032 \u2033 \u2035 \u2103 \u2109 \uFE6A \uFF05 \uFFE0];
$PO = [[:LineBreak = Postfix_Numeric:] - $POX];
$PRX = [\u00A4 \u00B1 \u20AC \u2116 \uFE69 \uFF04 \uFFE1 \uFFE5 \uFFE6];
$PR = [[:LineBreak = Prefix_Numeric:] - $PRX];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$BAXcm = $BAX $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$EXXcm = $EXX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NSXcm = $NSX $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$POXcm = $POX $CM*;
$PRcm = $PR $CM*;
$PRXcm = $PRX $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$BAX $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$EXX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NSX $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$POX $CM+;
$PR $CM+;
$PRX $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $EXX $HL $IS $SY $WJ $GL $OP $QU $BA $BAX $HY $NS $NSX $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $BAX $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
# Do not include $EXX here
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
# Do not break between closing punctuation and $NS, even with intervening spaces
# But DO allow a break between closing punctuation and $NSX, don't include it here
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 21 x (BA | HY | NS)
# BB x
#
# DO allow breaks here before $BAXcm and $NSXcm, so don't include them
$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm | $BAXcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
# $INcm $INcm; # delete this rule for CSS loose
$NUcm $INcm;
# LB 23
# Do not include $POX here
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
# Do not include $PRX here
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
($POcm | $POXcm) ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
# Here do not include $PRX at the beginning or $POX at the end
($PRcm | $POcm | $POXcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $PRXcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
# Do not include $POX or $PRX here
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $BAX;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $EXX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NSX;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $POX;
$CM+ $PR;
$CM+ $PRX;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR $PRX ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
# Do not include $EXX here
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
# Don't include $NSX here
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 21
# Don't include $BAX or $NSX here
$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA | $BAX) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
# $CM* $IN $CM* $IN; # delete this rule for CSS loose
$CM* $IN $CM* $NU;
# LB 23
# Do not include $POX here
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
# Do not include $PRX here
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* ($PO | $POX);
# LB 25
# Here do not include $POX at the beginning or $PRX at the end
($CM* ($PR | $PRX | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO | $POX))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
# Do not include $POX or $PRX here
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $PRX $HY $BA $dictionary];
$dictionary $dictionary;

700
icu4c/source/data/brkitr/line_loose_fi.txt Normal file
View file

@ -0,0 +1,700 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_loose_fi.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior both for Finnish and to correpond to CSS
# line-break=loose (BCP47 -u-lb-loose) as defined for languages other than
# Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
# In addition, it allows breaks before 3005, 303B, 309D, 309E, 30FD, 30FE (all NS).
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BA = [[:LineBreak = Break_After:] - [\u2010]];
$HH = [\u2010];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EX = [:LineBreak = Exclamation:];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NSX = [\u3005 \u303B \u309D \u309E \u30FD \u30FE];
$NS = [[:LineBreak = Nonstarter:] - $NSX];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$HHcm = $HH $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NSXcm = $NSX $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$PRcm = $PR $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$HH $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NSX $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$PR $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $NSX $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
# Do not break between closing punctuation and $NS, even with intervening spaces
# But DO allow a break between closing punctuation and $NSX, don't include it here
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 20.09 added rule for Finnish tailoring
# LB 21 x (BA | HY | NS)
# BB x
#
# DO allow breaks here before NSXcm, so don't include it
$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm) / $AL;
$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm);
($HY | $HH) $AL;
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm | $HHcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
$INcm $INcm;
$NUcm $INcm;
# $LB 23
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
$POcm ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $HH;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NSX;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $PR;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
# Don't include $NSX here
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 20.09 added rule for Finnish tailoring
$AL ($HY | $HH) / $SP;
# LB 21
# Don't include $NSX here
$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA | $HH) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
$CM* $IN $CM* $IN;
$CM* $IN $CM* $NU;
# LB 23
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PO;
# LB 25
($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA | $HH) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
$dictionary $dictionary;

680
icu4c/source/data/brkitr/line_normal.txt Normal file
View file

@ -0,0 +1,680 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_normal.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior to correspond to CSS
# line-break=normal (BCP47 -u-lb-normal) as defined for languages other than
# Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BA = [:LineBreak = Break_After:];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EX = [:LineBreak = Exclamation:];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NS = [:LineBreak = Nonstarter:];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$PRcm = $PR $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$PR $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 21 x (BA | HY | NS)
# BB x
#
$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
$INcm $INcm;
$NUcm $INcm;
# $LB 23
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
$POcm ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $PR;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 21
$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
$CM* $IN $CM* $IN;
$CM* $IN $CM* $NU;
# LB 23
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PO;
# LB 25
($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
$dictionary $dictionary;

694
icu4c/source/data/brkitr/line_normal_cj.txt Normal file
View file

@ -0,0 +1,694 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_normal_cj.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior to correspond to CSS
# line-break=normal (BCP47 -u-lb-normal) as defined for Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
# In addition, it allows breaks:
# * before hyphens 2010 & 2013 (both BA) and 301C, 30A0 (both NS)
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BAX = [\u2010 \u2013];
$BA = [[:LineBreak = Break_After:] - $BAX];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EX = [:LineBreak = Exclamation:];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NSX = [\u301C \u30A0];
$NS = [[:LineBreak = Nonstarter:] - $NSX];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$BAXcm = $BAX $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NSXcm = $NSX $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$PRcm = $PR $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$BAX $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NSX $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$PR $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $BAX $HY $NS $NSX $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $BAX $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
# Do not break between closing punctuation and $NS, even with intervening spaces
# But DO allow a break between closing punctuation and $NSX, don't include it here
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 21 x (BA | HY | NS)
# BB x
#
# DO allow breaks here before $BAXcm and $NSXcm, so don't include them
$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm | $BAXcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
$INcm $INcm;
$NUcm $INcm;
# $LB 23
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
$POcm ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $BAX;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NSX;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $PR;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $BAX $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
# Don't include $NSX here
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 21
# Don't include $BAX or $NSX here
$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA | $BAX) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
$CM* $IN $CM* $IN;
$CM* $IN $CM* $NU;
# LB 23
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PO;
# LB 25
($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
$dictionary $dictionary;

690
icu4c/source/data/brkitr/line_normal_fi.txt Normal file
View file

@ -0,0 +1,690 @@
# Copyright (c) 2002-2015 International Business Machines Corporation and
# others. All Rights Reserved.
#
# file: line_normal_fi.txt
#
# Line Breaking Rules
# Implement default line breaking as defined by
# Unicode Standard Annex #14 Revision 29 for Unicode 6.2
# http://www.unicode.org/reports/tr14/
# tailored as noted in 2nd paragraph below..
#
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
#
# This tailors the line break behavior both for Finnish and to correpond to CSS
# line-break=normal (BCP47 -u-lb-normal) as defined for languages other than
# Chinese & Japanese.
# It sets characters of class CJ to behave like ID.
#
# Character Classes defined by TR 14.
#
!!chain;
!!LBCMNoChain;
!!lookAheadHardBreak;
#
# !!lookAheadHardBreak Described here because it is (as yet) undocumented elsewhere
# and only used for the line break rules.
#
# It is used in the implementation of rule LB 10
# which says to treat any combining mark that is not attached to a base
# character as if it were of class AL (alphabetic).
#
# The problem occurs in the reverse rules.
#
# Consider a sequence like, with correct breaks as shown
# LF ID CM AL AL
# ^ ^ ^
# Then consider the sequence without the initial ID (ideographic)
# LF CM AL AL
# ^ ^
# Our CM, which in the first example was attached to the ideograph,
# is now unattached, becomes an alpha, and joins in with the other
# alphas.
#
# When iterating forwards, these sequences do not present any problems
# When iterating backwards, we need to look ahead when encountering
# a CM to see whether it attaches to something further on or not.
# (Look-ahead in a reverse rule is looking towards the start)
#
# If the CM is unattached, we need to force a break.
#
# !!lookAheadHardBreak forces the run time state machine to
# stop immediately when a look ahead rule ( '/' operator) matches,
# and set the match position to that of the look-ahead operator,
# no matter what other rules may be in play at the time.
#
# See rule LB 19 for an example.
#
$AI = [:LineBreak = Ambiguous:];
$AL = [:LineBreak = Alphabetic:];
$BA = [[:LineBreak = Break_After:] - [\u2010]];
$HH = [\u2010];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
$CL = [:LineBreak = Close_Punctuation:];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
$EX = [:LineBreak = Exclamation:];
$GL = [:LineBreak = Glue:];
$HL = [:LineBreak = Hebrew_Letter:];
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
$ID = [[:LineBreak = Ideographic:] $CJ];
$IN = [:LineBreak = Inseperable:];
$IS = [:LineBreak = Infix_Numeric:];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
$LF = [:LineBreak = Line_Feed:];
$NL = [:LineBreak = Next_Line:];
$NS = [:LineBreak = Nonstarter:];
$NU = [:LineBreak = Numeric:];
$OP = [:LineBreak = Open_Punctuation:];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
$RI = [:LineBreak = Regional_Indicator:];
$SA = [:LineBreak = Complex_Context:];
$SG = [:LineBreak = Surrogate:];
$SP = [:LineBreak = Space:];
$SY = [:LineBreak = Break_Symbols:];
$WJ = [:LineBreak = Word_Joiner:];
$XX = [:LineBreak = Unknown:];
$ZW = [:LineBreak = ZWSpace:];
# Dictionary character set, for triggering language-based break engines. Currently
# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
# 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
$dictionary = [:LineBreak = Complex_Context:];
#
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
# SA (South East Asian: Thai, Lao, Khmer)
# SG (Unpaired Surrogates)
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
$ALPlus = [$AL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
$BAcm = $BA $CM*;
$HHcm = $HH $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
$CPcm = $CP $CM*;
$EXcm = $EX $CM*;
$GLcm = $GL $CM*;
$HLcm = $HL $CM*;
$HYcm = $HY $CM*;
$H2cm = $H2 $CM*;
$H3cm = $H3 $CM*;
$IDcm = $ID $CM*;
$INcm = $IN $CM*;
$IScm = $IS $CM*;
$JLcm = $JL $CM*;
$JVcm = $JV $CM*;
$JTcm = $JT $CM*;
$NScm = $NS $CM*;
$NUcm = $NU $CM*;
$OPcm = $OP $CM*;
$POcm = $PO $CM*;
$PRcm = $PR $CM*;
$QUcm = $QU $CM*;
$RIcm = $RI $CM*;
$SYcm = $SY $CM*;
$WJcm = $WJ $CM*;
## -------------------------------------------------
!!forward;
#
# Each class of character can stand by itself as an unbroken token, with trailing combining stuff
#
$ALPlus $CM+;
$BA $CM+;
$HH $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
$CP $CM+;
$EX $CM+;
$GL $CM+;
$HL $CM+;
$HY $CM+;
$H2 $CM+;
$H3 $CM+;
$ID $CM+;
$IN $CM+;
$IS $CM+;
$JL $CM+;
$JV $CM+;
$JT $CM+;
$NS $CM+;
$NU $CM+;
$OP $CM+;
$PO $CM+;
$PR $CM+;
$QU $CM+;
$RI $CM+;
$SY $CM+;
$WJ $CM+;
#
# CAN_CM is the set of characters that may combine with CM combining chars.
# Note that Linebreak UAX 14's concept of a combining char and the rules
# for what they can combine with are _very_ different from the rest of Unicode.
#
# Note that $CM itself is left out of this set. If CM is needed as a base
# it must be listed separately in the rule.
#
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
#
# AL_FOLLOW set of chars that can unconditionally follow an AL
# Needed in rules where stand-alone $CM s are treated as AL.
# Chaining is disabled with CM because it causes other failures,
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
$AL_FOLLOW_CM = [$CL $CP $EX $HL $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
#
# Rule LB 4, 5 Mandatory (Hard) breaks.
#
$LB4Breaks = [$BK $CR $LF $NL];
$LB4NonBreaks = [^$BK $CR $LF $NL];
$CR $LF {100};
#
# LB 6 Do not break before hard line breaks.
#
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
$CAN_CM $CM* $LB4Breaks {100};
$CM+ $LB4Breaks {100};
# LB 7 x SP
# x ZW
$LB4NonBreaks [$SP $ZW];
$CAN_CM $CM* [$SP $ZW];
$CM+ [$SP $ZW];
#
# LB 8 Break after zero width space
# TODO: ZW SP* <break>
# An engine change is required to write the reverse rule for this.
# For now, leave the Unicode 5.2 rule, ZW <break>
#
$LB8Breaks = [$LB4Breaks $ZW];
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
# $CM not covered by the above needs to behave like $AL
# See definition of $CAN_CM.
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
$CM+;
#
# LB 11 Do not break before or after WORD JOINER & related characters.
#
$CAN_CM $CM* $WJcm;
$LB8NonBreaks $WJcm;
$CM+ $WJcm;
$WJcm $CANT_CM;
$WJcm $CAN_CM $CM*;
#
# LB 12 Do not break after NBSP and related characters.
# GL x
#
$GLcm $CAN_CM $CM*;
$GLcm $CANT_CM;
#
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm;
$CM+ GLcm;
#
# LB 13 Don't break before ']' or '!' or ';' or '/', even after spaces.
#
$LB8NonBreaks $CL;
$CAN_CM $CM* $CL;
$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $CP;
$CAN_CM $CM* $CP;
$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $EX;
$CAN_CM $CM* $EX;
$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $IS;
$CAN_CM $CM* $IS;
$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
$LB8NonBreaks $SY;
$CAN_CM $CM* $SY;
$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
#
# LB 14 Do not break after OP, even after spaces
#
$OPcm $SP* $CAN_CM $CM*;
$OPcm $SP* $CANT_CM;
$OPcm $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
# LB 15
$QUcm $SP* $OPcm;
# LB 16
($CLcm | $CPcm) $SP* $NScm;
# LB 17
$B2cm $SP* $B2cm;
#
# LB 18 Break after spaces.
#
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
$LB18Breaks = [$LB8Breaks $SP];
# LB 19
# x QU
$LB18NonBreaks $CM* $QUcm;
$CM+ $QUcm;
# QU x
$QUcm .?;
$QUcm $LB18NonBreaks $CM*; # Don't let a combining mark go onto $CR, $BK, etc.
# TODO: I don't think this rule is needed.
# LB 20
# <break> $CB
# $CB <break>
$LB20NonBreaks = [$LB18NonBreaks - $CB];
# LB 20.09 added rule for Finnish tailoring
# LB 21 x (BA | HY | NS)
# BB x
#
$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm) / $AL;
$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm);
($HY | $HH) $AL;
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
# LB 21a Don't break after Hebrew + Hyphen
# HL (HY | BA) x
#
$HLcm ($HYcm | $BAcm | $HHcm) [^$CB]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)
$SYcm $HLcm;
# LB 22
($ALcm | $HLcm) $INcm;
$CM+ $INcm; # by rule 10, any otherwise unattached CM behaves as AL
$IDcm $INcm;
$INcm $INcm;
$NUcm $INcm;
# $LB 23
$IDcm $POcm;
$ALcm $NUcm; # includes $LB19
$HLcm $NUcm;
$CM+ $NUcm; # Rule 10, any otherwise unattached CM behaves as AL
$NUcm $ALcm;
$NUcm $HLcm;
#
# LB 24
#
$PRcm $IDcm;
$PRcm ($ALcm | $HLcm);
$POcm ($ALcm | $HLcm);
#
# LB 25 Numbers.
#
($PRcm | $POcm)? ($OPcm | $HYcm)? $NUcm ($NUcm | $SYcm | $IScm)* ($CLcm | $CPcm)? ($PRcm | $POcm)?;
# LB 26 Do not break a Korean syllable
#
$JLcm ($JLcm | $JVcm | $H2cm | $H3cm);
($JVcm | $H2cm) ($JVcm | $JTcm);
($JTcm | $H3cm) $JTcm;
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $INcm;
($JLcm | $JVcm | $JTcm | $H2cm | $H3cm) $POcm;
$PRcm ($JLcm | $JVcm | $JTcm | $H2cm | $H3cm);
# LB 28 Do not break between alphabetics
#
($ALcm | $HLcm) ($ALcm | $HLcm);
$CM+ ($ALcm | $HLcm); # The $CM+ is from rule 10, an unattached CM is treated as AL
# LB 29
$IScm ($ALcm | $HLcm);
# LB 30
($ALcm | $HLcm | $NUcm) $OPcm;
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $HLcm | $NUcm);
# LB 30a Do not break between regional indicators.
$RIcm $RIcm;
#
# Reverse Rules.
#
## -------------------------------------------------
!!reverse;
$CM+ $ALPlus;
$CM+ $BA;
$CM+ $HH;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
$CM+ $CP;
$CM+ $EX;
$CM+ $GL;
$CM+ $HL;
$CM+ $HY;
$CM+ $H2;
$CM+ $H3;
$CM+ $ID;
$CM+ $IN;
$CM+ $IS;
$CM+ $JL;
$CM+ $JV;
$CM+ $JT;
$CM+ $NS;
$CM+ $NU;
$CM+ $OP;
$CM+ $PO;
$CM+ $PR;
$CM+ $QU;
$CM+ $RI;
$CM+ $SY;
$CM+ $WJ;
$CM+;
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] [whatever]
# The CM needs to behave as an AL
#
$AL_FOLLOW $CM+ / (
[$BK $CR $LF $NL $ZW {eof}] |
$SP+ $CM+ $SP |
$SP+ $CM* ([^$OP $CM $SP] | [$AL {eof}])); # if LB 14 will match, need to surpress this break.
# LB14 says OP SP* x .
# becomes OP SP* x AL
# becomes OP SP* x CM+ AL_FOLLOW
#
# Further note: the $AL in [$AL {eof}] is only to work around
# a rule compiler bug which complains about
# empty sets otherwise.
#
# Sequences of the form (shown forwards)
# [CANT_CM] <break> [CM] <break> [PR]
# The CM needs to behave as an AL
# This rule is concerned about getting the second of the two <breaks> in place.
#
[$PR ] / $CM+ [$BK $CR $LF $NL $ZW $SP {eof}];
# LB 4, 5, 5
$LB4Breaks [$LB4NonBreaks-$CM];
$LB4Breaks $CM+ $CAN_CM;
$LF $CR;
# LB 7 x SP
# x ZW
[$SP $ZW] [$LB4NonBreaks-$CM];
[$SP $ZW] $CM+ $CAN_CM;
# LB 8 ZW SP* <break>
# TODO: to implement this, we need more than one look-ahead hard break in play at a time.
# Requires an engine enhancement.
# / $SP* $ZW
# LB 9,10 Combining marks.
# X $CM needs to behave like X, where X is not $SP or controls.
# $CM not covered by the above needs to behave like $AL
# Stick together any combining sequences that don't match other rules.
$CM+ $CAN_CM;
# LB 11
$CM* $WJ $CM* $CAN_CM;
$CM* $WJ [$LB8NonBreaks-$CM];
$CANT_CM $CM* $WJ;
$CM* $CAN_CM $CM* $WJ;
# LB 12a
# [^SP BA HY] x GL
#
$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
# LB 12
# GL x
#
$CANT_CM $CM* $GL;
$CM* $CAN_CM $CM* $GL;
# LB 13
$CL $CM+ $CAN_CM;
$CP $CM+ $CAN_CM;
$EX $CM+ $CAN_CM;
$IS $CM+ $CAN_CM;
$SY $CM+ $CAN_CM;
$CL [$LB8NonBreaks-$CM];
$CP [$LB8NonBreaks-$CM];
$EX [$LB8NonBreaks-$CM];
$IS [$LB8NonBreaks-$CM];
$SY [$LB8NonBreaks-$CM];
# Rule 13 & 14 taken together for an edge case.
# Match this, shown forward
# OP SP+ ($CM+ behaving as $AL) (CL | CP | EX | IS | IY)
# This really wants to chain at the $CM+ (which is acting as an $AL)
# except for $CM chaining being disabled.
[$CL $CP $EX $IS $SY] $CM+ $SP+ $CM* $OP;
# LB 14 OP SP* x
#
$CM* $CAN_CM $SP* $CM* $OP;
$CANT_CM $SP* $CM* $OP;
$AL_FOLLOW? $CM+ $SP $SP* $CM* $OP; # by LB 10, behaves like $AL_FOLLOW? $AL $SP* $CM* $OP
$AL_FOLLOW_NOCM $CM+ $SP+ $CM* $OP;
$CM* $AL_FOLLOW_CM $CM+ $SP+ $CM* $OP;
$SY $CM $SP+ $OP; # TODO: Experiment. Remove.
# LB 15
$CM* $OP $SP* $CM* $QU;
# LB 16
$CM* $NS $SP* $CM* ($CL | $CP);
# LB 17
$CM* $B2 $SP* $CM* $B2;
# LB 18 break after spaces
# Nothing explicit needed here.
#
# LB 19
#
$CM* $QU $CM* $CAN_CM; # . x QU
$CM* $QU $LB18NonBreaks;
$CM* $CAN_CM $CM* $QU; # QU x .
$CANT_CM $CM* $QU;
#
# LB 20 Break before and after CB.
# nothing needed here.
#
# LB 20.09 added rule for Finnish tailoring
$AL ($HY | $HH) / $SP;
# LB 21
$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
# LB21a
[^$CB] $CM* ($HY | $BA | $HH) $CM* $HL;
# LB21b (reverse)
$CM* $HL $CM* $SY;
# LB 22
$CM* $IN $CM* ($ALPlus | $HL);
$CM* $IN $CM* $ID;
$CM* $IN $CM* $IN;
$CM* $IN $CM* $NU;
# LB 23
$CM* $PO $CM* $ID;
$CM* $NU $CM* ($ALPlus | $HL);
$CM* ($ALPlus | $HL) $CM* $NU;
# LB 24
$CM* $ID $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PR;
$CM* ($ALPlus | $HL) $CM* $PO;
# LB 25
($CM* ($PR | $PO))? ($CM* ($CL | $CP))? ($CM* ($NU | $IS | $SY))* $CM* $NU ($CM* ($OP | $HY))? ($CM* ($PR | $PO))?;
# LB 26
$CM* ($H3 | $H2 | $JV | $JL) $CM* $JL;
$CM* ($JT | $JV) $CM* ($H2 | $JV);
$CM* $JT $CM* ($H3 | $JT);
# LB 27
$CM* $IN $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* $PO $CM* ($H3 | $H2 | $JT | $JV | $JL);
$CM* ($H3 | $H2 | $JT | $JV | $JL) $CM* $PR;
# LB 28
$CM* ($ALPlus | $HL) $CM* ($ALPlus | $HL);
# LB 29
$CM* ($ALPlus | $HL) $CM* $IS;
# LB 30
$CM* $OP $CM* ($ALPlus | $HL | $NU);
$CM* ($ALPlus | $HL | $NU) $CM* $CP;
# LB 30a
$CM* $RI $CM* $RI;
## -------------------------------------------------
!!safe_reverse;
# LB 9
$CM+ [^$CM $BK $CR $LF $NL $ZW $SP];
$CM+ $SP / .;
# LB 14
$SP+ $CM* $OP;
# LB 15
$SP+ $CM* $QU;
# LB 16
$SP+ $CM* ($CL | $CP);
# LB 17
$SP+ $CM* $B2;
# LB 21
$CM* ($HY | $BA | $HH) $CM* $HL;
# LB 25
($CM* ($IS | $SY))+ $CM* $NU;
($CL | $CP) $CM* ($NU | $IS | $SY);
# For dictionary-based break
$dictionary $dictionary;
## -------------------------------------------------
!!safe_forward;
# Skip forward over all character classes that are involved in
# rules containing patterns with possibly more than one char
# of context.
#
# It might be slightly more efficient to have specific rules
# instead of one generic one, but only if we could
# turn off rule chaining. We don't want to move more
# than necessary.
#
[$CM $OP $QU $CL $CP $B2 $PR $HY $BA $SP $dictionary]+ [^$CM $OP $QU $CL $CP $B2 $PR $HY $BA $dictionary];
$dictionary $dictionary;

5
icu4c/source/data/brkitr/root.txt
View file

@ -1,6 +1,6 @@
// ***************************************************************************
// *
// * Copyright (C) 2014 International Business Machines
// * Copyright (C) 2015 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: org.unicode.cldr.icu.NewLdml2IcuConverter
// * Source File: <path>/common/segments/root.xml ../../xml/brkitr/root.xml
@ -11,6 +11,9 @@ root{
boundaries{
grapheme:process(dependency){"char.brk"}
line:process(dependency){"line.brk"}
line_loose:process(dependency){"line_loose.brk"}
line_normal:process(dependency){"line_normal.brk"}
line_strict:process(dependency){"line.brk"}
sentence:process(dependency){"sent.brk"}
title:process(dependency){"title.brk"}
word:process(dependency){"word.brk"}

17
icu4c/source/data/brkitr/zh.txt Normal file
View file

@ -0,0 +1,17 @@
// ***************************************************************************
// *
// * Copyright (C) 2015 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: org.unicode.cldr.icu.NewLdml2IcuConverter
// * Source File: <path>/common/segments/zh.xml ../../xml/brkitr/zh.xml
// *
// ***************************************************************************
zh{
Version{"2.0.82.42"}
boundaries{
line:process(dependency){"line.brk"}
line_loose:process(dependency){"line_loose_cj.brk"}
line_normal:process(dependency){"line_normal_cj.brk"}
line_strict:process(dependency){"line.brk"}
}
}

17
icu4c/source/data/brkitr/zh_Hant.txt Normal file
View file

@ -0,0 +1,17 @@
// ***************************************************************************
// *
// * Copyright (C) 2015 International Business Machines
// * Corporation and others. All Rights Reserved.
// * Tool: org.unicode.cldr.icu.NewLdml2IcuConverter
// * Source File: <path>/common/segments/zh_Hant.xml ../../xml/brkitr/zh_Hant.xml
// *
// ***************************************************************************
zh_Hant{
Version{"2.0.82.42"}
boundaries{
line:process(dependency){"line.brk"}
line_loose:process(dependency){"line_loose_cj.brk"}
line_normal:process(dependency){"line_normal_cj.brk"}
line_strict:process(dependency){"line.brk"}
}
}

5
icu4c/source/data/xml/brkitr/fi.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2010 International Business Machines Corporation and others. All rights reserved.
Copyright (c) 2010-2015 International Business Machines Corporation and others. All rights reserved.
-->
<!DOCTYPE ldml SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldml.dtd"
[
@ -18,6 +18,9 @@
<icu:breakIteratorData>
<icu:boundaries>
<icu:line icu:dependency="line_fi.brk"/>
<icu:line icu:alt="loose" icu:dependency="line_loose_fi.brk"/>
<icu:line icu:alt="normal" icu:dependency="line_normal_fi.brk"/>
<icu:line icu:alt="strict" icu:dependency="line_fi.brk"/>
</icu:boundaries>
</icu:breakIteratorData>
</special>

7
icu4c/source/data/xml/brkitr/ja.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2006-2012 International Business Machines Corporation and others. All rights reserved.
Copyright (c) 2006-2015 International Business Machines Corporation and others. All rights reserved.
-->
<!DOCTYPE ldml SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldml.dtd"
[
@ -17,7 +17,10 @@
<special xmlns:icu="http://www.icu-project.org/">
<icu:breakIteratorData>
<icu:boundaries>
<icu:line icu:dependency="line_ja.brk"/>
<icu:line icu:dependency="line_normal.brk"/>
<icu:line icu:alt="loose" icu:dependency="line_loose_cj.brk"/>
<icu:line icu:alt="normal" icu:dependency="line_normal_cj.brk"/>
<icu:line icu:alt="strict" icu:dependency="line.brk"/>
</icu:boundaries>
</icu:breakIteratorData>
</special>

5
icu4c/source/data/xml/brkitr/root.xml
View file

@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2010-2014 International Business Machines Corporation and others. All rights reserved.
Copyright (c) 2010-2015 International Business Machines Corporation and others. All rights reserved.
-->
<!DOCTYPE ldml SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldml.dtd"
[
@ -20,6 +20,9 @@
<icu:grapheme icu:dependency="char.brk"/>
<icu:word icu:dependency="word.brk"/>
<icu:line icu:dependency="line.brk"/>
<icu:line icu:alt="loose" icu:dependency="line_loose.brk"/>
<icu:line icu:alt="normal" icu:dependency="line_normal.brk"/>
<icu:line icu:alt="strict" icu:dependency="line.brk"/>
<icu:sentence icu:dependency="sent.brk"/>
<icu:title icu:dependency="title.brk"/>
</icu:boundaries>

28
icu4c/source/data/xml/brkitr/zh.xml Normal file
View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2006-2015 International Business Machines Corporation and others. All rights reserved.
-->
<!DOCTYPE ldml SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldml.dtd"
[
<!ENTITY % icu SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldmlICU.dtd">
%icu;
]
>
<ldml>
<identity>
<version number="$ $"/>
<generation date="$ $"/>
<language type="zh"/>
</identity>
<special xmlns:icu="http://www.icu-project.org/">
<icu:breakIteratorData>
<icu:boundaries>
<icu:line icu:dependency="line.brk"/>
<icu:line icu:alt="loose" icu:dependency="line_loose_cj.brk"/>
<icu:line icu:alt="normal" icu:dependency="line_normal_cj.brk"/>
<icu:line icu:alt="strict" icu:dependency="line.brk"/>
</icu:boundaries>
</icu:breakIteratorData>
</special>
</ldml>

28
icu4c/source/data/xml/brkitr/zh_Hant.xml Normal file
View file

@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Copyright (c) 2006-2015 International Business Machines Corporation and others. All rights reserved.
-->
<!DOCTYPE ldml SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldml.dtd"
[
<!ENTITY % icu SYSTEM "http://www.unicode.org/repos/cldr/trunk/common/dtd/ldmlICU.dtd">
%icu;
]
>
<ldml>
<identity>
<version number="$ $"/>
<generation date="$ $"/>
<language type="zh_Hant"/>
</identity>
<special xmlns:icu="http://www.icu-project.org/">
<icu:breakIteratorData>
<icu:boundaries>
<icu:line icu:dependency="line.brk"/>
<icu:line icu:alt="loose" icu:dependency="line_loose_cj.brk"/>
<icu:line icu:alt="normal" icu:dependency="line_normal_cj.brk"/>
<icu:line icu:alt="strict" icu:dependency="line.brk"/>
</icu:boundaries>
</icu:breakIteratorData>
</special>
</ldml>