From a2a9c893b2e64514368498eb468b34c0a7cbb3f0 Mon Sep 17 00:00:00 2001 From: Peter Edberg Date: Tue, 21 Apr 2009 22:42:37 +0000 Subject: [PATCH] ICU-6853 Update ICU tailored break iterators to match CLDR 1.7 X-SVN-Rev: 25856 --- .gitattributes | 4 + icu4c/source/data/brkitr/brkfiles.mk | 4 +- icu4c/source/data/brkitr/char_th.txt | 68 ++++++++++++++ icu4c/source/data/brkitr/el.txt | 14 +++ icu4c/source/data/brkitr/sent_el.txt | 119 ++++++++++++++++++++++++ icu4c/source/data/brkitr/th.txt | 14 +++ icu4c/source/data/brkitr/word_POSIX.txt | 6 +- icu4c/source/data/brkitr/word_ja.txt | 2 +- 8 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 icu4c/source/data/brkitr/char_th.txt create mode 100644 icu4c/source/data/brkitr/el.txt create mode 100644 icu4c/source/data/brkitr/sent_el.txt create mode 100644 icu4c/source/data/brkitr/th.txt diff --git a/.gitattributes b/.gitattributes index 3c1f058d315..4e9b47fab0b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -55,6 +55,10 @@ icu4c/source/common/unicode/dtintrv.h -text icu4c/source/config/gmakever.mk -text icu4c/source/config/pkgdataMakefile.in -text icu4c/source/configure.mk -text +icu4c/source/data/brkitr/char_th.txt -text +icu4c/source/data/brkitr/el.txt -text +icu4c/source/data/brkitr/sent_el.txt -text +icu4c/source/data/brkitr/th.txt -text icu4c/source/data/coll/af.txt -text icu4c/source/data/coll/af_NA.txt -text icu4c/source/data/coll/af_ZA.txt -text diff --git a/icu4c/source/data/brkitr/brkfiles.mk b/icu4c/source/data/brkitr/brkfiles.mk index 63f33df6b1e..3e8fb31d9bb 100644 --- a/icu4c/source/data/brkitr/brkfiles.mk +++ b/icu4c/source/data/brkitr/brkfiles.mk @@ -38,9 +38,9 @@ BRK_CTD_SOURCE = thaidict.txt # List of break iterator files (brk). -BRK_SOURCE = word_POSIX.txt word_ja.txt char.txt word.txt line.txt sent.txt title.txt +BRK_SOURCE = word_POSIX.txt word_ja.txt sent_el.txt char_th.txt char.txt word.txt line.txt sent.txt title.txt # Ordinary resources -BRK_RES_SOURCE = en.txt en_US.txt en_US_POSIX.txt ja.txt +BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt ja.txt th.txt diff --git a/icu4c/source/data/brkitr/char_th.txt b/icu4c/source/data/brkitr/char_th.txt new file mode 100644 index 00000000000..e0135dd42a3 --- /dev/null +++ b/icu4c/source/data/brkitr/char_th.txt @@ -0,0 +1,68 @@ +# +# Copyright (C) 2002-2009, International Business Machines Corporation and others. +# All Rights Reserved. +# +# file: char_th.txt +# +# ICU Character Break Rules, also known as Grapheme Cluster Boundaries +# See Unicode Standard Annex #29. +# These rules are based on TR29 Revision 13, for Unicode Version 5.1 +# + +# +# Character Class Definitions. +# +$CR = [\p{Grapheme_Cluster_Break = CR}]; +$LF = [\p{Grapheme_Cluster_Break = LF}]; +$Control = [\p{Grapheme_Cluster_Break = Control}]; +$Extend = [\p{Grapheme_Cluster_Break = Extend} - [\u0E30 \u0E32 \u0E45 \u0EB0 \u0EB2]]; +$SpacingMark = [\p{Grapheme_Cluster_Break = SpacingMark}]; + +# +# Korean Syllable Definitions +# +$L = [\p{Grapheme_Cluster_Break = L}]; +$V = [\p{Grapheme_Cluster_Break = V}]; +$T = [\p{Grapheme_Cluster_Break = T}]; + +$LV = [\p{Grapheme_Cluster_Break = LV}]; +$LVT = [\p{Grapheme_Cluster_Break = LVT}]; + + +## ------------------------------------------------- +!!chain; + +!!forward; + +$CR $LF; + +$L ($L | $V | $LV | $LVT); +($LV | $V) ($V | $T); +($LVT | $T) $T; + +[^$Control $CR $LF] $Extend; + +[^$Control $CR $LF] $SpacingMark; + + +## ------------------------------------------------- + +!!reverse; +$LF $CR; +($L | $V | $LV | $LVT) $L; +($V | $T) ($LV | $V); +$T ($LVT | $T); + +$Extend [^$Control $CR $LF]; +$SpacingMark [^$Control $CR $LF]; + + +## ------------------------------------------------- + +!!safe_reverse; + + +## ------------------------------------------------- + +!!safe_forward; + diff --git a/icu4c/source/data/brkitr/el.txt b/icu4c/source/data/brkitr/el.txt new file mode 100644 index 00000000000..bf024b4363e --- /dev/null +++ b/icu4c/source/data/brkitr/el.txt @@ -0,0 +1,14 @@ +// *************************************************************************** +// * +// * Copyright (C) 2009 International Business Machines +// * Corporation and others. All Rights Reserved. +// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java +// * Source File:/xml/brkitr/el.xml +// * +// *************************************************************************** +el{ + Version{"1.1"} + boundaries{ + sentence:process(dependency){"sent_el.brk"} + } +} diff --git a/icu4c/source/data/brkitr/sent_el.txt b/icu4c/source/data/brkitr/sent_el.txt new file mode 100644 index 00000000000..3b92f8452c0 --- /dev/null +++ b/icu4c/source/data/brkitr/sent_el.txt @@ -0,0 +1,119 @@ +# +# Copyright (C) 2002-2009, International Business Machines Corporation and others. +# All Rights Reserved. +# +# file: sent_el.txt +# +# ICU Sentence Break Rules +# See Unicode Standard Annex #29. +# These rules are based on UAX 29 Revision 13 for Unicode Version 5.1.0 +# + + +# +# Character categories as defined in TR 29 +# +$CR = [\p{Sentence_Break = CR}]; +$LF = [\p{Sentence_Break = LF}]; +$Extend = [\p{Sentence_Break = Extend}]; +$Sep = [\p{Sentence_Break = Sep}]; +$Format = [\p{Sentence_Break = Format}]; +$Sp = [\p{Sentence_Break = Sp}]; +$Lower = [\p{Sentence_Break = Lower}]; +$Upper = [\p{Sentence_Break = Upper}]; +$OLetter = [\p{Sentence_Break = OLetter}]; +$Numeric = [\p{Sentence_Break = Numeric}]; +$ATerm = [\p{Sentence_Break = ATerm}]; +$SContinue = [\p{Sentence_Break = SContinue}]; +$STerm = [\p{Sentence_Break = STerm} [\u003B \u037E]]; +$Close = [\p{Sentence_Break = Close}]; + +# +# Define extended forms of the character classes, +# incorporate trailing Extend or Format chars. +# Rules 4 and 5. + +$SpEx = $Sp ($Extend | $Format)*; +$LowerEx = $Lower ($Extend | $Format)*; +$UpperEx = $Upper ($Extend | $Format)*; +$OLetterEx = $OLetter ($Extend | $Format)*; +$NumericEx = $Numeric ($Extend | $Format)*; +$ATermEx = $ATerm ($Extend | $Format)*; +$SContinueEx= $SContinue ($Extend | $Format)*; +$STermEx = $STerm ($Extend | $Format)*; +$CloseEx = $Close ($Extend | $Format)*; + + +## ------------------------------------------------- + +!!chain; +!!forward; + +# Rule 3 - break after separators. Keep CR/LF together. +# +$CR $LF; + + +# Rule 4 - Break after $Sep. +# Rule 5 - Ignore $Format and $Extend +# +[^$Sep $CR $LF]? ($Extend | $Format)*; + + +# Rule 6 +$ATermEx $NumericEx; + +# Rule 7 +$UpperEx $ATermEx $UpperEx; + +#Rule 8 +$NotLettersEx = [^$OLetter $Upper $Lower $Sep $CR $LF $ATerm $STerm] ($Extend | $Format)*; +$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower; + +# Rule 8a +($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx); + +#Rule 9, 10, 11 +($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?; + +#Rule 12 +[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .; +[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep $LF $CR {eof}] | $CR $LF){100}; + +## ------------------------------------------------- + +!!reverse; + +$SpEx_R = ($Extend | $Format)* $Sp; +$ATermEx_R = ($Extend | $Format)* $ATerm; +$STermEx_R = ($Extend | $Format)* $STerm; +$CloseEx_R = ($Extend | $Format)* $Close; + +# +# Reverse rules. +# For now, use the old style inexact reverse rules, which are easier +# to write, but less efficient. +# TODO: exact reverse rules. It appears that exact reverse rules +# may require improving support for look-ahead breaks in the +# builder. Needs more investigation. +# + +[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*; +#.*; + +# Explanation for this rule: +# +# It needs to back over +# The $Sep at which we probably begin +# All of the non $Sep chars leading to the preceding $Sep +# The preceding $Sep, which will be the second one that the rule matches. +# Any immediately preceding STerm or ATerm sequences. We need to see these +# to get the correct rule status when moving forwards again. +# +# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match +# the entire string. +# +# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be +# at the beginning of the string at this point, and we don't want to fail. +# Can only use {eof} once, and it is used later. +# diff --git a/icu4c/source/data/brkitr/th.txt b/icu4c/source/data/brkitr/th.txt new file mode 100644 index 00000000000..b8013fd819d --- /dev/null +++ b/icu4c/source/data/brkitr/th.txt @@ -0,0 +1,14 @@ +// *************************************************************************** +// * +// * Copyright (C) 2009 International Business Machines +// * Corporation and others. All Rights Reserved. +// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java +// * Source File:/xml/brkitr/th.xml +// * +// *************************************************************************** +th{ + Version{"1.1"} + boundaries{ + grapheme:process(dependency){"char_th.brk"} + } +} diff --git a/icu4c/source/data/brkitr/word_POSIX.txt b/icu4c/source/data/brkitr/word_POSIX.txt index 315bdb254ff..78f36824370 100644 --- a/icu4c/source/data/brkitr/word_POSIX.txt +++ b/icu4c/source/data/brkitr/word_POSIX.txt @@ -31,9 +31,9 @@ $Extend = [\p{Word_Break = Extend}]; $Format = [\p{Word_Break = Format}]; $Katakana = [\p{Word_Break = Katakana}]; $ALetter = [\p{Word_Break = ALetter}]; -$MidNumLet = [\p{Word_Break = MidNumLet}]; -$MidLetter = [\p{Word_Break = MidLetter} - [ \:]]; -$MidNum = [\p{Word_Break = MidNum}]; +$MidNumLet = [\p{Word_Break = MidNumLet} - [.]]; +$MidLetter = [\p{Word_Break = MidLetter} - [\:]]; +$MidNum = [\p{Word_Break = MidNum} [.]]; $Numeric = [\p{Word_Break = Numeric}]; $ExtendNumLet = [\p{Word_Break = ExtendNumLet}]; diff --git a/icu4c/source/data/brkitr/word_ja.txt b/icu4c/source/data/brkitr/word_ja.txt index 696bd34caca..05d9b36f7b0 100644 --- a/icu4c/source/data/brkitr/word_ja.txt +++ b/icu4c/source/data/brkitr/word_ja.txt @@ -61,7 +61,7 @@ $NumericEx = $Numeric ($Extend | $Format)*; $ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*; $Hiragana = [\p{script=Hiragana}]; -$Ideographic = [\p{Ideographic}]; +$Ideographic = [\p{Ideographic} [\u3005 \u3007 \u303B]]; $HiraganaEx = $Hiragana ($Extend | $Format)*; $IdeographicEx = $Ideographic ($Extend | $Format)*;