ICU-6853 Update ICU tailored break iterators to match CLDR 1.7

X-SVN-Rev: 25856

.gitattributes

@@ -55,6 +55,10 @@ icu4c/source/common/unicode/dtintrv.h -text
 icu4c/source/config/gmakever.mk -text
 icu4c/source/config/pkgdataMakefile.in -text
 icu4c/source/configure.mk -text
+icu4c/source/data/brkitr/char_th.txt -text
+icu4c/source/data/brkitr/el.txt -text
+icu4c/source/data/brkitr/sent_el.txt -text
+icu4c/source/data/brkitr/th.txt -text
 icu4c/source/data/coll/af.txt -text
 icu4c/source/data/coll/af_NA.txt -text
 icu4c/source/data/coll/af_ZA.txt -text

icu4c/source/data/brkitr/brkfiles.mk

@@ -38,9 +38,9 @@ BRK_CTD_SOURCE = thaidict.txt
 
 
 # List of break iterator files (brk).
-BRK_SOURCE = word_POSIX.txt word_ja.txt char.txt word.txt line.txt sent.txt title.txt 
+BRK_SOURCE = word_POSIX.txt word_ja.txt sent_el.txt char_th.txt char.txt word.txt line.txt sent.txt title.txt 
 
 
 # Ordinary resources
-BRK_RES_SOURCE = en.txt en_US.txt en_US_POSIX.txt ja.txt
+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt ja.txt th.txt
 

icu4c/source/data/brkitr/char_th.txt

@@ -0,0 +1,68 @@
+#
+#   Copyright (C) 2002-2009, International Business Machines Corporation and others.
+#       All Rights Reserved.
+#
+#   file:  char_th.txt 
+#
+#   ICU Character Break Rules, also known as Grapheme Cluster Boundaries
+#      See Unicode Standard Annex #29.
+#      These rules are based on TR29 Revision 13, for Unicode Version 5.1
+#
+
+#
+#  Character Class Definitions.
+#
+$CR          = [\p{Grapheme_Cluster_Break = CR}];
+$LF          = [\p{Grapheme_Cluster_Break = LF}];
+$Control     = [\p{Grapheme_Cluster_Break = Control}];
+$Extend      = [\p{Grapheme_Cluster_Break = Extend} - [\u0E30 \u0E32 \u0E45 \u0EB0 \u0EB2]];
+$SpacingMark = [\p{Grapheme_Cluster_Break = SpacingMark}];
+
+#
+# Korean Syllable Definitions
+#
+$L       = [\p{Grapheme_Cluster_Break = L}];
+$V       = [\p{Grapheme_Cluster_Break = V}];
+$T       = [\p{Grapheme_Cluster_Break = T}];
+
+$LV      = [\p{Grapheme_Cluster_Break = LV}];
+$LVT     = [\p{Grapheme_Cluster_Break = LVT}];
+
+
+## -------------------------------------------------
+!!chain;
+
+!!forward;
+
+$CR $LF;
+
+$L ($L | $V | $LV | $LVT);
+($LV | $V) ($V | $T);
+($LVT | $T) $T;
+
+[^$Control $CR $LF] $Extend;
+
+[^$Control $CR $LF] $SpacingMark;
+
+
+## -------------------------------------------------
+
+!!reverse;
+$LF $CR;
+($L | $V | $LV | $LVT) $L;
+($V | $T) ($LV | $V);
+$T ($LVT | $T);
+
+$Extend      [^$Control $CR $LF];
+$SpacingMark [^$Control $CR $LF];
+
+
+## -------------------------------------------------
+
+!!safe_reverse;
+
+
+## -------------------------------------------------
+
+!!safe_forward;
+

icu4c/source/data/brkitr/el.txt

@@ -0,0 +1,14 @@
+// ***************************************************************************
+// *
+// * Copyright (C) 2009 International Business Machines
+// * Corporation and others.  All Rights Reserved.
+// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
+// * Source File:<path>/xml/brkitr/el.xml
+// *
+// ***************************************************************************
+el{
+    Version{"1.1"}
+    boundaries{
+        sentence:process(dependency){"sent_el.brk"}
+    }
+}

icu4c/source/data/brkitr/sent_el.txt

@@ -0,0 +1,119 @@
+#
+#   Copyright (C) 2002-2009, International Business Machines Corporation and others.
+#       All Rights Reserved.
+#
+#   file:  sent_el.txt
+#
+#   ICU Sentence Break Rules
+#      See Unicode Standard Annex #29.
+#      These rules are based on UAX 29 Revision 13 for Unicode Version 5.1.0
+#
+
+
+#
+# Character categories as defined in TR 29
+#
+$CR        = [\p{Sentence_Break = CR}];
+$LF        = [\p{Sentence_Break = LF}];
+$Extend    = [\p{Sentence_Break = Extend}];
+$Sep       = [\p{Sentence_Break = Sep}];
+$Format    = [\p{Sentence_Break = Format}];
+$Sp        = [\p{Sentence_Break = Sp}];
+$Lower     = [\p{Sentence_Break = Lower}];
+$Upper     = [\p{Sentence_Break = Upper}];
+$OLetter   = [\p{Sentence_Break = OLetter}];
+$Numeric   = [\p{Sentence_Break = Numeric}];
+$ATerm     = [\p{Sentence_Break = ATerm}];
+$SContinue = [\p{Sentence_Break = SContinue}];
+$STerm     = [\p{Sentence_Break = STerm} [\u003B \u037E]];
+$Close     = [\p{Sentence_Break = Close}];
+
+#
+# Define extended forms of the character classes,
+#   incorporate trailing Extend or Format chars.
+#   Rules 4 and 5.  
+
+$SpEx       = $Sp      ($Extend | $Format)*;
+$LowerEx    = $Lower   ($Extend | $Format)*;
+$UpperEx    = $Upper   ($Extend | $Format)*;
+$OLetterEx  = $OLetter ($Extend | $Format)*;
+$NumericEx  = $Numeric ($Extend | $Format)*;
+$ATermEx    = $ATerm   ($Extend | $Format)*;
+$SContinueEx= $SContinue ($Extend | $Format)*;
+$STermEx    = $STerm   ($Extend | $Format)*;
+$CloseEx    = $Close   ($Extend | $Format)*;
+
+
+## -------------------------------------------------
+
+!!chain;
+!!forward;
+
+# Rule 3 - break after separators.  Keep CR/LF together.
+#
+$CR $LF;
+
+
+# Rule 4 - Break after $Sep.
+# Rule 5 - Ignore $Format and $Extend
+#
+[^$Sep $CR $LF]? ($Extend | $Format)*;
+
+
+# Rule 6
+$ATermEx $NumericEx;
+
+# Rule 7
+$UpperEx $ATermEx $UpperEx;
+
+#Rule 8
+$NotLettersEx = [^$OLetter $Upper $Lower $Sep $CR $LF $ATerm $STerm] ($Extend | $Format)*;
+$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
+
+# Rule 8a
+($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx);
+
+#Rule 9, 10, 11
+($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?;
+
+#Rule 12
+[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .;
+[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep $LF $CR {eof}] | $CR $LF){100};
+
+## -------------------------------------------------
+
+!!reverse;
+
+$SpEx_R       = ($Extend | $Format)* $Sp;
+$ATermEx_R    = ($Extend | $Format)* $ATerm;
+$STermEx_R    = ($Extend | $Format)* $STerm;
+$CloseEx_R    = ($Extend | $Format)* $Close;
+
+#
+#  Reverse rules.
+#     For now, use the old style inexact reverse rules, which are easier
+#     to write, but less efficient.
+#     TODO:  exact reverse rules.  It appears that exact reverse rules
+#            may require improving support for look-ahead breaks in the
+#            builder.  Needs more investigation.
+#
+
+[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
+#.*;
+
+# Explanation for this rule:
+#
+#    It needs to back over
+#        The $Sep at which we probably begin
+#        All of the non $Sep chars leading to the preceding $Sep
+#        The preceding $Sep, which will be the second one that the rule matches.
+#        Any immediately preceding STerm or ATerm sequences.  We need to see these
+#              to get the correct rule status when moving forwards again.
+#        
+# [{bof}]           inhibit rule chaining.  Without this, rule would loop on itself and match
+#                   the entire string.
+#
+# (.? | $LF $CR)    Match one $Sep instance.  Use .? rather than $Sep because position might be
+#                   at the beginning of the string at this point, and we don't want to fail.
+#                   Can only use {eof} once, and it is used later.
+#

icu4c/source/data/brkitr/th.txt

@@ -0,0 +1,14 @@
+// ***************************************************************************
+// *
+// * Copyright (C) 2009 International Business Machines
+// * Corporation and others.  All Rights Reserved.
+// * Tool: com.ibm.icu.dev.tool.cldr.LDML2ICUConverter.java
+// * Source File:<path>/xml/brkitr/th.xml
+// *
+// ***************************************************************************
+th{
+    Version{"1.1"}
+    boundaries{
+        grapheme:process(dependency){"char_th.brk"}
+    }
+}

icu4c/source/data/brkitr/word_POSIX.txt

@@ -31,9 +31,9 @@ $Extend       = [\p{Word_Break = Extend}];
 $Format       = [\p{Word_Break = Format}];
 $Katakana     = [\p{Word_Break = Katakana}];
 $ALetter      = [\p{Word_Break = ALetter}];
-$MidNumLet    = [\p{Word_Break = MidNumLet}];
-$MidLetter    = [\p{Word_Break = MidLetter} - [ \:]];
-$MidNum       = [\p{Word_Break = MidNum}];
+$MidNumLet    = [\p{Word_Break = MidNumLet} - [.]];
+$MidLetter    = [\p{Word_Break = MidLetter} - [\:]];
+$MidNum       = [\p{Word_Break = MidNum} [.]];
 $Numeric      = [\p{Word_Break = Numeric}];
 $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
 

icu4c/source/data/brkitr/word_ja.txt

@@ -61,7 +61,7 @@ $NumericEx      = $Numeric      ($Extend |  $Format)*;
 $ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
 
 $Hiragana       = [\p{script=Hiragana}];
-$Ideographic    = [\p{Ideographic}];
+$Ideographic    = [\p{Ideographic} [\u3005 \u3007 \u303B]];
 $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
 $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;