mirror of
https://github.com/unicode-org/icu.git
synced 2025-04-06 14:05:32 +00:00
ICU-21699 Revise rule file
1. \u30fc doesn't belong to Hira, Kana nor Han. Add it into CJK dictionary 2. Include fullwidth char into ALPlus
This commit is contained in:
allenwtsu
Markus Scherer
parent
205144e391
commit
c882c94d83
8 changed files with 45 additions and 22 deletions
|
|
@ -93,9 +93,8 @@ $ZWJ = [:LineBreak = ZWJ:];
|
|||
# without a formal name. Because ICU rules require multiple uses of the expressions,
|
||||
# give them a single definition with a name
|
||||
|
||||
$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
|
||||
$OP30 = [$OP - $EAFWH];
|
||||
$CP30 = [$CP - $EAFWH];
|
||||
$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
|
||||
$ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
|
||||
|
||||
|
|
@ -115,7 +114,7 @@ $Katakana = [:Katakana:];
|
|||
$Hiragana = [:Hiragana:];
|
||||
$HangulSyllable = [\uac00-\ud7a3];
|
||||
$ComplexContext = [:LineBreak = Complex_Context:];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana \u30fc];
|
||||
$dictionaryCJK = [$KanaKanji $HangulSyllable];
|
||||
$dictionary = [$ComplexContext $dictionaryCJK];
|
||||
|
||||
|
|
@ -127,7 +126,9 @@ $dictionary = [$ComplexContext $dictionaryCJK];
|
|||
# XX (Unknown, unassigned)
|
||||
# as $AL (Alphabetic)
|
||||
#
|
||||
$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
# Let fullwidth-ASCII digits and letters be part of words.
|
||||
$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
|
||||
$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
|
||||
|
||||
## -------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -81,9 +81,8 @@ $ZWJ = [:LineBreak = ZWJ:];
|
|||
# without a formal name. Because ICU rules require multiple uses of the expressions,
|
||||
# give them a single definition with a name
|
||||
|
||||
$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
|
||||
$OP30 = [$OP - $EAFWH];
|
||||
$CP30 = [$CP - $EAFWH];
|
||||
$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
|
||||
$ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
|
||||
|
||||
|
|
@ -103,7 +102,7 @@ $Katakana = [:Katakana:];
|
|||
$Hiragana = [:Hiragana:];
|
||||
$HangulSyllable = [\uac00-\ud7a3];
|
||||
$ComplexContext = [:LineBreak = Complex_Context:];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana \u30fc];
|
||||
$dictionaryCJK = [$KanaKanji $HangulSyllable];
|
||||
$dictionary = [$ComplexContext $dictionaryCJK];
|
||||
|
||||
|
|
@ -115,7 +114,9 @@ $dictionary = [$ComplexContext $dictionaryCJK];
|
|||
# XX (Unknown, unassigned)
|
||||
# as $AL (Alphabetic)
|
||||
#
|
||||
$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
# Let fullwidth-ASCII digits and letters be part of words.
|
||||
$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
|
||||
$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
|
||||
|
||||
## -------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -75,9 +75,8 @@ $ZWJ = [:LineBreak = ZWJ:];
|
|||
# without a formal name. Because ICU rules require multiple uses of the expressions,
|
||||
# give them a single definition with a name
|
||||
|
||||
$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
|
||||
$OP30 = [$OP - $EAFWH];
|
||||
$CP30 = [$CP - $EAFWH];
|
||||
$OP30 = [$OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
$CP30 = [$CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
|
||||
|
||||
$ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
|
||||
|
||||
|
|
@ -97,7 +96,7 @@ $Katakana = [:Katakana:];
|
|||
$Hiragana = [:Hiragana:];
|
||||
$HangulSyllable = [\uac00-\ud7a3];
|
||||
$ComplexContext = [:LineBreak = Complex_Context:];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana \u30fc];
|
||||
$dictionaryCJK = [$KanaKanji $HangulSyllable];
|
||||
$dictionary = [$ComplexContext $dictionaryCJK];
|
||||
|
||||
|
|
@ -109,7 +108,9 @@ $dictionary = [$ComplexContext $dictionaryCJK];
|
|||
# XX (Unknown, unassigned)
|
||||
# as $AL (Alphabetic)
|
||||
#
|
||||
$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
# Let fullwidth-ASCII digits and letters be part of words.
|
||||
$FW_alphanum = [\uff10-\uff19\uff21-\uff3a\uff41-\uff5a];
|
||||
$ALPlus = [$AL $AI $SG $XX $FW_alphanum [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
|
||||
|
||||
## -------------------------------------------------
|
||||
|
|
|
|||
10
icu4c/source/test/testdata/rbbitst.txt
vendored
10
icu4c/source/test/testdata/rbbitst.txt
vendored
|
|
@ -1901,6 +1901,16 @@ Bangkok)•</data>
|
|||
#Kana supplement: 𛁈(U+1B048) -> \uD82C\uDC48, 𛀸(U+1B038) -> \uD82C\uDC38, 𛀙(U+1B019)-> \uD82C\uDC19</data>
|
||||
#𛁈る𛀸（しるこ）、あ𛀙よろし（あかよろし） -> 𛁈る𛀸•（しるこ）、•あ𛀙よろし•（あ•かよろし）
|
||||
<data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
|
||||
#最初に目に入るのは、「許諾なき写真禁止」のサインである。 -> 最初に▁目に▁入るのは、▁「許諾なき▁写真▁禁止」▁の▁サインで▁ある。
|
||||
<data>•\u6700\u521D\u306B•\u76EE\u306B•\u5165\u308B\u306E\u306F\u3001•\u300C\u8A31\u8AFE\u306A\u304D•\u5199\u771F•\u7981\u6B62\u300D•\u306E•\u30B5\u30A4\u30F3\u3067•\u3042\u308B\u3002•</data>
|
||||
# ｄｏｃｏｍｏのサイト情報によると、７８０００パケット以上▁使うならパケ放題がいいとか -> ｄｏｃｏｍｏの▁サイト▁情報によると、▁７８０００パケット▁以上▁使う▁なら▁パケ▁放題が▁いい▁とか
|
||||
<data>•\uFF44\uFF4F\uFF43\uFF4F\uFF4D\uFF4F\u306E•\u30B5\u30A4\u30C8•\u60C5\u5831•\u306B\u3088\u308B\u3068\u3001•\uFF17\uFF18\uFF10\uFF10\uFF10\u30D1\u30B1\u30C3\u30C8•\u4EE5\u4E0A•\u4F7F\u3046•\u306A\u3089•\u30D1\u30B1•\u653E\u984C\u304C•\u3044\u3044•\u3068\u304B•</data>
|
||||
#日本の携帯はＣＤＭＡの形だといわれています -> 日本の▁携帯は▁ＣＤＭＡの▁形だと▁いわれ▁ています
|
||||
<data>•\u65E5\u672C\u306E•\u643A\u5E2F\u306F\uFF23\uFF24\uFF2D\uFF21\u306E•\u5F62\u3060\u3068•\u3044\u308F\u308C•\u3066\u3044\u307E\u3059•</data>
|
||||
#中国の携帯は約５００元から５０００元です -> 中国の▁携帯は▁約▁５００元から▁５０００元です
|
||||
<data>•\u4E2D\u56FD\u306E•\u643A\u5E2F\u306F•\u7D04•\uFF15\uFF10\uFF10\u5143\u304B\u3089•\uFF15\uFF10\uFF10\uFF10\u5143\u3067\u3059•</data>
|
||||
#プリペイドカード携帯布教 -> プリペイドカード▁携帯▁布教
|
||||
<data>•\u30D7\u30EA\u30DA\u30A4\u30C9\u30AB\u30FC\u30C9•\u643A\u5E2F•\u5E03\u6559•</data>
|
||||
|
||||
####################################################################################
|
||||
#
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a53ab1162a9a25cf8f1e3abef3d4aa24acb11cb7677652be8d76e6843cbe5b35
|
||||
size 13871000
|
||||
oid sha256:658d41e63f73089111e306f0d90b2b3aa4757146d4eda963f8e181c0acf2b9ab
|
||||
size 13866832
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:57ada8a76a3d3d728a4ffd58fbcbe6aba7de2cf3b36d08ee95b2a9a222403880
|
||||
size 96439
|
||||
oid sha256:79e4e94b0fc31ed37701b94512b985c6bf80429bb3420adb645abc4f26c913d7
|
||||
size 96440
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ccde7f7d4e6bf0a92d2c3d6999afabbb063a154b7d66b3fd79d6ab0707e1fad7
|
||||
size 826063
|
||||
oid sha256:c20f10955116ac255c504e8743a9c4b2eab9c0bac7dc2250f1fbe775100f3b8a
|
||||
size 826064
|
||||
|
|
|
|||
|
|
@ -1901,7 +1901,17 @@ Bangkok)•</data>
|
|||
#Kana supplement: 𛁈(U+1B048) -> \uD82C\uDC48, 𛀸(U+1B038) -> \uD82C\uDC38, 𛀙(U+1B019)-> \uD82C\uDC19</data>
|
||||
#𛁈る𛀸（しるこ）、あ𛀙よろし（あかよろし） -> 𛁈る𛀸•（しるこ）、•あ𛀙よろし•（あ•かよろし）
|
||||
<data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
|
||||
|
||||
<data>•\uD82C\uDC48\u308B\uD82C\uDC38•\uFF08\u3057\u308B\u3053\uFF09\u3001•\u3042\uD82C\uDC19\u3088\u308D\u3057•\uFF08\u3042•\u304B\u3088\u308D\u3057\uFF09•</data>
|
||||
#最初に目に入るのは、「許諾なき写真禁止」のサインである。 -> 最初に▁目に▁入るのは、▁「許諾なき▁写真▁禁止」▁の▁サインで▁ある。
|
||||
<data>•\u6700\u521D\u306B•\u76EE\u306B•\u5165\u308B\u306E\u306F\u3001•\u300C\u8A31\u8AFE\u306A\u304D•\u5199\u771F•\u7981\u6B62\u300D•\u306E•\u30B5\u30A4\u30F3\u3067•\u3042\u308B\u3002•</data>
|
||||
# ｄｏｃｏｍｏのサイト情報によると、７８０００パケット以上▁使うならパケ放題がいいとか -> ｄｏｃｏｍｏの▁サイト▁情報によると、▁７８０００パケット▁以上▁使う▁なら▁パケ▁放題が▁いい▁とか
|
||||
<data>•\uFF44\uFF4F\uFF43\uFF4F\uFF4D\uFF4F\u306E•\u30B5\u30A4\u30C8•\u60C5\u5831•\u306B\u3088\u308B\u3068\u3001•\uFF17\uFF18\uFF10\uFF10\uFF10\u30D1\u30B1\u30C3\u30C8•\u4EE5\u4E0A•\u4F7F\u3046•\u306A\u3089•\u30D1\u30B1•\u653E\u984C\u304C•\u3044\u3044•\u3068\u304B•</data>
|
||||
#日本の携帯はＣＤＭＡの形だといわれています -> 日本の▁携帯は▁ＣＤＭＡの▁形だと▁いわれ▁ています
|
||||
<data>•\u65E5\u672C\u306E•\u643A\u5E2F\u306F\uFF23\uFF24\uFF2D\uFF21\u306E•\u5F62\u3060\u3068•\u3044\u308F\u308C•\u3066\u3044\u307E\u3059•</data>
|
||||
#中国の携帯は約５００元から５０００元です -> 中国の▁携帯は▁約▁５００元から▁５０００元です
|
||||
<data>•\u4E2D\u56FD\u306E•\u643A\u5E2F\u306F•\u7D04•\uFF15\uFF10\uFF10\u5143\u304B\u3089•\uFF15\uFF10\uFF10\uFF10\u5143\u3067\u3059•</data>
|
||||
#プリペイドカード携帯布教 -> プリペイドカード▁携帯▁布教
|
||||
<data>•\u30D7\u30EA\u30DA\u30A4\u30C9\u30AB\u30FC\u30C9•\u643A\u5E2F•\u5E03\u6559•</data>
|
||||
|
||||
####################################################################################
|
||||
#
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue