organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-10 07:39:16 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 1

ICU-22707 UTC-180? Give up on 16.0β rules, amend LB10 and LB21a instead.

Browse source
This commit is contained in:
Robin Leroy 2024-07-05 00:55:05 +02:00 • committed by Markus Scherer
parent cc64ec7c94
commit 20fdebcb35
4 changed files with 16 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
icu4c/source/data/brkitr/rules/line.txt
View file

@ -284,47 +284,12 @@ $LB18Breaks = [$LB8Breaks $SP];
$LB18NonBreaks $CM* $QU;
^$CM+ $QU;
# OP and GL are subtracted because of LB14 and LB12 (there is no break after them).
# BA is subtracted because of LB21a:
# We must not poke a hole into HL U+3000 × [\p{Pi} & QU] [\p{ea=F}\p{ea=W}\p{ea=H}],
# where U+3000 is lb=BA and ea=W.
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}] - [$OP $GL $BA]] / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}] - [$OP $GL $BA]] $CM* $CMX / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^[$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^[$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* $CMX / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$OP $CM* $SP+ [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
($OP $CM* $SP+ | [$OP [$QU-\p{Pi}] $GL] $CM*) ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$LB8NonBreaks [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$CAN_CM $CM* [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^$CM+ [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}] - [$OP $GL]] / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}] - [$OP $GL]] $CM* $CMX / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$QU $CM* .;
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
[$LB18NonBreaks & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] $CM* $CMX / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
^[$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
^[$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] $CM* $CMX / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
$OP $CM* $SP+ [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
($OP $CM* $SP+ | [$OP [$QU-\p{Pi}] $GL] $CM*) ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
^([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
$LB8NonBreaks [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
$CAN_CM $CM* [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
^$CM+ [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$CM & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* [\p{Pf} & $QU] ($CM* $CMX)? / [ [\p{ea=F}\p{ea=W}\p{ea=H}] - [$NS $BA $EX $CL $IN $IS $GL $CM]];
^$CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^$CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* $CMX / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$OP $CM* $SP+ [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
($OP $CM* $SP+ | [$OP [$QU-\p{Pi}] $GL] $CM*) ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^([\p{Pi} & $QU] $CM* $SP*)+ $SP [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$LB8NonBreaks [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$CAN_CM $CM* [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
^$CM+ [\p{Pf} & $QU] $CM* ([\p{Pi} & $QU] $CM* $SP*)+ $SP [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$SP? $IS $CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
# LB 20
# <break> $CB
@ -332,11 +297,6 @@ $SP? $IS $CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $
#
$LB20NonBreaks = [$LB18NonBreaks - $CB];
[$LB20NonBreaks - [$HL $IS $RI]] $CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
[$LB20NonBreaks - [$HL $IS $RI]] $CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] $CM* $CMX / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
$CB $CM* $ZWJ [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
($RI $CM*)? $RI $CM* [$BA & [\p{ea=F}\p{ea=W}\p{ea=H}]] ($CM* $CMX)? / [\p{Pi} & $QU] $CM* [ [\p{ea=F}\p{ea=W}\p{ea=H}] - $CM];
# LB 20a Don't break between Hyphens and Letters when there is a break preceding the hyphen.
# Originally added as a Finnish tailoring, promoted to default ICU behavior (ICU-8151),
# and then to default UAX #14 behaviour (UTC-179-C32).
@ -369,7 +329,7 @@ $BB $CM* $LB20NonBreaks;
# LB 21a Do not break after the hyphen in Hebrew + Hyphen + non-Hebrew
# HL (HY | BA) x [^HL]
#
$HL $CM* ($HY | $BA) $CM* [^$CB $HL]?;
$HL $CM* ($HY | [ $BA - [\p{ea=F}\p{ea=W}\p{ea=H}] ] ) $CM* [^$CB $HL]?;
# LB 21b (forward) Don't break between SY and HL
# (break between HL and SY already disallowed by LB 13 above)

24
icu4c/source/test/intltest/rbbitst.cpp
View file

@ -2892,23 +2892,9 @@ void RBBILineMonkey::rule9Adjust(int32_t pos, UChar32 *posChar, int32_t *nextPos
// LB 9 Treat X CM* as if it were x.
// No explicit action required.
// LB 10 Treat any remaining combining mark as AL, but preserve its East
// Asian Width.
// LB 10 Treat any remaining combining mark as lb=AL, ea=Na.
if (fCM->contains(*posChar)) {
switch (u_getIntPropertyValue(*posChar, UCHAR_EAST_ASIAN_WIDTH)) {
case U_EA_WIDE:
*posChar = u'♈';
break;
case U_EA_NEUTRAL:
*posChar = u'ᴬ';
break;
case U_EA_AMBIGUOUS:
*posChar = u'Ⓐ';
break;
default:
puts("Unexpected ea value for lb=CM");
std::terminate();
}
*posChar = u'A';
}
// Push the updated nextPos and nextChar back to our caller.
@ -3281,7 +3267,8 @@ int32_t RBBILineMonkey::next(int32_t startPos) {
}
breakObliviousPrevPosX2 = beforeCM;
}
if (!feaFWH->contains(fText->char32At(breakObliviousPrevPosX2))) {
if (!feaFWH->contains(fText->char32At(breakObliviousPrevPosX2)) ||
fCM->contains(fText->char32At(breakObliviousPrevPosX2))) {
setAppliedRule(pos, "LB 19a [^\\p{ea=F}\\p{ea=W}\\p{ea=H}] QU ×");
continue;
}
@ -3332,7 +3319,8 @@ int32_t RBBILineMonkey::next(int32_t startPos) {
continue;
}
if (fHL->contains(prevCharX2) && (fHY->contains(prevChar) || fBA->contains(prevChar)) &&
if (fHL->contains(prevCharX2) &&
(fHY->contains(prevChar) || (fBA->contains(prevChar) && !feaFWH->contains(prevChar))) &&
!fHL->contains(thisChar)) {
setAppliedRule(pos, "LB 21a HL (HY | BA) x [^HL]");
continue;

23
icu4c/source/test/testdata/break_rules/line.txt vendored
View file

@ -77,13 +77,10 @@ OP30 = [OP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
CP30 = [CP - [\p{ea=F}\p{ea=W}\p{ea=H}]];
eaFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
eaFWHminusOPGL = [ eaFWH - [OP GL] ];
eaFWHandCM = [ CMS & eaFWH ];
eaFWHminusCMOPGL = [ eaFWH - [CMS OP GL] ];
eaFWHminusCM = [ eaFWH - CMS ];
eaFWHBreakableAtLB19 = [ eaFWH - [NS BA EX CL IN IS GL CMS] ];
# An annoying special case, \p{lb=BA} & [\p{ea=F}\p{ea=W}\p{ea=H}].
ideographicSpace = [\u3000];
BAminuseaFWH = [BA - eaFWH ];
PiQU = [\p{Pi}&QU];
PfQU = [\p{Pf}&QU];
@ -127,14 +124,10 @@ LB15a.1: ( OP CM* SP* | QU CM* | GL CM* ) (PiQU CM* SP*)+ .;
# Moved before LB14. These are really the cases where LB19a does not apply, but
# the old LB19 would. This is to avoid many instances of chaining over two code
# points.
LB19a.1: eaFWHminusOPGL ÷ PiQU CM* eaFWHminusCM;
LB19a.2: eaFWHminusOPGL CM* CMS ÷ PiQU CM* eaFWHminusCM;
LB19a.3: ^eaFWHandCM ÷ PiQU CM* eaFWHminusCM;
LB19a.4: ^eaFWHandCM CM* CMS ÷ PiQU CM* eaFWHminusCM;
LB19a.5: eaFWH CM* PfQU ÷ eaFWHBreakableAtLB19;
LB19a.6: eaFWH CM* PfQU CM* CMS ÷ eaFWHBreakableAtLB19;
LB19a.7: ^eaFWHandCM CM* PfQU ÷ eaFWHBreakableAtLB19;
LB19a.8: ^eaFWHandCM CM* PfQU CM* CMS ÷ eaFWHBreakableAtLB19;
LB19a.1: eaFWHminusCMOPGL ÷ PiQU CM* eaFWHminusCM;
LB19a.2: eaFWHminusCMOPGL CM* CMS ÷ PiQU CM* eaFWHminusCM;
LB19a.5: eaFWHminusCM CM* PfQU ÷ eaFWHBreakableAtLB19;
LB19a.6: eaFWHminusCM CM* PfQU CM* CMS ÷ eaFWHBreakableAtLB19;
# Moved up, before LB7, because they can match a longer sequence that would also match LB7.
# For example, the sequence "OP CM SP AL" matches LB14
@ -215,9 +208,7 @@ LB20a.1: ^(HY | HH) CM* AL;
# Note: Rule 21a must come before 21 to prevent 21.1 from matching HL BA, then
# not picking up the continuing match after the BA from 21a.
# Chains over two characters with the LB19a break rule.
LB21a.1: HL CM* ideographicSpace CM* PfQU (CM* CMS)? ÷ eaFWHBreakableAtLB19;
LB21a: HL CM* (HY | BA) CM* [^CM CB HL];
LB21a: HL CM* (HY | BAminuseaFWH) CM* [^CM CB HL];
LB21.1: . CM* [BA HY NS];
LB21.2: BB CM* [^CM CB];

13
icu4c/source/test/testdata/rbbitst.txt vendored
View file

@ -2223,16 +2223,3 @@ Bangkok)•</data>
<data>•« Complex »« chaining » •</data>
<data>•« .618 »•</data> # Interaction with the ICU tailoring to break before such numbers.
# Non-breaking lb=SP (from LB14 and LB15a) followed by a lb=CM-as-AL that is
# ea=W, in a position that would match initial context for LB19a if it were not
# ea=W.
# See https://github.com/unicode-org/icu/pull/3028#issuecomment-2200259320.
<data>•︷ \U00016FF1•⸠ᅛᆅ•</data>
<data>•︷ « \U00016FF1•⸠ᅛᆅ•</data>
<data>•A »« \U00016FF1•⸠ᅛᆅ•</data>
<data>•︷ \U00016FF1\u302B•⸠ᅛᆅ•</data>
<data>•︷ « \U00016FF1\u302B•⸠ᅛᆅ•</data>
<data>•A »« \U00016FF1\u302B•⸠ᅛᆅ•</data>
<data>•❲ \u3035⸍•굼•</data>
<data>•❲ « \u3035⸍•굼•</data>
<data>•A »« \u3035⸍•굼•</data>