ICU-5456 modern Greek toUpperCase(), remove most diacritics

X-SVN-Rev: 39228
This commit is contained in:
Markus Scherer 2016-09-14 23:17:11 +00:00
parent bdd4180fe1
commit 15415832df
7 changed files with 925 additions and 2 deletions

View file

@ -621,6 +621,18 @@ ucase_getCaseLocale(const char *locale, int32_t *locCache) {
result=UCASE_LOC_LITHUANIAN;
}
}
} else if(is_e(c)) {
/* el or ell? */
c=*locale++;
if(is_l(c)) {
c=*locale++;
if(is_l(c)) {
c=*locale;
}
if(is_sep(c)) {
result=UCASE_LOC_GREEK;
}
}
} else if(is_n(c)) {
/* nl or nld? */
c=*locale++;

View file

@ -64,6 +64,7 @@ enum {
UCASE_LOC_ROOT,
UCASE_LOC_TURKISH,
UCASE_LOC_LITHUANIAN,
UCASE_LOC_GREEK,
UCASE_LOC_DUTCH
};
@ -158,7 +159,7 @@ U_NAMESPACE_END
U_CAPI int32_t U_EXPORT2
ucase_getType(const UCaseProps *csp, UChar32 c);
/** @return same as ucase_getType(), or <0 if c is case-ignorable */
/** @return same as ucase_getType() and set bit 2 if c is case-ignorable */
U_CAPI int32_t U_EXPORT2
ucase_getTypeOrIgnorable(const UCaseProps *csp, UChar32 c);

View file

@ -176,6 +176,15 @@ appendResult(uint8_t *dest, int32_t destIndex, int32_t destCapacity,
return destIndex;
}
static inline int32_t
appendUChar(uint8_t *dest, int32_t destIndex, int32_t destCapacity, UChar c) {
int32_t limit=destIndex+U8_LENGTH(c);
if(limit<destCapacity) {
U8_APPEND_UNSAFE(dest, destIndex, c);
}
return limit;
}
static UChar32 U_CALLCONV
utf8_caseContextIterator(void *context, int8_t dir) {
UCaseContext *csc=(UCaseContext *)context;
@ -388,6 +397,141 @@ ucasemap_internalUTF8ToTitle(const UCaseMap *csm,
#endif
U_NAMESPACE_BEGIN
namespace GreekUpper {
UBool isFollowedByCasedLetter(const UCaseProps *csp, const uint8_t *s, int32_t i, int32_t length) {
while (i < length) {
UChar32 c;
U8_NEXT(s, i, length, c);
int32_t type = ucase_getTypeOrIgnorable(csp, c);
if ((type & UCASE_IGNORABLE) != 0) {
// Case-ignorable, continue with the loop.
} else if (type != UCASE_NONE) {
return TRUE; // Followed by cased letter.
} else {
return FALSE; // Uncased and not case-ignorable.
}
}
return FALSE; // Not followed by cased letter.
}
// Keep this consistent with the UTF-16 version in ustrcase.cpp and the Java version in CaseMap.java.
int32_t toUpper(const UCaseMap *csm,
uint8_t *dest, int32_t destCapacity,
const uint8_t *src, int32_t srcLength,
UErrorCode *pErrorCode) {
int32_t locCache = UCASE_LOC_GREEK;
int32_t destIndex=0;
uint32_t state = 0;
for (int32_t i = 0; i < srcLength;) {
int32_t nextIndex = i;
UChar32 c;
U8_NEXT(src, nextIndex, srcLength, c);
uint32_t nextState = 0;
int32_t type = ucase_getTypeOrIgnorable(csm->csp, c);
if ((type & UCASE_IGNORABLE) != 0) {
// c is case-ignorable
nextState |= (state & AFTER_CASED);
} else if (type != UCASE_NONE) {
// c is cased
nextState |= AFTER_CASED;
}
uint32_t data = getLetterData(c);
if (data > 0) {
uint32_t upper = data & UPPER_MASK;
// Add a dialytika to this iota or ypsilon vowel
// if we removed a tonos from the previous vowel,
// and that previous vowel did not also have (or gain) a dialytika.
// Adding one only to the final vowel in a longer sequence
// (which does not occur in normal writing) would require lookahead.
// Set the same flag as for preserving an existing dialytika.
if ((data & HAS_VOWEL) != 0 && (state & AFTER_VOWEL_WITH_ACCENT) != 0 &&
(upper == 0x399 || upper == 0x3A5)) {
data |= HAS_DIALYTIKA;
}
int32_t numYpogegrammeni = 0; // Map each one to a trailing, spacing, capital iota.
if ((data & HAS_YPOGEGRAMMENI) != 0) {
numYpogegrammeni = 1;
}
// Skip combining diacritics after this Greek letter.
int32_t nextNextIndex = nextIndex;
while (nextIndex < srcLength) {
UChar32 c2;
U8_NEXT(src, nextNextIndex, srcLength, c2);
uint32_t diacriticData = getDiacriticData(c2);
if (diacriticData != 0) {
data |= diacriticData;
if ((diacriticData & HAS_YPOGEGRAMMENI) != 0) {
++numYpogegrammeni;
}
nextIndex = nextNextIndex;
} else {
break; // not a Greek diacritic
}
}
if ((data & HAS_VOWEL_AND_ACCENT_AND_DIALYTIKA) == HAS_VOWEL_AND_ACCENT) {
nextState |= AFTER_VOWEL_WITH_ACCENT;
}
// Map according to Greek rules.
UBool addTonos = FALSE;
if (upper == 0x397 &&
(data & HAS_ACCENT) != 0 &&
numYpogegrammeni == 0 &&
(state & AFTER_CASED) == 0 &&
!isFollowedByCasedLetter(csm->csp, src, nextIndex, srcLength)) {
// Keep disjunctive "or" with (only) a tonos.
// We use the same "word boundary" conditions as for the Final_Sigma test.
if (i == nextIndex) {
upper = 0x389; // Preserve the precomposed form.
} else {
addTonos = TRUE;
}
} else if ((data & HAS_DIALYTIKA) != 0) {
// Preserve a vowel with dialytika in precomposed form if it exists.
if (upper == 0x399) {
upper = 0x3AA;
data &= ~HAS_EITHER_DIALYTIKA;
} else if (upper == 0x3A5) {
upper = 0x3AB;
data &= ~HAS_EITHER_DIALYTIKA;
}
}
destIndex=appendUChar(dest, destIndex, destCapacity, (UChar)upper);
if ((data & HAS_EITHER_DIALYTIKA) != 0) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x308); // restore or add a dialytika
}
if (addTonos) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x301);
}
while (numYpogegrammeni > 0) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x399);
--numYpogegrammeni;
}
} else {
const UChar *s;
UChar32 c2 = 0;
c=ucase_toFullUpper(csm->csp, c, NULL, NULL, &s, csm->locale, &locCache);
if((destIndex<destCapacity) && (c<0 ? (c2=~c)<=0x7f : UCASE_MAX_STRING_LENGTH<c && (c2=c)<=0x7f)) {
/* fast path version of appendResult() for ASCII results */
dest[destIndex++]=(uint8_t)c2;
} else {
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
}
}
i = nextIndex;
state = nextState;
}
if(destIndex>destCapacity) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
return destIndex;
}
} // namespace GreekUpper
U_NAMESPACE_END
static int32_t U_CALLCONV
ucasemap_internalUTF8ToLower(const UCaseMap *csm,
uint8_t *dest, int32_t destCapacity,
@ -408,6 +552,10 @@ ucasemap_internalUTF8ToUpper(const UCaseMap *csm,
uint8_t *dest, int32_t destCapacity,
const uint8_t *src, int32_t srcLength,
UErrorCode *pErrorCode) {
int32_t locCache = csm->locCache;
if (ucase_getCaseLocale(csm->locale, &locCache) == UCASE_LOC_GREEK) {
return GreekUpper::toUpper(csm, dest, destCapacity, src, srcLength, pErrorCode);
}
UCaseContext csc=UCASECONTEXT_INITIALIZER;
csc.p=(void *)src;
csc.limit=srcLength;

View file

@ -220,6 +220,44 @@ ucasemap_mapUTF8(const UCaseMap *csm,
UTF8CaseMapper *stringCaseMapper,
UErrorCode *pErrorCode);
#ifdef __cplusplus
U_NAMESPACE_BEGIN
namespace GreekUpper {
// Data bits.
static const uint32_t UPPER_MASK = 0x3ff;
static const uint32_t HAS_VOWEL = 0x1000;
static const uint32_t HAS_YPOGEGRAMMENI = 0x2000;
static const uint32_t HAS_ACCENT = 0x4000;
static const uint32_t HAS_DIALYTIKA = 0x8000;
// Further bits during data building and processing, not stored in the data map.
static const uint32_t HAS_COMBINING_DIALYTIKA = 0x10000;
static const uint32_t HAS_OTHER_GREEK_DIACRITIC = 0x20000;
static const uint32_t HAS_VOWEL_AND_ACCENT = HAS_VOWEL | HAS_ACCENT;
static const uint32_t HAS_VOWEL_AND_ACCENT_AND_DIALYTIKA =
HAS_VOWEL_AND_ACCENT | HAS_DIALYTIKA;
static const uint32_t HAS_EITHER_DIALYTIKA = HAS_DIALYTIKA | HAS_COMBINING_DIALYTIKA;
// State bits.
static const uint32_t AFTER_CASED = 1;
static const uint32_t AFTER_VOWEL_WITH_ACCENT = 2;
uint32_t getLetterData(UChar32 c);
/**
* Returns a non-zero value for each of the Greek combining diacritics
* listed in The Unicode Standard, version 8, chapter 7.2 Greek,
* plus some perispomeni look-alikes.
*/
uint32_t getDiacriticData(UChar32 c);
} // namespace GreekUpper
U_NAMESPACE_END
#endif // __cplusplus
U_CAPI int32_t U_EXPORT2
ustr_hashUCharsN(const UChar *str, int32_t length);

View file

@ -89,6 +89,14 @@ appendResult(UChar *dest, int32_t destIndex, int32_t destCapacity,
return destIndex;
}
static inline int32_t
appendUChar(UChar *dest, int32_t destIndex, int32_t destCapacity, UChar c) {
if(destIndex<destCapacity) {
dest[destIndex]=c;
}
return destIndex+1;
}
static UChar32 U_CALLCONV
utf16_caseContextIterator(void *context, int8_t dir) {
UCaseContext *csc=(UCaseContext *)context;
@ -295,6 +303,597 @@ ustrcase_internalToTitle(const UCaseMap *csm,
#endif // !UCONFIG_NO_BREAK_ITERATION
U_NAMESPACE_BEGIN
namespace GreekUpper {
// Data generated by prototype code, see
// http://site.icu-project.org/design/case/greek-upper
// TODO: Move this data into ucase.icu.
static const uint16_t data0370[] = {
// U+0370..03FF
0x0370,
0x0370,
0x0372,
0x0372,
0,
0,
0x0376,
0x0376,
0,
0,
0x037A,
0x03FD,
0x03FE,
0x03FF,
0,
0x037F,
0,
0,
0,
0,
0,
0,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0,
0x039F | HAS_VOWEL | HAS_ACCENT,
0,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x0391 | HAS_VOWEL,
0x0392,
0x0393,
0x0394,
0x0395 | HAS_VOWEL,
0x0396,
0x0397 | HAS_VOWEL,
0x0398,
0x0399 | HAS_VOWEL,
0x039A,
0x039B,
0x039C,
0x039D,
0x039E,
0x039F | HAS_VOWEL,
0x03A0,
0x03A1,
0,
0x03A3,
0x03A4,
0x03A5 | HAS_VOWEL,
0x03A6,
0x03A7,
0x03A8,
0x03A9 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_DIALYTIKA,
0x03A5 | HAS_VOWEL | HAS_DIALYTIKA,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x0391 | HAS_VOWEL,
0x0392,
0x0393,
0x0394,
0x0395 | HAS_VOWEL,
0x0396,
0x0397 | HAS_VOWEL,
0x0398,
0x0399 | HAS_VOWEL,
0x039A,
0x039B,
0x039C,
0x039D,
0x039E,
0x039F | HAS_VOWEL,
0x03A0,
0x03A1,
0x03A3,
0x03A3,
0x03A4,
0x03A5 | HAS_VOWEL,
0x03A6,
0x03A7,
0x03A8,
0x03A9 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_DIALYTIKA,
0x03A5 | HAS_VOWEL | HAS_DIALYTIKA,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03CF,
0x0392,
0x0398,
0x03D2,
0x03D2 | HAS_ACCENT,
0x03D2 | HAS_DIALYTIKA,
0x03A6,
0x03A0,
0x03CF,
0x03D8,
0x03D8,
0x03DA,
0x03DA,
0x03DC,
0x03DC,
0x03DE,
0x03DE,
0x03E0,
0x03E0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0x039A,
0x03A1,
0x03F9,
0x037F,
0x03F4,
0x0395 | HAS_VOWEL,
0,
0x03F7,
0x03F7,
0x03F9,
0x03FA,
0x03FA,
0x03FC,
0x03FD,
0x03FE,
0x03FF,
};
static const uint16_t data1F00[] = {
// U+1F00..1FFF
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL,
0x0395 | HAS_VOWEL,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0,
0,
0x0395 | HAS_VOWEL,
0x0395 | HAS_VOWEL,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0,
0,
0x0397 | HAS_VOWEL,
0x0397 | HAS_VOWEL,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL,
0x0397 | HAS_VOWEL,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL,
0x039F | HAS_VOWEL,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0,
0,
0x039F | HAS_VOWEL,
0x039F | HAS_VOWEL,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0,
0,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0,
0x03A5 | HAS_VOWEL,
0,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL,
0x03A9 | HAS_VOWEL,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL,
0x03A9 | HAS_VOWEL,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0,
0,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_ACCENT,
0x0391 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0,
0x0399 | HAS_VOWEL,
0,
0,
0,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0395 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_ACCENT,
0x0397 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0,
0,
0,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x0399 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0,
0,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0x0399 | HAS_VOWEL | HAS_ACCENT,
0,
0,
0,
0,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x03A5 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x03A1,
0x03A1,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT | HAS_DIALYTIKA,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A5 | HAS_VOWEL | HAS_ACCENT,
0x03A1,
0,
0,
0,
0,
0,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x039F | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_ACCENT,
0x03A9 | HAS_VOWEL | HAS_YPOGEGRAMMENI,
0,
0,
0,
};
// U+2126 Ohm sign
static const uint16_t data2126 = 0x03A9 | HAS_VOWEL;
uint32_t getLetterData(UChar32 c) {
if (c < 0x370 || 0x2126 < c || (0x3ff < c && c < 0x1f00)) {
return 0;
} else if (c <= 0x3ff) {
return data0370[c - 0x370];
} else if (c <= 0x1fff) {
return data1F00[c - 0x1f00];
} else if (c == 0x2126) {
return data2126;
} else {
return 0;
}
}
uint32_t getDiacriticData(UChar32 c) {
switch (c) {
case 0x0300: // varia
case 0x0301: // tonos = oxia
case 0x0342: // perispomeni
case 0x0302: // circumflex can look like perispomeni
case 0x0303: // tilde can look like perispomeni
case 0x0311: // inverted breve can look like perispomeni
return HAS_ACCENT;
case 0x0308: // dialytika = diaeresis
return HAS_COMBINING_DIALYTIKA;
case 0x0344: // dialytika tonos
return HAS_COMBINING_DIALYTIKA | HAS_ACCENT;
case 0x0345: // ypogegrammeni = iota subscript
return HAS_YPOGEGRAMMENI;
case 0x0304: // macron
case 0x0306: // breve
case 0x0313: // comma above
case 0x0314: // reversed comma above
case 0x0343: // koronis
return HAS_OTHER_GREEK_DIACRITIC;
default:
return 0;
}
}
UBool isFollowedByCasedLetter(const UCaseProps *csp, const UChar *s, int32_t i, int32_t length) {
while (i < length) {
UChar32 c;
U16_NEXT(s, i, length, c);
int32_t type = ucase_getTypeOrIgnorable(csp, c);
if ((type & UCASE_IGNORABLE) != 0) {
// Case-ignorable, continue with the loop.
} else if (type != UCASE_NONE) {
return TRUE; // Followed by cased letter.
} else {
return FALSE; // Uncased and not case-ignorable.
}
}
return FALSE; // Not followed by cased letter.
}
/**
* Greek string uppercasing with a state machine.
* Probably simpler than a stateless function that has to figure out complex context-before
* for each character.
* TODO: Try to re-consolidate one way or another with the non-Greek function.
*/
int32_t toUpper(const UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, int32_t srcLength,
UErrorCode *pErrorCode) {
int32_t locCache = UCASE_LOC_GREEK;
int32_t destIndex=0;
uint32_t state = 0;
for (int32_t i = 0; i < srcLength;) {
int32_t nextIndex = i;
UChar32 c;
U16_NEXT(src, nextIndex, srcLength, c);
uint32_t nextState = 0;
int32_t type = ucase_getTypeOrIgnorable(csm->csp, c);
if ((type & UCASE_IGNORABLE) != 0) {
// c is case-ignorable
nextState |= (state & AFTER_CASED);
} else if (type != UCASE_NONE) {
// c is cased
nextState |= AFTER_CASED;
}
uint32_t data = getLetterData(c);
if (data > 0) {
uint32_t upper = data & UPPER_MASK;
// Add a dialytika to this iota or ypsilon vowel
// if we removed a tonos from the previous vowel,
// and that previous vowel did not also have (or gain) a dialytika.
// Adding one only to the final vowel in a longer sequence
// (which does not occur in normal writing) would require lookahead.
// Set the same flag as for preserving an existing dialytika.
if ((data & HAS_VOWEL) != 0 && (state & AFTER_VOWEL_WITH_ACCENT) != 0 &&
(upper == 0x399 || upper == 0x3A5)) {
data |= HAS_DIALYTIKA;
}
int32_t numYpogegrammeni = 0; // Map each one to a trailing, spacing, capital iota.
if ((data & HAS_YPOGEGRAMMENI) != 0) {
numYpogegrammeni = 1;
}
// Skip combining diacritics after this Greek letter.
while (nextIndex < srcLength) {
uint32_t diacriticData = getDiacriticData(src[nextIndex]);
if (diacriticData != 0) {
data |= diacriticData;
if ((diacriticData & HAS_YPOGEGRAMMENI) != 0) {
++numYpogegrammeni;
}
++nextIndex;
} else {
break; // not a Greek diacritic
}
}
if ((data & HAS_VOWEL_AND_ACCENT_AND_DIALYTIKA) == HAS_VOWEL_AND_ACCENT) {
nextState |= AFTER_VOWEL_WITH_ACCENT;
}
// Map according to Greek rules.
UBool addTonos = FALSE;
if (upper == 0x397 &&
(data & HAS_ACCENT) != 0 &&
numYpogegrammeni == 0 &&
(state & AFTER_CASED) == 0 &&
!isFollowedByCasedLetter(csm->csp, src, nextIndex, srcLength)) {
// Keep disjunctive "or" with (only) a tonos.
// We use the same "word boundary" conditions as for the Final_Sigma test.
if (i == nextIndex) {
upper = 0x389; // Preserve the precomposed form.
} else {
addTonos = TRUE;
}
} else if ((data & HAS_DIALYTIKA) != 0) {
// Preserve a vowel with dialytika in precomposed form if it exists.
if (upper == 0x399) {
upper = 0x3AA;
data &= ~HAS_EITHER_DIALYTIKA;
} else if (upper == 0x3A5) {
upper = 0x3AB;
data &= ~HAS_EITHER_DIALYTIKA;
}
}
destIndex=appendUChar(dest, destIndex, destCapacity, (UChar)upper);
if ((data & HAS_EITHER_DIALYTIKA) != 0) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x308); // restore or add a dialytika
}
if (addTonos) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x301);
}
while (numYpogegrammeni > 0) {
destIndex=appendUChar(dest, destIndex, destCapacity, 0x399);
--numYpogegrammeni;
}
} else {
const UChar *s;
UChar32 c2 = 0;
c=ucase_toFullUpper(csm->csp, c, NULL, NULL, &s, csm->locale, &locCache);
if((destIndex<destCapacity) && (c<0 ? (c2=~c)<=0xffff : UCASE_MAX_STRING_LENGTH<c && (c2=c)<=0xffff)) {
/* fast path version of appendResult() for BMP results */
dest[destIndex++]=(UChar)c2;
} else {
destIndex=appendResult(dest, destIndex, destCapacity, c, s);
}
}
i = nextIndex;
state = nextState;
}
if(destIndex>destCapacity) {
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
}
return destIndex;
}
} // namespace GreekUpper
U_NAMESPACE_END
/* functions available in the common library (for unistr_case.cpp) */
U_CFUNC int32_t U_CALLCONV
@ -317,6 +916,10 @@ ustrcase_internalToUpper(const UCaseMap *csm,
UChar *dest, int32_t destCapacity,
const UChar *src, int32_t srcLength,
UErrorCode *pErrorCode) {
int32_t locCache = csm->locCache;
if (ucase_getCaseLocale(csm->locale, &locCache) == UCASE_LOC_GREEK) {
return GreekUpper::toUpper(csm, dest, destCapacity, src, srcLength, pErrorCode);
}
UCaseContext csc=UCASECONTEXT_INITIALIZER;
csc.p=(void *)src;
csc.limit=srcLength;

View file

@ -18,6 +18,7 @@
* Test file for string casing C++ API functions.
*/
#include "unicode/std_string.h"
#include "unicode/uchar.h"
#include "unicode/ures.h"
#include "unicode/uloc.h"
@ -30,6 +31,8 @@
#include "unicode/tstdtmod.h"
#include "cmemory.h"
StringCaseTest::StringCaseTest() : GREEK_LOCALE_("el") {}
StringCaseTest::~StringCaseTest() {}
void
@ -43,6 +46,7 @@ StringCaseTest::runIndexedTest(int32_t index, UBool exec, const char *&name, cha
TESTCASE_AUTO(TestCasing);
#endif
TESTCASE_AUTO(TestFullCaseFoldingIterator);
TESTCASE_AUTO(TestGreekUpper);
TESTCASE_AUTO_END;
}
@ -573,3 +577,113 @@ StringCaseTest::TestFullCaseFoldingIterator() {
errln("error: FullCaseFoldingIterator yielded only %d (cp, full) pairs", (int)count);
}
}
void
StringCaseTest::assertGreekUpper(const char *s, const char *expected) {
UnicodeString s16 = UnicodeString(s).unescape();
UnicodeString expected16 = UnicodeString(expected).unescape();
UnicodeString msg = UnicodeString("UnicodeString::toUpper/Greek(\"") + s16 + "\")";
UnicodeString result16(s16);
result16.toUpper(GREEK_LOCALE_);
assertEquals(msg, expected16, result16);
#if U_HAVE_STD_STRING
UErrorCode errorCode = U_ZERO_ERROR;
LocalUCaseMapPointer csm(ucasemap_open("el", 0, &errorCode));
assertSuccess("ucasemap_open", errorCode);
std::string s8;
s16.toUTF8String(s8);
msg = UnicodeString("ucasemap_utf8ToUpper/Greek(\"") + s16 + "\")";
char dest[1000];
int32_t length = ucasemap_utf8ToUpper(csm.getAlias(), dest, UPRV_LENGTHOF(dest),
s8.data(), s8.length(), &errorCode);
assertSuccess("ucasemap_utf8ToUpper", errorCode);
StringPiece result8(dest, length);
UnicodeString result16From8 = UnicodeString::fromUTF8(result8);
assertEquals(msg, expected16, result16From8);
#endif
}
void
StringCaseTest::TestGreekUpper() {
// See UCharacterCaseTest.java for human-readable strings.
// http://bugs.icu-project.org/trac/ticket/5456
assertGreekUpper("\\u03AC\\u03B4\\u03B9\\u03BA\\u03BF\\u03C2, "
"\\u03BA\\u03B5\\u03AF\\u03BC\\u03B5\\u03BD\\u03BF, "
"\\u03AF\\u03C1\\u03B9\\u03B4\\u03B1",
"\\u0391\\u0394\\u0399\\u039A\\u039F\\u03A3, "
"\\u039A\\u0395\\u0399\\u039C\\u0395\\u039D\\u039F, "
"\\u0399\\u03A1\\u0399\\u0394\\u0391");
// https://bugzilla.mozilla.org/show_bug.cgi?id=307039
// https://bug307039.bmoattachments.org/attachment.cgi?id=194893
assertGreekUpper("\\u03A0\\u03B1\\u03C4\\u03AC\\u03C4\\u03B1",
"\\u03A0\\u0391\\u03A4\\u0391\\u03A4\\u0391");
assertGreekUpper("\\u0391\\u03AD\\u03C1\\u03B1\\u03C2, "
"\\u039C\\u03C5\\u03C3\\u03C4\\u03AE\\u03C1\\u03B9\\u03BF, "
"\\u03A9\\u03C1\\u03B1\\u03AF\\u03BF",
"\\u0391\\u0395\\u03A1\\u0391\\u03A3, "
"\\u039C\\u03A5\\u03A3\\u03A4\\u0397\\u03A1\\u0399\\u039F, "
"\\u03A9\\u03A1\\u0391\\u0399\\u039F");
assertGreekUpper("\\u039C\\u03B1\\u0390\\u03BF\\u03C5, \\u03A0\\u03CC\\u03C1\\u03BF\\u03C2, "
"\\u03A1\\u03CD\\u03B8\\u03BC\\u03B9\\u03C3\\u03B7",
"\\u039C\\u0391\\u03AA\\u039F\\u03A5, \\u03A0\\u039F\\u03A1\\u039F\\u03A3, "
"\\u03A1\\u03A5\\u0398\\u039C\\u0399\\u03A3\\u0397");
assertGreekUpper("\\u03B0, \\u03A4\\u03B7\\u03C1\\u03CE, \\u039C\\u03AC\\u03B9\\u03BF\\u03C2",
"\\u03AB, \\u03A4\\u0397\\u03A1\\u03A9, \\u039C\\u0391\\u03AA\\u039F\\u03A3");
assertGreekUpper("\\u03AC\\u03C5\\u03BB\\u03BF\\u03C2",
"\\u0391\\u03AB\\u039B\\u039F\\u03A3");
assertGreekUpper("\\u0391\\u03AB\\u039B\\u039F\\u03A3",
"\\u0391\\u03AB\\u039B\\u039F\\u03A3");
assertGreekUpper("\\u0386\\u03BA\\u03BB\\u03B9\\u03C4\\u03B1 "
"\\u03C1\\u03AE\\u03BC\\u03B1\\u03C4\\u03B1 \\u03AE "
"\\u03AC\\u03BA\\u03BB\\u03B9\\u03C4\\u03B5\\u03C2 "
"\\u03BC\\u03B5\\u03C4\\u03BF\\u03C7\\u03AD\\u03C2",
"\\u0391\\u039A\\u039B\\u0399\\u03A4\\u0391 "
"\\u03A1\\u0397\\u039C\\u0391\\u03A4\\u0391 \\u0397\\u0301 "
"\\u0391\\u039A\\u039B\\u0399\\u03A4\\u0395\\u03A3 "
"\\u039C\\u0395\\u03A4\\u039F\\u03A7\\u0395\\u03A3");
// http://www.unicode.org/udhr/d/udhr_ell_monotonic.html
assertGreekUpper("\\u0395\\u03C0\\u03B5\\u03B9\\u03B4\\u03AE \\u03B7 "
"\\u03B1\\u03BD\\u03B1\\u03B3\\u03BD\\u03CE\\u03C1\\u03B9\\u03C3\\u03B7 "
"\\u03C4\\u03B7\\u03C2 \\u03B1\\u03BE\\u03B9\\u03BF\\u03C0\\u03C1\\u03AD"
"\\u03C0\\u03B5\\u03B9\\u03B1\\u03C2",
"\\u0395\\u03A0\\u0395\\u0399\\u0394\\u0397 \\u0397 "
"\\u0391\\u039D\\u0391\\u0393\\u039D\\u03A9\\u03A1\\u0399\\u03A3\\u0397 "
"\\u03A4\\u0397\\u03A3 \\u0391\\u039E\\u0399\\u039F\\u03A0\\u03A1\\u0395"
"\\u03A0\\u0395\\u0399\\u0391\\u03A3");
assertGreekUpper("\\u03BD\\u03BF\\u03BC\\u03B9\\u03BA\\u03BF\\u03CD \\u03AE "
"\\u03B4\\u03B9\\u03B5\\u03B8\\u03BD\\u03BF\\u03CD\\u03C2",
"\\u039D\\u039F\\u039C\\u0399\\u039A\\u039F\\u03A5 \\u0397\\u0301 "
"\\u0394\\u0399\\u0395\\u0398\\u039D\\u039F\\u03A5\\u03A3");
// http://unicode.org/udhr/d/udhr_ell_polytonic.html
assertGreekUpper("\\u1F18\\u03C0\\u03B5\\u03B9\\u03B4\\u1F74 \\u1F21 "
"\\u1F00\\u03BD\\u03B1\\u03B3\\u03BD\\u1F7D\\u03C1\\u03B9\\u03C3\\u03B7",
"\\u0395\\u03A0\\u0395\\u0399\\u0394\\u0397 \\u0397 "
"\\u0391\\u039D\\u0391\\u0393\\u039D\\u03A9\\u03A1\\u0399\\u03A3\\u0397");
assertGreekUpper("\\u03BD\\u03BF\\u03BC\\u03B9\\u03BA\\u03BF\\u1FE6 \\u1F22 "
"\\u03B4\\u03B9\\u03B5\\u03B8\\u03BD\\u03BF\\u1FE6\\u03C2",
"\\u039D\\u039F\\u039C\\u0399\\u039A\\u039F\\u03A5 \\u0397\\u0301 "
"\\u0394\\u0399\\u0395\\u0398\\u039D\\u039F\\u03A5\\u03A3");
// From Google bug report
assertGreekUpper("\\u039D\\u03AD\\u03BF, "
"\\u0394\\u03B7\\u03BC\\u03B9\\u03BF\\u03C5\\u03C1\\u03B3\\u03AF\\u03B1",
"\\u039D\\u0395\\u039F, "
"\\u0394\\u0397\\u039C\\u0399\\u039F\\u03A5\\u03A1\\u0393\\u0399\\u0391");
// http://crbug.com/234797
assertGreekUpper("\\u0395\\u03BB\\u03AC\\u03C4\\u03B5 \\u03BD\\u03B1 \\u03C6\\u03AC\\u03C4\\u03B5 "
"\\u03C4\\u03B1 \\u03BA\\u03B1\\u03BB\\u03CD\\u03C4\\u03B5\\u03C1\\u03B1 "
"\\u03C0\\u03B1\\u03CA\\u03B4\\u03AC\\u03BA\\u03B9\\u03B1!",
"\\u0395\\u039B\\u0391\\u03A4\\u0395 \\u039D\\u0391 \\u03A6\\u0391\\u03A4\\u0395 "
"\\u03A4\\u0391 \\u039A\\u0391\\u039B\\u03A5\\u03A4\\u0395\\u03A1\\u0391 "
"\\u03A0\\u0391\\u03AA\\u0394\\u0391\\u039A\\u0399\\u0391!");
assertGreekUpper("\\u039C\\u03B1\\u0390\\u03BF\\u03C5, \\u03C4\\u03C1\\u03CC\\u03BB\\u03B5\\u03CA",
"\\u039C\\u0391\\u03AA\\u039F\\u03A5, \\u03A4\\u03A1\\u039F\\u039B\\u0395\\u03AA");
assertGreekUpper("\\u03A4\\u03BF \\u03AD\\u03BD\\u03B1 \\u03AE \\u03C4\\u03BF "
"\\u03AC\\u03BB\\u03BB\\u03BF.",
"\\u03A4\\u039F \\u0395\\u039D\\u0391 \\u0397\\u0301 \\u03A4\\u039F "
"\\u0391\\u039B\\u039B\\u039F.");
// http://multilingualtypesetting.co.uk/blog/greek-typesetting-tips/
assertGreekUpper("\\u03C1\\u03C9\\u03BC\\u03AD\\u03B9\\u03BA\\u03B1",
"\\u03A1\\u03A9\\u039C\\u0395\\u03AA\\u039A\\u0391");
}

View file

@ -9,6 +9,7 @@
#ifndef UNICODESTRINGTEST_H
#define UNICODESTRINGTEST_H
#include "unicode/locid.h"
#include "unicode/unistr.h"
#include "intltest.h"
@ -95,7 +96,7 @@ public:
class StringCaseTest: public IntlTest {
public:
StringCaseTest() {}
StringCaseTest();
virtual ~StringCaseTest();
void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=0);
@ -108,6 +109,12 @@ public:
void *iter, const char *localeID, uint32_t options);
void TestCasing();
void TestFullCaseFoldingIterator();
void TestGreekUpper();
private:
void assertGreekUpper(const char *s, const char *expected);
Locale GREEK_LOCALE_;
};
#endif