organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-16 10:17:23 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 10

ICU-6024 Add dynamic buffer to collIterate to extend the size of CEs if needed to limit buffer overflow errors. CE buffer overflow cintltst test is changed to reflect this change.

Browse source 
X-SVN-Rev: 23089
This commit is contained in:
Michael Ow 2007-12-15 00:30:35 +00:00
parent b29d3066db
commit d9958662e3
6 changed files with 81 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
icu4c/source/i18n/coleitr.cpp
View file

@ -461,13 +461,26 @@ const CollationElementIterator& CollationElementIterator::operator=(
}
/* CE buffer */
int32_t CEsize = (int32_t)(othercoliter->CEpos - othercoliter->CEs);
if (CEsize > 0) {
uprv_memcpy(coliter->CEs, othercoliter->CEs, CEsize);
int32_t CEsize;
if (coliter->extendCEs) {
uprv_memcpy(coliter->CEs, othercoliter->CEs, sizeof(uint32_t) * UCOL_EXPAND_CE_BUFFER_SIZE);
CEsize = sizeof(othercoliter->extendCEs);
if (CEsize > 0) {
othercoliter->extendCEs = (uint32_t *)uprv_malloc(CEsize);
uprv_memcpy(coliter->extendCEs, othercoliter->extendCEs, CEsize);
}
coliter->toReturn = coliter->extendCEs +
(othercoliter->toReturn - othercoliter->extendCEs);
coliter->CEpos = coliter->extendCEs + CEsize;
} else {
CEsize = (int32_t)(othercoliter->CEpos - othercoliter->CEs);
if (CEsize > 0) {
uprv_memcpy(coliter->CEs, othercoliter->CEs, CEsize);
}
coliter->toReturn = coliter->CEs +
(othercoliter->toReturn - othercoliter->CEs);
coliter->CEpos = coliter->CEs + CEsize;
}
coliter->toReturn = coliter->CEs +
(othercoliter->toReturn - othercoliter->CEs);
coliter->CEpos = coliter->CEs + CEsize;
if (othercoliter->fcdPosition != NULL) {
coliter->fcdPosition = coliter->string +

62
icu4c/source/i18n/ucol.cpp
View file

@ -128,6 +128,8 @@ inline void IInit_collIterate(const UCollator *collator, const UChar *sourceStr
/* change to enable easier checking for end of string for fcdpositon */
(s)->endp = NULL;
}
(s)->extendCEs = NULL;
(s)->extendCEsSize = 0;
(s)->CEpos = (s)->toReturn = (s)->CEs;
(s)->writableBuffer = (s)->stackWritableBuffer;
(s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE;
@ -1411,7 +1413,7 @@ inline uint32_t ucol_IGetNextCE(const UCollator *coll, collIterate *collationSou
if (collationSource->CEpos > collationSource->toReturn) { /* Are there any CEs from previous expansions? */
order = *(collationSource->toReturn++); /* if so, return them */
if(collationSource->CEpos == collationSource->toReturn) {
collationSource->CEpos = collationSource->toReturn = collationSource->CEs;
collationSource->CEpos = collationSource->toReturn = collationSource->extendCEs ? collationSource->extendCEs : collationSource->CEs;
}
return order;
}
@ -1791,10 +1793,11 @@ inline uint32_t ucol_IGetPrevCE(const UCollator *coll, collIterate *data,
UErrorCode *status)
{
uint32_t result = (uint32_t)UCOL_NULLORDER;
if (data->toReturn > data->CEs) {
if ((data->extendCEs && data->toReturn > data->extendCEs) ||
(!data->extendCEs && data->toReturn > data->CEs)) {
data->toReturn --;
result = *(data->toReturn);
if (data->CEs == data->toReturn) {
if (data->CEs == data->toReturn || data->extendCEs == data->toReturn) {
data->CEpos = data->toReturn;
}
}
@ -3285,6 +3288,7 @@ uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
uint32_t *endCEBuffer;
UChar *strbuffer;
int32_t noChars = 0;
int32_t CECount = 0;
for(;;)
{
@ -3462,22 +3466,54 @@ uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE,
//IInit_collIterate(coll, UCharOffset, -1, &temp);
IInit_collIterate(coll, UCharOffset, noChars, &temp);
temp.flags &= ~UCOL_ITER_NORM;
CE = ucol_IGetNextCE(coll, &temp, status);
endCEBuffer = source->CEs + UCOL_EXPAND_CE_BUFFER_SIZE;
if (source->extendCEs) {
endCEBuffer = source->extendCEs + source->extendCEsSize;
CECount = (source->CEpos - source->extendCEs)/sizeof(uint32_t);
} else {
endCEBuffer = source->CEs + UCOL_EXPAND_CE_BUFFER_SIZE;
CECount = (source->CEpos - source->CEs)/sizeof(uint32_t);
}
while (CE != UCOL_NO_MORE_CES) {
*(source->CEpos ++) = CE;
CECount++;
if (source->CEpos == endCEBuffer) {
/* ran out of CE space, bail.
/* ran out of CE space, reallocate to new buffer.
If reallocation fails, reset pointers and bail out,
there's no guarantee of the right character position after
this bail*/
*status = U_BUFFER_OVERFLOW_ERROR;
source->CEpos = source->CEs;
freeHeapWritableBuffer(&temp);
if (strbuffer != buffer) {
uprv_free(strbuffer);
}
return (uint32_t)UCOL_NULLORDER;
if (source->extendCEs == NULL) {
source->extendCEs = (uint32_t *)uprv_malloc(sizeof(uint32_t) *
(source->extendCEsSize =UCOL_EXPAND_CE_BUFFER_SIZE + UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE));
if (source->extendCEs == NULL) {
CECount = -1;
} else {
source->extendCEs = (uint32_t *)uprv_memcpy(source->extendCEs, source->CEs, UCOL_EXPAND_CE_BUFFER_SIZE * sizeof(uint32_t));
}
} else {
uint32_t *temp = source->extendCEs;
source->extendCEs = (uint32_t *)uprv_realloc(source->extendCEs,
sizeof(uint32_t) * (source->extendCEsSize += UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE));
if (source->extendCEs == NULL) {
CECount = -1;
source->extendCEs = temp;
uprv_free(source->extendCEs);
source->extendCEs = NULL;
source->extendCEsSize = 0;
}
}
if (CECount == -1) {
*status = U_BUFFER_OVERFLOW_ERROR;
source->CEpos = source->CEs;
freeHeapWritableBuffer(&temp);
if (strbuffer != buffer) {
uprv_free(strbuffer);
}
return (uint32_t)UCOL_NULLORDER;
}
source->CEpos = source->extendCEs + CECount;
endCEBuffer = source->extendCEs + source->extendCEsSize;
}
CE = ucol_IGetNextCE(coll, &temp, status);
}

7
icu4c/source/i18n/ucol_imp.h
View file

@ -222,7 +222,10 @@ minimum number for special Jamo
/* WARNING THIS BUFFER DOES NOT HAVE MALLOC FALLBACK. If you make it too small, you'll get in trouble */
/* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */
/* This is the longest expansion sequence we can handle without bombing out */
#define UCOL_EXPAND_CE_BUFFER_SIZE 512 /* synwee :TODO revert back 64*/
#define UCOL_EXPAND_CE_BUFFER_SIZE 64 /* synwee :TODO revert back 64*/
/* This is the size to increase the buffer for expansion CE's */
#define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64
/* Unsafe UChar hash table table size. */
@ -276,6 +279,8 @@ typedef struct collIterate {
const UCollator *coll;
uint8_t flags;
uint8_t origFlags;
uint32_t *extendCEs; /* This is use if CEs is not big enough */
int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */
uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */
UChar stackWritableBuffer[UCOL_WRITABLE_BUFFER_SIZE]; /* A writable buffer. */
UCharIterator *iterator;

3
icu4c/source/i18n/ucoleitr.cpp
View file

@ -72,6 +72,9 @@ ucol_closeElements(UCollationElements *elems)
{
uprv_free(elems->iteratordata_.string);
}
if (ci->extendCEs) {
uprv_free(ci->extendCEs);
}
uprv_free(elems);
}

2
icu4c/source/i18n/usearch.cpp
View file

@ -44,7 +44,7 @@ inline void setColEIterOffset(UCollationElements *elems,
{
collIterate *ci = &(elems->iteratordata_);
ci->pos = ci->string + offset;
ci->CEpos = ci->toReturn = ci->CEs;
ci->CEpos = ci->toReturn = ci->extendCEs ? ci->extendCEs : ci->CEs;
if (ci->flags & UCOL_ITER_INNORMBUF) {
ci->flags = ci->origFlags;
}

6
icu4c/source/test/cintltst/citertst.c
View file

@ -1462,9 +1462,9 @@ static void TestCEBufferOverflow()
str[UCOL_EXPAND_CE_BUFFER_SIZE] = 0x0042; /* 'B' */
iter = ucol_openElements(coll, str, UCOL_EXPAND_CE_BUFFER_SIZE + 1,
&status);
if (ucol_previous(iter, &status) != UCOL_NULLORDER ||
status != U_BUFFER_OVERFLOW_ERROR) {
log_err("CE buffer expected to overflow with long string of trail surrogates\n");
if (ucol_previous(iter, &status) == UCOL_NULLORDER ||
status == U_BUFFER_OVERFLOW_ERROR) {
log_err("CE buffer should not overflow with long string of trail surrogates\n");
}
ucol_closeElements(iter);
ucol_close(coll);