organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-06 22:15:31 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 1

ICU-11537 move shared resource bundle string values to pool.res, .res formatVersion 3; merge --integrate from dev branch, plus make toolutil/pkgitems.cpp set poolBundleStrings so that icupkg can check dependencies

Browse source 
X-SVN-Rev: 37731
This commit is contained in:
Markus Scherer 2015-08-07 18:44:38 +00:00
parent 0a10ba1d7e
commit 36e5d9e75f
15 changed files with 742 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
icu4c/source/common/uresbund.cpp
View file

@ -4,7 +4,7 @@
* others. All Rights Reserved.
******************************************************************************
*
* File URESBUND.C
* File uresbund.cpp
*
* Modification History:
*
@ -377,6 +377,7 @@ static UResourceDataEntry *init_entry(const char *localeID, const char *path, UE
const int32_t *poolIndexes = r->fPool->fData.pRoot + 1;
if(r->fData.pRoot[1 + URES_INDEX_POOL_CHECKSUM] == poolIndexes[URES_INDEX_POOL_CHECKSUM]) {
r->fData.poolBundleKeys = (const char *)(poolIndexes + (poolIndexes[URES_INDEX_LENGTH] & 0xff));
r->fData.poolBundleStrings = r->fPool->fData.p16BitUnits;
} else {
r->fBogus = *status = U_INVALID_FORMAT_ERROR;
}

42
icu4c/source/common/uresdata.c
View file

@ -1,6 +1,6 @@
/*
*******************************************************************************
* Copyright (C) 1999-2014, International Business Machines Corporation
* Copyright (C) 1999-2015, International Business Machines Corporation
* and others. All Rights Reserved.
*******************************************************************************
* file name: uresdata.c
@ -145,7 +145,7 @@ isAcceptable(void *context,
pInfo->dataFormat[1]==0x65 &&
pInfo->dataFormat[2]==0x73 &&
pInfo->dataFormat[3]==0x42 &&
(pInfo->formatVersion[0]==1 || pInfo->formatVersion[0]==2));
(1<=pInfo->formatVersion[0] || pInfo->formatVersion[0]<=3));
}
/* semi-public functions ---------------------------------------------------- */
@ -198,11 +198,20 @@ res_init(ResourceData *pResData,
if(indexes[URES_INDEX_KEYS_TOP]>(1+indexLength)) {
pResData->localKeyLimit=indexes[URES_INDEX_KEYS_TOP]<<2;
}
if(formatVersion[0]>=3) {
// In formatVersion 1, the indexLength took up this whole int.
// In version 2, bits 31..8 were reserved and always 0.
// In version 3, they contain bits 23..0 of the poolStringIndexLimit.
// Bits 27..24 are in indexes[URES_INDEX_ATTRIBUTES] bits 15..12.
pResData->poolStringIndexLimit=(int32_t)((uint32_t)indexes[URES_INDEX_LENGTH]>>8);
}
if(indexLength>URES_INDEX_ATTRIBUTES) {
int32_t att=indexes[URES_INDEX_ATTRIBUTES];
pResData->noFallback=(UBool)(att&URES_ATT_NO_FALLBACK);
pResData->isPoolBundle=(UBool)((att&URES_ATT_IS_POOL_BUNDLE)!=0);
pResData->usesPoolBundle=(UBool)((att&URES_ATT_USES_POOL_BUNDLE)!=0);
pResData->poolStringIndexLimit|=(att&0xf000)<<12; // bits 15..12 -> 27..24
pResData->poolStringIndex16Limit=(int32_t)((uint32_t)att>>16);
}
if((pResData->isPoolBundle || pResData->usesPoolBundle) && indexLength<=URES_INDEX_POOL_CHECKSUM) {
*errorCode=U_INVALID_FORMAT_ERROR;
@ -301,7 +310,11 @@ res_getString(const ResourceData *pResData, Resource res, int32_t *pLength) {
int32_t length;
if(RES_GET_TYPE(res)==URES_STRING_V2) {
int32_t first;
p=(const UChar *)(pResData->p16BitUnits+offset);
if(offset<pResData->poolStringIndexLimit) {
p=(const UChar *)pResData->poolBundleStrings+offset;
} else {
p=(const UChar *)pResData->p16BitUnits+(offset-pResData->poolStringIndexLimit);
}
first=*p;
if(!U16_IS_TRAIL(first)) {
length=u_strlen(p);
@ -410,6 +423,18 @@ res_countArrayItems(const ResourceData *pResData, Resource res) {
}
}
static Resource
makeResourceFrom16(const ResourceData *pResData, int32_t res16) {
if(res16<pResData->poolStringIndex16Limit) {
// Pool string, nothing to do.
} else {
// Local string, adjust the 16-bit offset to a regular one,
// with a larger pool string index limit.
res16=res16-pResData->poolStringIndex16Limit+pResData->poolStringIndexLimit;
}
return URES_MAKE_RESOURCE(URES_STRING_V2, res16);
}
U_CAPI Resource U_EXPORT2
res_getTableItemByKey(const ResourceData *pResData, Resource table,
int32_t *indexR, const char **key) {
@ -437,7 +462,7 @@ res_getTableItemByKey(const ResourceData *pResData, Resource table,
length=*p++;
*indexR=idx=_res_findTableItem(pResData, p, length, *key, key);
if(idx>=0) {
return URES_MAKE_RESOURCE(URES_STRING_V2, p[length+idx]);
return makeResourceFrom16(pResData, p[length+idx]);
}
break;
}
@ -486,7 +511,7 @@ res_getTableItemByIndex(const ResourceData *pResData, Resource table,
if(key!=NULL) {
*key=RES_GET_KEY16(pResData, p[indexR]);
}
return URES_MAKE_RESOURCE(URES_STRING_V2, p[length+indexR]);
return makeResourceFrom16(pResData, p[length+indexR]);
}
break;
}
@ -533,7 +558,7 @@ res_getArrayItem(const ResourceData *pResData, Resource array, int32_t indexR) {
case URES_ARRAY16: {
const uint16_t *p=pResData->p16BitUnits+offset;
if(indexR<*p) {
return URES_MAKE_RESOURCE(URES_STRING_V2, p[1+indexR]);
return makeResourceFrom16(pResData, p[1+indexR]);
}
break;
}
@ -990,8 +1015,9 @@ ures_swap(const UDataSwapper *ds,
pInfo->dataFormat[1]==0x65 &&
pInfo->dataFormat[2]==0x73 &&
pInfo->dataFormat[3]==0x42 &&
((pInfo->formatVersion[0]==1 && pInfo->formatVersion[1]>=1) || /* formatVersion 1.1+ or 2.x */
pInfo->formatVersion[0]==2)
/* formatVersion 1.1+ or 2.x or 3.x */
((pInfo->formatVersion[0]==1 && pInfo->formatVersion[1]>=1) ||
pInfo->formatVersion[0]==2 || pInfo->formatVersion[0]==3)
)) {
udata_printError(ds, "ures_swap(): data format %02x.%02x.%02x.%02x (format version %02x.%02x) is not a resource bundle\n",
pInfo->dataFormat[0], pInfo->dataFormat[1],

116
icu4c/source/common/uresdata.h
View file

@ -1,9 +1,7 @@
/*
******************************************************************************
* *
* Copyright (C) 1999-2012, International Business Machines *
* Corporation and others. All Rights Reserved. *
* *
* Copyright (C) 1999-2015, International Business Machines
* Corporation and others. All Rights Reserved.
******************************************************************************
* file name: uresdata.h
* encoding: US-ASCII
@ -51,6 +49,8 @@ typedef enum {
* All values are URES_STRING_V2 strings.
*/
URES_ARRAY16=9
/* Resource type 15 is not defined but effectively used by RES_BOGUS=0xffffffff. */
} UResInternalType;
/*
@ -61,6 +61,7 @@ typedef enum {
typedef uint32_t Resource;
#define RES_BOGUS 0xffffffff
#define RES_MAX_OFFSET 0x0fffffff
#define RES_GET_TYPE(res) ((int32_t)((res)>>28UL))
#define RES_GET_OFFSET(res) ((res)&0x0fffffff)
@ -84,22 +85,48 @@ typedef uint32_t Resource;
/* indexes[] value names; indexes are generally 32-bit (Resource) indexes */
enum {
URES_INDEX_LENGTH, /* [0] contains URES_INDEX_TOP==the length of indexes[];
* formatVersion==1: all bits contain the length of indexes[]
* but the length is much less than 0xff;
* formatVersion>1:
* only bits 7..0 contain the length of indexes[],
* bits 31..8 are reserved and set to 0 */
URES_INDEX_KEYS_TOP, /* [1] contains the top of the key strings, */
/* same as the bottom of resources or UTF-16 strings, rounded up */
URES_INDEX_RESOURCES_TOP, /* [2] contains the top of all resources */
URES_INDEX_BUNDLE_TOP, /* [3] contains the top of the bundle, */
/* in case it were ever different from [2] */
URES_INDEX_MAX_TABLE_LENGTH,/* [4] max. length of any table */
URES_INDEX_ATTRIBUTES, /* [5] attributes bit set, see URES_ATT_* (new in formatVersion 1.2) */
URES_INDEX_16BIT_TOP, /* [6] top of the 16-bit units (UTF-16 string v2 UChars, URES_TABLE16, URES_ARRAY16),
* rounded up (new in formatVersion 2.0, ICU 4.4) */
URES_INDEX_POOL_CHECKSUM, /* [7] checksum of the pool bundle (new in formatVersion 2.0, ICU 4.4) */
/**
* [0] contains the length of indexes[]
* which is at most URES_INDEX_TOP of the latest format version
*
* formatVersion==1: all bits contain the length of indexes[]
* but the length is much less than 0xff;
* formatVersion>1:
* only bits 7..0 contain the length of indexes[],
* bits 31..8 are reserved and set to 0
* formatVersion>=3:
* bits 31..8 poolStringIndexLimit bits 23..0
*/
URES_INDEX_LENGTH,
/**
* [1] contains the top of the key strings,
* same as the bottom of resources or UTF-16 strings, rounded up
*/
URES_INDEX_KEYS_TOP,
/** [2] contains the top of all resources */
URES_INDEX_RESOURCES_TOP,
/**
* [3] contains the top of the bundle,
* in case it were ever different from [2]
*/
URES_INDEX_BUNDLE_TOP,
/** [4] max. length of any table */
URES_INDEX_MAX_TABLE_LENGTH,
/**
* [5] attributes bit set, see URES_ATT_* (new in formatVersion 1.2)
*
* formatVersion>=3:
* bits 31..16 poolStringIndex16Limit
* bits 15..12 poolStringIndexLimit bits 27..24
*/
URES_INDEX_ATTRIBUTES,
/**
* [6] top of the 16-bit units (UTF-16 string v2 UChars, URES_TABLE16, URES_ARRAY16),
* rounded up (new in formatVersion 2.0, ICU 4.4)
*/
URES_INDEX_16BIT_TOP,
/** [7] checksum of the pool bundle (new in formatVersion 2.0, ICU 4.4) */
URES_INDEX_POOL_CHECKSUM,
URES_INDEX_TOP
};
@ -126,9 +153,41 @@ enum {
#define URES_ATT_USES_POOL_BUNDLE 4
/*
* File format for .res resource bundle files (formatVersion=2, ICU 4.4)
* File format for .res resource bundle files
*
* New in formatVersion 2 compared with 1.3: -------------
* ICU 56: New in formatVersion 3 compared with 2: -------------
*
* Resource bundles can optionally use shared string-v2 values
* stored in the pool bundle.
* If so, then the indexes[] contain two new values
* in previously-unused bits of existing indexes[] slots:
* - poolStringIndexLimit:
* String-v2 offsets (in 32-bit Resource words) below this limit
* point to pool bundle string-v2 values.
* - poolStringIndex16Limit:
* Resource16 string-v2 offsets below this limit
* point to pool bundle string-v2 values.
* Guarantee: poolStringIndex16Limit <= poolStringIndexLimit
*
* The local bundle's poolStringIndexLimit is greater than
* any pool bundle string index used in the local bundle.
* The poolStringIndexLimit should not be greater than
* the maximum possible pool bundle string index.
*
* The maximum possible pool bundle string index is the index to the last non-NUL
* pool string character, due to suffix sharing.
*
* In the pool bundle, there is no structure that lists the strings.
* (The root resource is an empty Table.)
* If the strings need to be enumerated (as genrb --usePoolBundle does),
* then iterate through the pool bundle's 16-bit-units array from the beginning.
* Stop at the end of the array, or when an explicit or implicit string length
* would lead beyond the end of the array,
* or when an apparent string is not NUL-terminated.
* (Future genrb version might terminate the strings with
* what looks like a large explicit string length.)
*
* ICU 4.4: New in formatVersion 2 compared with 1.3: -------------
*
* Three new resource types -- String-v2, Table16 and Array16 -- have their
* values stored in a new array of 16-bit units between the table key strings
@ -270,8 +329,12 @@ enum {
* at 2-byte offsets from the start of a contiguous 16-bit-unit array between
* the table key strings and the other resources. (new in formatVersion 2/ICU 4.4)
* At offset 0 of that array is a 16-bit zero value for empty 16-bit resources.
*
* Resource16 values in Table16 and Array16 are 16-bit offsets to String-v2
* resources, with the offsets relative to the start of the 16-bit-units array.
* Starting with formatVersion 3/ICU 56, if offset<poolStringIndex16Limit
* then use the pool bundle's 16-bit-units array,
* otherwise subtract that limit and use the local 16-bit-units array.
*
* Type Name Memory layout of values
* (in parentheses: scalar, non-offset values)
@ -293,6 +356,12 @@ enum {
* - if first<0xdfff then length=((first-0xdfef)<<16) | second UChar
* - if first==0xdfff then length=((second UChar)<<16) | third UChar
* (stored in the 16-bit-units array; new in formatVersion 2/ICU 4.4)
*
* Starting with formatVersion 3/ICU 56, if offset<poolStringIndexLimit
* then use the pool bundle's 16-bit-units array,
* otherwise subtract that limit and use the local 16-bit-units array.
* (Note different limits for Resource16 vs. Resource.)
*
* 7 Integer: (28-bit offset is integer value)
* 8 Array: int32_t count, Resource[count]
* 9 Array16: uint16_t count, Resource16[count]
@ -317,6 +386,9 @@ typedef struct {
const char *poolBundleKeys;
Resource rootRes;
int32_t localKeyLimit;
const uint16_t *poolBundleStrings;
int32_t poolStringIndexLimit;
int32_t poolStringIndex16Limit;
UBool noFallback; /* see URES_ATT_NO_FALLBACK */
UBool isPoolBundle;
UBool usesPoolBundle;

BIN
icu4c/source/data/curr/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/lang/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/locales/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/region/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/unit/pool.res
View file

Binary file not shown.

BIN
icu4c/source/data/zone/pool.res
View file

Binary file not shown.

178
icu4c/source/tools/genrb/genrb.cpp
View file

@ -16,9 +16,11 @@
*******************************************************************************
*/
#include <assert.h>
#include "genrb.h"
#include "unicode/localpointer.h"
#include "unicode/uclean.h"
#include "unicode/utf16.h"
#include "reslist.h"
#include "ucmndata.h" /* TODO: for reading the pool bundle */
@ -41,8 +43,10 @@ const char *gCurrentFileName = theCurrentFileName;
#endif
void ResFile::close() {
uprv_free(fBytes);
delete[] fBytes;
fBytes = NULL;
delete fStrings;
fStrings = NULL;
}
enum
@ -136,7 +140,7 @@ main(int argc,
}
if(options[FORMAT_VERSION].doesOccur) {
const char *s = options[FORMAT_VERSION].value;
if(uprv_strlen(s) != 1 || (s[0] != '1' && s[0] != '2')) {
if(uprv_strlen(s) != 1 || (s[0] < '1' && '3' < s[0])) {
fprintf(stderr, "%s: unsupported --formatVersion %s\n", argv[0], s);
argc = -1;
} else if(s[0] == '1' &&
@ -336,62 +340,141 @@ main(int argc,
fprintf(stderr, "the pool bundle file %s is too small\n", theCurrentFileName);
return 1;
}
poolBundle.fBytes = (uint8_t *)uprv_malloc((poolFileSize + 15) & ~15);
poolBundle.fBytes = new uint8_t[(poolFileSize + 15) & ~15];
if (poolFileSize > 0 && poolBundle.fBytes == NULL) {
fprintf(stderr, "unable to allocate memory for the pool bundle file %s\n", theCurrentFileName);
return U_MEMORY_ALLOCATION_ERROR;
} else {
UDataSwapper *ds;
const DataHeader *header;
int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize);
int32_t keysBottom;
if (bytesRead != poolFileSize) {
fprintf(stderr, "unable to read the pool bundle file %s\n", theCurrentFileName);
return 1;
}
/*
* Swap the pool bundle so that a single checked-in file can be used.
* The swapper functions also test that the data looks like
* a well-formed .res file.
*/
ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead,
U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status);
udata_closeSwapper(ds);
if (U_FAILURE(status)) {
fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
header = (const DataHeader *)poolBundle.fBytes;
if (header->info.formatVersion[0]!=2) {
fprintf(stderr, "invalid format of pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
poolBundle.fKeys = (const char *)header + header->dataHeader.headerSize;
poolBundle.fIndexes = (const int32_t *)poolBundle.fKeys + 1;
indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff;
if (indexLength <= URES_INDEX_POOL_CHECKSUM) {
fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
keysBottom = (1 + indexLength) * 4;
poolBundle.fKeys += keysBottom;
poolBundle.fKeysLength = (poolBundle.fIndexes[URES_INDEX_KEYS_TOP] * 4) - keysBottom;
poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM];
}
UDataSwapper *ds;
const DataHeader *header;
int32_t bytesRead = T_FileStream_read(poolFile, poolBundle.fBytes, poolFileSize);
if (bytesRead != poolFileSize) {
fprintf(stderr, "unable to read the pool bundle file %s\n", theCurrentFileName);
return 1;
}
/*
* Swap the pool bundle so that a single checked-in file can be used.
* The swapper functions also test that the data looks like
* a well-formed .res file.
*/
ds = udata_openSwapperForInputData(poolBundle.fBytes, bytesRead,
U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &status);
if (U_FAILURE(status)) {
fprintf(stderr, "udata_openSwapperForInputData(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
ures_swap(ds, poolBundle.fBytes, bytesRead, poolBundle.fBytes, &status);
udata_closeSwapper(ds);
if (U_FAILURE(status)) {
fprintf(stderr, "ures_swap(pool bundle %s) failed: %s\n",
theCurrentFileName, u_errorName(status));
return status;
}
header = (const DataHeader *)poolBundle.fBytes;
if (header->info.formatVersion[0] < 2) {
fprintf(stderr, "invalid format of pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
const int32_t *pRoot = (const int32_t *)(
(const char *)header + header->dataHeader.headerSize);
poolBundle.fIndexes = pRoot + 1;
indexLength = poolBundle.fIndexes[URES_INDEX_LENGTH] & 0xff;
if (indexLength <= URES_INDEX_POOL_CHECKSUM) {
fprintf(stderr, "insufficient indexes[] in pool bundle file %s\n", theCurrentFileName);
return U_INVALID_FORMAT_ERROR;
}
int32_t keysBottom = 1 + indexLength;
int32_t keysTop = poolBundle.fIndexes[URES_INDEX_KEYS_TOP];
poolBundle.fKeys = (const char *)(pRoot + keysBottom);
poolBundle.fKeysLength = (keysTop - keysBottom) * 4;
poolBundle.fChecksum = poolBundle.fIndexes[URES_INDEX_POOL_CHECKSUM];
for (i = 0; i < poolBundle.fKeysLength; ++i) {
if (poolBundle.fKeys[i] == 0) {
++poolBundle.fKeysCount;
}
}
// 16BitUnits[] begins with strings-v2.
// The strings-v2 may optionally be terminated by what looks like
// an explicit string length that exceeds the number of remaining 16-bit units.
int32_t stringUnitsLength = (poolBundle.fIndexes[URES_INDEX_16BIT_TOP] - keysTop) * 2;
if (stringUnitsLength >= 2 && getFormatVersion() >= 3) {
poolBundle.fStrings = new PseudoListResource(NULL, status);
if (poolBundle.fStrings == NULL) {
fprintf(stderr, "unable to allocate memory for the pool bundle strings %s\n", theCurrentFileName);
return U_MEMORY_ALLOCATION_ERROR;
}
// The PseudoListResource constructor call did not allocate further memory.
assert(U_SUCCESS(status));
const UChar *p = (const UChar *)(pRoot + keysTop);
int32_t remaining = stringUnitsLength;
do {
int32_t first = *p;
int8_t numCharsForLength;
int32_t length;
if (!U16_IS_TRAIL(first)) {
// NUL-terminated
numCharsForLength = 0;
for (length = 0;
length < remaining && p[length] != 0;
++length) {}
} else if (first < 0xdfef) {
numCharsForLength = 1;
length = first & 0x3ff;
} else if (first < 0xdfff && remaining >= 2) {
numCharsForLength = 2;
length = ((first - 0xdfef) << 16) | p[1];
} else if (first == 0xdfff && remaining >= 3) {
numCharsForLength = 3;
length = ((int32_t)p[1] << 16) | p[2];
} else {
break; // overrun
}
// Check for overrun before changing remaining,
// so that it is always accurate after the loop body.
if ((numCharsForLength + length) >= remaining ||
p[numCharsForLength + length] != 0) {
break; // overrun or explicitly terminated
}
int32_t poolStringIndex = stringUnitsLength - remaining;
// Maximum pool string index when suffix-sharing the last character.
int32_t maxStringIndex = poolStringIndex + numCharsForLength + length - 1;
if (maxStringIndex >= RES_MAX_OFFSET) {
// pool string index overrun
break;
}
p += numCharsForLength;
remaining -= numCharsForLength;
if (length != 0) {
StringResource *sr =
new StringResource(poolStringIndex, numCharsForLength,
p, length, status);
if (sr == NULL) {
fprintf(stderr, "unable to allocate memory for a pool bundle string %s\n",
theCurrentFileName);
return U_MEMORY_ALLOCATION_ERROR;
}
poolBundle.fStrings->add(sr);
poolBundle.fStringIndexLimit = maxStringIndex + 1;
// The StringResource constructor did not allocate further memory.
assert(U_SUCCESS(status));
}
p += length + 1;
remaining -= length + 1;
} while (remaining > 0);
}
T_FileStream_close(poolFile);
setUsePoolBundle(TRUE);
if (isVerbose() && poolBundle.fStrings != NULL) {
printf("number of shared strings: %d\n", (int)poolBundle.fStrings->fCount);
int32_t length = poolBundle.fStringIndexLimit + 1; // incl. last NUL
printf("16-bit units for strings: %6d = %6d bytes\n",
(int)length, (int)length * 2);
}
}
if(options[INCLUDE_UNIHAN_COLL].doesOccur) {
@ -565,6 +648,7 @@ processFile(const char *filename, const char *cp,
goto finish;
}
if(options[WRITE_POOL_BUNDLE].doesOccur) {
data->fWritePoolBundle = newPoolBundle;
data->compactKeys(*status);
int32_t newKeysLength;
const char *newKeys = data->getKeyBytes(&newKeysLength);

2
icu4c/source/tools/genrb/genrb.h
View file

@ -32,7 +32,7 @@
#include "unicode/uloc.h"
/* The version of genrb */
#define GENRB_VERSION "3.3"
#define GENRB_VERSION "56"
U_CDECL_BEGIN

577
icu4c/source/tools/genrb/reslist.cpp
View file

@ -46,21 +46,33 @@
*/
#define BIN_ALIGNMENT 16
// This numeric constant must be at least 1.
// If StringResource.fNumUnitsSaved == 0 then the string occurs only once,
// and it makes no sense to move it to the pool bundle.
// The larger the threshold for fNumUnitsSaved
// the smaller the savings, and the smaller the pool bundle.
// We trade some total size reduction to reduce the pool bundle a bit,
// so that one can reasonably save data size by
// removing bundle files without rebuilding the pool bundle.
// This can also help to keep the pool and total (pool+local) string indexes
// within 16 bits, that is, within range of Table16 and Array16 containers.
#ifndef GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING
# define GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING 10
#endif
U_NAMESPACE_USE
static UBool gIncludeCopyright = FALSE;
static UBool gUsePoolBundle = FALSE;
static int32_t gFormatVersion = 2;
static int32_t gFormatVersion = 3;
/* How do we store string values? */
enum {
STRINGS_UTF16_V1, /* formatVersion 1: int length + UChars + NUL + padding to 4 bytes */
STRINGS_UTF16_V2 /* formatVersion 2: optional length in 1..3 UChars + UChars + NUL */
STRINGS_UTF16_V2 /* formatVersion 2 & up: optional length in 1..3 UChars + UChars + NUL */
};
enum {
MAX_IMPLICIT_STRING_LENGTH = 40 /* do not store the length explicitly for such strings */
};
static const int32_t MAX_IMPLICIT_STRING_LENGTH = 40; /* do not store the length explicitly for such strings */
static const ResFile kNoPoolBundle;
@ -85,11 +97,14 @@ static UDataInfo dataInfo= {
{1, 4, 0, 0} /* dataVersion take a look at version inside parsed resb*/
};
static const UVersionInfo gFormatVersions[3] = { /* indexed by a major-formatVersion integer */
static const UVersionInfo gFormatVersions[4] = { /* indexed by a major-formatVersion integer */
{ 0, 0, 0, 0 },
{ 1, 3, 0, 0 },
{ 2, 0, 0, 0 }
{ 2, 0, 0, 0 },
{ 3, 0, 0, 0 }
};
// Remember to update genrb.h GENRB_VERSION when changing the data format.
// (Or maybe we should remove GENRB_VERSION and report the ICU version number?)
static uint8_t calcPadding(uint32_t size) {
/* returns space we need to pad */
@ -109,6 +124,10 @@ void setFormatVersion(int32_t formatVersion) {
gFormatVersion = formatVersion;
}
int32_t getFormatVersion() {
return gFormatVersion;
}
void setUsePoolBundle(UBool use) {
gUsePoolBundle = use;
}
@ -119,15 +138,15 @@ struct SResource* res_none() {
}
SResource::SResource()
: fType(URES_NONE), fWritten(FALSE), fRes(RES_BOGUS), fKey(-1), line(0),
fNext(NULL) {
: fType(URES_NONE), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1), fKey(-1), fKey16(-1),
line(0), fNext(NULL) {
ustr_init(&fComment);
}
SResource::SResource(SRBRoot *bundle, const char *tag, int8_t type, const UString* comment,
UErrorCode &errorCode)
: fType(type), fWritten(FALSE), fRes(RES_BOGUS),
fKey(bundle->addTag(tag, errorCode)),
: fType(type), fWritten(FALSE), fRes(RES_BOGUS), fRes16(-1),
fKey(bundle != NULL ? bundle->addTag(tag, errorCode) : -1), fKey16(-1),
line(0), fNext(NULL) {
ustr_init(&fComment);
if(comment != NULL) {
@ -228,6 +247,16 @@ void ArrayResource::add(SResource *res) {
}
}
PseudoListResource::~PseudoListResource() {}
void PseudoListResource::add(SResource *res) {
if (res != NULL && res != &kNoResource) {
res->fNext = fFirst;
fFirst = res;
++fCount;
}
}
StringBaseResource::StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type,
const UChar *value, int32_t len,
const UString* comment, UErrorCode &errorCode)
@ -245,6 +274,29 @@ StringBaseResource::StringBaseResource(SRBRoot *bundle, const char *tag, int8_t
}
}
StringBaseResource::StringBaseResource(SRBRoot *bundle, int8_t type,
const icu::UnicodeString &value, UErrorCode &errorCode)
: SResource(bundle, NULL, type, NULL, errorCode), fString(value) {
if (value.isEmpty() && gFormatVersion > 1) {
fRes = URES_MAKE_EMPTY_RESOURCE(type);
fWritten = TRUE;
return;
}
fString.getTerminatedBuffer(); // Some code relies on NUL-termination.
if (U_SUCCESS(errorCode) && fString.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
}
}
// Pool bundle string, alias the buffer. Guaranteed NUL-terminated and not empty.
StringBaseResource::StringBaseResource(int8_t type, const UChar *value, int32_t len,
UErrorCode &errorCode)
: SResource(NULL, NULL, type, NULL, errorCode), fString(TRUE, value, len) {
assert(len > 0);
assert(!fString.isBogus());
}
StringBaseResource::~StringBaseResource() {}
static int32_t U_CALLCONV
@ -268,7 +320,7 @@ IntResource::IntResource(SRBRoot *bundle, const char *tag, int32_t value,
const UString* comment, UErrorCode &errorCode)
: SResource(bundle, tag, URES_INT, comment, errorCode) {
fValue = value;
fRes = URES_MAKE_RESOURCE(URES_INT, value & 0x0FFFFFFF);
fRes = URES_MAKE_RESOURCE(URES_INT, value & RES_MAX_OFFSET);
fWritten = TRUE;
}
@ -335,17 +387,27 @@ BinaryResource::~BinaryResource() {
void
StringResource::handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet,
UErrorCode &errorCode) {
assert(fSame == NULL);
fSame = static_cast<StringResource *>(uhash_get(stringSet, this));
if (fSame != NULL) {
return; /* This is a duplicate of an earlier-visited string. */
// This is a duplicate of a pool bundle string or of an earlier-visited string.
if (++fSame->fNumCopies == 1) {
assert(fSame->fWritten);
int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(fSame->fRes);
if (poolStringIndex >= bundle->fPoolStringIndexLimit) {
bundle->fPoolStringIndexLimit = poolStringIndex + 1;
}
}
return;
}
/* Put this string into the set for finding duplicates. */
fNumCopies = 1;
uhash_put(stringSet, this, this, &errorCode);
if (bundle->fStringsForm != STRINGS_UTF16_V1) {
const UChar *s = getBuffer();
int32_t len = length();
if (len <= MAX_IMPLICIT_STRING_LENGTH && !U16_IS_TRAIL(s[0]) && len == u_strlen(s)) {
if (len <= MAX_IMPLICIT_STRING_LENGTH &&
!U16_IS_TRAIL(fString[0]) && fString.indexOf((UChar)0) < 0) {
/*
* This string will be stored without an explicit length.
* Runtime will detect !U16_IS_TRAIL(s[0]) and call u_strlen().
@ -392,16 +454,25 @@ SResource::handlePreflightStrings(SRBRoot * /*bundle*/, UHashtable * /*stringSet
/* Neither a string nor a container. */
}
static int32_t
makeRes16(uint32_t resWord) {
uint32_t type, offset;
int32_t
SRBRoot::makeRes16(uint32_t resWord) const {
if (resWord == 0) {
return 0; /* empty string */
}
type = RES_GET_TYPE(resWord);
offset = RES_GET_OFFSET(resWord);
if (type == URES_STRING_V2 && offset <= 0xffff) {
return (int32_t)offset;
uint32_t type = RES_GET_TYPE(resWord);
int32_t offset = (int32_t)RES_GET_OFFSET(resWord);
if (type == URES_STRING_V2) {
assert(offset > 0);
if (offset < fPoolStringIndexLimit) {
if (offset < fPoolStringIndex16Limit) {
return offset;
}
} else {
offset = offset - fPoolStringIndexLimit + fPoolStringIndex16Limit;
if (offset <= 0xffff) {
return offset;
}
}
}
return -1;
}
@ -429,22 +500,13 @@ SRBRoot::mapKey(int32_t oldpos) const {
return map[start].newpos;
}
uint16_t
SRBRoot::makeKey16(int32_t key) const {
if (key >= 0) {
return (uint16_t)key;
} else {
return (uint16_t)(key + fLocalKeyLimit); /* offset in the pool bundle */
}
}
/*
* Only called for UTF-16 v1 strings and duplicate UTF-16 v2 strings.
* For unique UTF-16 v2 strings, write16() sees fRes != RES_BOGUS
* and exits early.
*/
void
StringResource::handleWrite16(SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
StringResource::handleWrite16(SRBRoot * /*bundle*/) {
SResource *same;
if ((same = fSame) != NULL) {
/* This is a duplicate. */
@ -455,7 +517,15 @@ StringResource::handleWrite16(SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/)
}
void
ArrayResource::handleWrite16(SRBRoot *bundle, UErrorCode &errorCode) {
ContainerResource::writeAllRes16(SRBRoot *bundle) {
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
bundle->f16BitUnits.append((UChar)current->fRes16);
}
fWritten = TRUE;
}
void
ArrayResource::handleWrite16(SRBRoot *bundle) {
if (fCount == 0 && gFormatVersion > 1) {
fRes = URES_MAKE_EMPTY_RESOURCE(URES_ARRAY);
fWritten = TRUE;
@ -464,76 +534,43 @@ ArrayResource::handleWrite16(SRBRoot *bundle, UErrorCode &errorCode) {
int32_t res16 = 0;
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
current->write16(bundle, errorCode);
res16 |= makeRes16(current->fRes);
current->write16(bundle);
res16 |= current->fRes16;
}
if (U_SUCCESS(errorCode) && fCount <= 0xffff && res16 >= 0 && gFormatVersion > 1) {
if (fCount <= 0xffff && res16 >= 0 && gFormatVersion > 1) {
fRes = URES_MAKE_RESOURCE(URES_ARRAY16, bundle->f16BitUnits.length());
bundle->f16BitUnits.append((UChar)fCount);
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
bundle->f16BitUnits.append((UChar)makeRes16(current->fRes));
}
fWritten = TRUE;
if (bundle->f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
}
writeAllRes16(bundle);
}
}
void
TableResource::handleWrite16(SRBRoot *bundle, UErrorCode &errorCode) {
int32_t maxKey = 0, maxPoolKey = 0x80000000;
int32_t res16 = 0;
UBool hasLocalKeys = FALSE, hasPoolKeys = FALSE;
TableResource::handleWrite16(SRBRoot *bundle) {
if (fCount == 0 && gFormatVersion > 1) {
fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE);
fWritten = TRUE;
return;
}
/* Find the smallest table type that fits the data. */
int32_t key16 = 0;
int32_t res16 = 0;
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
int32_t key;
current->write16(bundle, errorCode);
key = current->fKey = bundle->mapKey(current->fKey);
if (key >= 0) {
hasLocalKeys = TRUE;
if (key > maxKey) {
maxKey = key;
}
} else {
hasPoolKeys = TRUE;
if (key > maxPoolKey) {
maxPoolKey = key;
}
}
res16 |= makeRes16(current->fRes);
}
if (U_FAILURE(errorCode)) {
return;
current->write16(bundle);
key16 |= current->fKey16;
res16 |= current->fRes16;
}
if(fCount > (uint32_t)bundle->fMaxTableLength) {
bundle->fMaxTableLength = fCount;
}
maxPoolKey &= 0x7fffffff;
if (fCount <= 0xffff &&
(!hasLocalKeys || maxKey < bundle->fLocalKeyLimit) &&
(!hasPoolKeys || maxPoolKey < (0x10000 - bundle->fLocalKeyLimit))
) {
if (fCount <= 0xffff && key16 >= 0) {
if (res16 >= 0 && gFormatVersion > 1) {
/* 16-bit count, key offsets and values */
fRes = URES_MAKE_RESOURCE(URES_TABLE16, bundle->f16BitUnits.length());
bundle->f16BitUnits.append((UChar)fCount);
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
bundle->f16BitUnits.append((UChar)bundle->makeKey16(current->fKey));
}
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
bundle->f16BitUnits.append((UChar)makeRes16(current->fRes));
}
fWritten = TRUE;
if (bundle->f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
bundle->f16BitUnits.append((UChar)current->fKey16);
}
writeAllRes16(bundle);
} else {
/* 16-bit count, 16-bit key offsets, 32-bit values */
fTableType = URES_TABLE;
@ -545,24 +582,51 @@ TableResource::handleWrite16(SRBRoot *bundle, UErrorCode &errorCode) {
}
void
SResource::write16(SRBRoot *bundle, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
if (fRes != RES_BOGUS) {
/*
* The resource item word was already precomputed, which means
* no further data needs to be written.
* This might be an integer, or an empty or UTF-16 v2 string,
* an empty binary, etc.
*/
return;
}
handleWrite16(bundle, errorCode);
PseudoListResource::handleWrite16(SRBRoot * /*bundle*/) {
fRes = URES_MAKE_EMPTY_RESOURCE(URES_TABLE);
fWritten = TRUE;
}
void
SResource::handleWrite16(SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
SResource::write16(SRBRoot *bundle) {
if (fKey >= 0) {
// A tagged resource has a non-negative key index into the parsed key strings.
// compactKeys() built a map from parsed key index to the final key index.
// After the mapping, negative key indexes are used for shared pool bundle keys.
fKey = bundle->mapKey(fKey);
// If the key index fits into a Key16 for a Table or Table16,
// then set the fKey16 field accordingly.
// Otherwise keep it at -1.
if (fKey >= 0) {
if (fKey < bundle->fLocalKeyLimit) {
fKey16 = fKey;
}
} else {
int32_t poolKeyIndex = fKey & 0x7fffffff;
if (poolKeyIndex <= 0xffff) {
poolKeyIndex += bundle->fLocalKeyLimit;
if (poolKeyIndex <= 0xffff) {
fKey16 = poolKeyIndex;
}
}
}
}
/*
* fRes != RES_BOGUS:
* The resource item word was already precomputed, which means
* no further data needs to be written.
* This might be an integer, or an empty or UTF-16 v2 string,
* an empty binary, etc.
*/
if (fRes == RES_BOGUS) {
handleWrite16(bundle);
}
// Compute fRes16 for precomputed as well as just-computed fRes.
fRes16 = bundle->makeRes16(fRes);
}
void
SResource::handleWrite16(SRBRoot * /*bundle*/) {
/* Only a few resource types write 16-bit units. */
}
@ -572,16 +636,14 @@ SResource::handleWrite16(SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
* and exits early.
*/
void
StringBaseResource::handlePreWrite(uint32_t *byteOffset, SRBRoot * /*bundle*/,
UErrorCode & /*errorCode*/) {
StringBaseResource::handlePreWrite(uint32_t *byteOffset) {
/* Write the UTF-16 v1 string. */
fRes = URES_MAKE_RESOURCE(fType, *byteOffset >> 2);
*byteOffset += 4 + (length() + 1) * U_SIZEOF_UCHAR;
}
void
IntVectorResource::handlePreWrite(uint32_t *byteOffset, SRBRoot * /*bundle*/,
UErrorCode & /*errorCode*/) {
IntVectorResource::handlePreWrite(uint32_t *byteOffset) {
if (fCount == 0 && gFormatVersion > 1) {
fRes = URES_MAKE_EMPTY_RESOURCE(URES_INT_VECTOR);
fWritten = TRUE;
@ -592,8 +654,7 @@ IntVectorResource::handlePreWrite(uint32_t *byteOffset, SRBRoot * /*bundle*/,
}
void
BinaryResource::handlePreWrite(uint32_t *byteOffset, SRBRoot * /*bundle*/,
UErrorCode & /*errorCode*/) {
BinaryResource::handlePreWrite(uint32_t *byteOffset) {
uint32_t pad = 0;
uint32_t dataStart = *byteOffset + sizeof(fLength);
@ -606,19 +667,22 @@ BinaryResource::handlePreWrite(uint32_t *byteOffset, SRBRoot * /*bundle*/,
}
void
ArrayResource::handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode) {
ContainerResource::preWriteAllRes(uint32_t *byteOffset) {
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
current->preWrite(byteOffset, bundle, errorCode);
current->preWrite(byteOffset);
}
}
void
ArrayResource::handlePreWrite(uint32_t *byteOffset) {
preWriteAllRes(byteOffset);
fRes = URES_MAKE_RESOURCE(URES_ARRAY, *byteOffset >> 2);
*byteOffset += (1 + fCount) * 4;
}
void
TableResource::handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode) {
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
current->preWrite(byteOffset, bundle, errorCode);
}
TableResource::handlePreWrite(uint32_t *byteOffset) {
preWriteAllRes(byteOffset);
if (fTableType == URES_TABLE) {
/* 16-bit count, 16-bit key offsets, 32-bit values */
fRes = URES_MAKE_RESOURCE(URES_TABLE, *byteOffset >> 2);
@ -631,10 +695,7 @@ TableResource::handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode
}
void
SResource::preWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
SResource::preWrite(uint32_t *byteOffset) {
if (fRes != RES_BOGUS) {
/*
* The resource item word was already precomputed, which means
@ -644,14 +705,13 @@ SResource::preWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode
*/
return;
}
handlePreWrite(byteOffset, bundle, errorCode);
handlePreWrite(byteOffset);
*byteOffset += calcPadding(*byteOffset);
}
void
SResource::handlePreWrite(uint32_t * /*byteOffset*/, SRBRoot * /*bundle*/,
UErrorCode &errorCode) {
errorCode = U_INTERNAL_PROGRAM_ERROR;
SResource::handlePreWrite(uint32_t * /*byteOffset*/) {
assert(FALSE);
}
/*
@ -659,8 +719,7 @@ SResource::handlePreWrite(uint32_t * /*byteOffset*/, SRBRoot * /*bundle*/,
* write() sees fWritten and exits early.
*/
void
StringBaseResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
StringBaseResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
/* Write the UTF-16 v1 string. */
int32_t len = length();
udata_write32(mem, len);
@ -670,24 +729,32 @@ StringBaseResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
}
void
ArrayResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot *bundle, UErrorCode &errorCode) {
ContainerResource::writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset) {
uint32_t i = 0;
for (SResource *current = fFirst; current != NULL; ++i, current = current->fNext) {
current->write(mem, byteOffset, bundle, errorCode);
current->write(mem, byteOffset);
}
assert(i == fCount);
udata_write32(mem, fCount);
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
udata_write32(mem, current->fRes);
}
*byteOffset += (1 + fCount) * 4;
}
void
IntVectorResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
ContainerResource::writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset) {
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
udata_write32(mem, current->fRes);
}
*byteOffset += fCount * 4;
}
void
ArrayResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
writeAllRes(mem, byteOffset);
udata_write32(mem, fCount);
*byteOffset += 4;
writeAllRes32(mem, byteOffset);
}
void
IntVectorResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
udata_write32(mem, fCount);
for(uint32_t i = 0; i < fCount; ++i) {
udata_write32(mem, fArray[i]);
@ -696,8 +763,7 @@ IntVectorResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
}
void
BinaryResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot * /*bundle*/, UErrorCode & /*errorCode*/) {
BinaryResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
uint32_t pad = 0;
uint32_t dataStart = *byteOffset + sizeof(fLength);
@ -715,19 +781,12 @@ BinaryResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
}
void
TableResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot *bundle, UErrorCode &errorCode) {
uint32_t i = 0;
for (SResource *current = fFirst; current != NULL; ++i, current = current->fNext) {
assert(i < fCount);
current->write(mem, byteOffset, bundle, errorCode);
}
assert(i == fCount);
TableResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset) {
writeAllRes(mem, byteOffset);
if(fTableType == URES_TABLE) {
udata_write16(mem, (uint16_t)fCount);
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
udata_write16(mem, bundle->makeKey16(current->fKey));
udata_write16(mem, current->fKey16);
}
*byteOffset += (1 + fCount)* 2;
if ((fCount & 1) == 0) {
@ -742,23 +801,16 @@ TableResource::handleWrite(UNewDataMemory *mem, uint32_t *byteOffset,
}
*byteOffset += (1 + fCount)* 4;
}
for (SResource *current = fFirst; current != NULL; current = current->fNext) {
udata_write32(mem, current->fRes);
}
*byteOffset += fCount * 4;
writeAllRes32(mem, byteOffset);
}
void
SResource::write(UNewDataMemory *mem, uint32_t *byteOffset,
SRBRoot *bundle, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
SResource::write(UNewDataMemory *mem, uint32_t *byteOffset) {
if (fWritten) {
assert(fRes != RES_BOGUS);
return;
}
handleWrite(mem, byteOffset, bundle, errorCode);
handleWrite(mem, byteOffset);
uint8_t paddingSize = calcPadding(*byteOffset);
if (paddingSize > 0) {
udata_writePadding(mem, paddingSize);
@ -768,9 +820,8 @@ SResource::write(UNewDataMemory *mem, uint32_t *byteOffset,
}
void
SResource::handleWrite(UNewDataMemory * /*mem*/, uint32_t * /*byteOffset*/,
SRBRoot * /*bundle*/, UErrorCode &errorCode) {
errorCode = U_INTERNAL_PROGRAM_ERROR;
SResource::handleWrite(UNewDataMemory * /*mem*/, uint32_t * /*byteOffset*/) {
assert(FALSE);
}
void SRBRoot::write(const char *outputDir, const char *outputPkg,
@ -811,6 +862,17 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
UHashtable *stringSet;
if (gFormatVersion > 1) {
stringSet = uhash_open(string_hash, string_comp, string_comp, &errorCode);
if (U_SUCCESS(errorCode) &&
fUsePoolBundle != NULL && fUsePoolBundle->fStrings != NULL) {
for (SResource *current = fUsePoolBundle->fStrings->fFirst;
current != NULL;
current = current->fNext) {
StringResource *sr = static_cast<StringResource *>(current);
sr->fNumCopies = 0;
sr->fNumUnitsSaved = 0;
uhash_put(stringSet, sr, sr, &errorCode);
}
}
fRoot->preflightStrings(this, stringSet, errorCode);
} else {
stringSet = NULL;
@ -823,7 +885,28 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
return;
}
fRoot->write16(this, errorCode);
if (fPoolStringIndexLimit != 0) {
int32_t sum = fPoolStringIndexLimit + fLocalStringIndexLimit;
if ((sum - 1) > RES_MAX_OFFSET) {
errorCode = U_BUFFER_OVERFLOW_ERROR;
return;
}
if (fPoolStringIndexLimit < 0x10000 && sum <= 0x10000) {
// 16-bit indexes work for all pool + local strings.
fPoolStringIndex16Limit = fPoolStringIndexLimit;
} else {
// Set the pool index threshold so that 16-bit indexes work
// for some pool strings and some local strings.
fPoolStringIndex16Limit = (int32_t)(
((int64_t)fPoolStringIndexLimit * 0xffff) / sum);
}
}
fRoot->write16(this);
if (f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
if (f16BitUnits.length() & 1) {
f16BitUnits.append((UChar)0xaaaa); /* pad to multiple of 4 bytes */
}
@ -832,15 +915,11 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
fKeyMap = NULL;
byteOffset = fKeysTop + f16BitUnits.length() * 2;
fRoot->preWrite(&byteOffset, this, errorCode);
fRoot->preWrite(&byteOffset);
/* total size including the root item */
top = byteOffset;
if (U_FAILURE(errorCode)) {
return;
}
if (writtenFilename && writtenFilenameLen) {
*writtenFilename = 0;
}
@ -908,7 +987,7 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
/*
* formatVersion 1.1 (ICU 2.8):
* write int32_t indexes[] after root and before the strings
* write int32_t indexes[] after root and before the key strings
* to make it easier to parse resource bundles in icuswap or from Java etc.
*/
uprv_memset(indexes, 0, sizeof(indexes));
@ -936,15 +1015,38 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
if (URES_INDEX_POOL_CHECKSUM < fIndexLength) {
if (fIsPoolBundle) {
indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_IS_POOL_BUNDLE | URES_ATT_NO_FALLBACK;
indexes[URES_INDEX_POOL_CHECKSUM] =
(int32_t)computeCRC((char *)(fKeys + fKeysBottom),
(uint32_t)(fKeysTop - fKeysBottom),
0);
uint32_t checksum = computeCRC((const char *)(fKeys + fKeysBottom),
(uint32_t)(fKeysTop - fKeysBottom), 0);
if (f16BitUnits.length() <= 1) {
// no pool strings to checksum
} else if (U_IS_BIG_ENDIAN) {
checksum = computeCRC((const char *)f16BitUnits.getBuffer(),
(uint32_t)f16BitUnits.length() * 2, checksum);
} else {
// Swap to big-endian so we get the same checksum on all platforms
// (except for charset family, due to the key strings).
UnicodeString s(f16BitUnits);
s.append((UChar)1); // Ensure that we own this buffer.
assert(!s.isBogus());
uint16_t *p = (uint16_t *)s.getBuffer();
for (int32_t count = f16BitUnits.length(); count > 0; --count) {
uint16_t x = *p;
*p++ = (uint16_t)((x << 8) | (x >> 8));
}
checksum = computeCRC((const char *)p,
(uint32_t)f16BitUnits.length() * 2, checksum);
}
indexes[URES_INDEX_POOL_CHECKSUM] = (int32_t)checksum;
} else if (gUsePoolBundle) {
indexes[URES_INDEX_ATTRIBUTES] |= URES_ATT_USES_POOL_BUNDLE;
indexes[URES_INDEX_POOL_CHECKSUM] = fUsePoolBundle->fChecksum;
}
}
// formatVersion 3 (ICU 56):
// share string values via pool bundle strings
indexes[URES_INDEX_LENGTH] |= fPoolStringIndexLimit << 8; // bits 23..0 -> 31..8
indexes[URES_INDEX_ATTRIBUTES] |= (fPoolStringIndexLimit >> 12) & 0xf000; // bits 27..24 -> 15..12
indexes[URES_INDEX_ATTRIBUTES] |= fPoolStringIndex16Limit << 16;
/* write the indexes[] */
udata_writeBlock(mem, indexes, fIndexLength*4);
@ -958,7 +1060,7 @@ void SRBRoot::write(const char *outputDir, const char *outputPkg,
/* write all of the bundle contents: the root item and its children */
byteOffset = fKeysTop + f16BitUnits.length() * 2;
fRoot->write(mem, &byteOffset, this, errorCode);
fRoot->write(mem, &byteOffset);
assert(byteOffset == top);
size = udata_finish(mem, &errorCode);
@ -1016,7 +1118,9 @@ SRBRoot::SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCo
fKeys(NULL), fKeyMap(NULL),
fKeysBottom(0), fKeysTop(0), fKeysCapacity(0), fKeysCount(0), fLocalKeyLimit(0),
f16BitUnits(), f16BitStringsLength(0),
fUsePoolBundle(&kNoPoolBundle) {
fUsePoolBundle(&kNoPoolBundle),
fPoolStringIndexLimit(0), fPoolStringIndex16Limit(0), fLocalStringIndexLimit(0),
fWritePoolBundle(NULL) {
if (U_FAILURE(errorCode)) {
return;
}
@ -1028,7 +1132,11 @@ SRBRoot::SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCo
}
fKeys = (char *) uprv_malloc(sizeof(char) * KEY_SPACE_SIZE);
fRoot = new TableResource(this, NULL, comment, errorCode);
if (isPoolBundle) {
fRoot = new PseudoListResource(this, errorCode);
} else {
fRoot = new TableResource(this, NULL, comment, errorCode);
}
if (fKeys == NULL || fRoot == NULL || U_FAILURE(errorCode)) {
if (U_SUCCESS(errorCode)) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
@ -1037,7 +1145,7 @@ SRBRoot::SRBRoot(const UString *comment, UBool isPoolBundle, UErrorCode &errorCo
}
fKeysCapacity = KEY_SPACE_SIZE;
/* formatVersion 1.1: start fKeysTop after the root item and indexes[] */
/* formatVersion 1.1 and up: start fKeysTop after the root item and indexes[] */
if (gUsePoolBundle || isPoolBundle) {
fIndexLength = URES_INDEX_POOL_CHECKSUM + 1;
} else if (gFormatVersion >= 2) {
@ -1365,13 +1473,23 @@ compareStringLengths(const void * /*context*/, const void *l, const void *r) {
return diff;
}
/* sort by ascending string length */
return left->length() - right->length();
diff = left->length() - right->length();
if (diff != 0) {
return diff;
}
// sort by descending size reduction
diff = right->fNumUnitsSaved - left->fNumUnitsSaved;
if (diff != 0) {
return diff;
}
// sort lexically
return left->fString.compare(right->fString);
}
void
StringResource::writeUTF16v2(UnicodeString &dest) {
StringResource::writeUTF16v2(int32_t base, UnicodeString &dest) {
int32_t len = length();
fRes = URES_MAKE_RESOURCE(URES_STRING_V2, dest.length());
fRes = URES_MAKE_RESOURCE(URES_STRING_V2, base + dest.length());
fWritten = TRUE;
switch(fNumCharsForLength) {
case 0:
@ -1429,16 +1547,33 @@ SRBRoot::compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode) {
* suffixes of this one.
*/
StringResource *res = array[i];
res->fNumUnitsSaved = (res->fNumCopies - 1) * res->get16BitStringsLength();
// Whole duplicates of pool strings are already account for in fPoolStringIndexLimit,
// see StringResource::handlePreflightStrings().
int32_t j;
for (j = i + 1; j < count; ++j) {
StringResource *suffixRes = array[j];
/* Is it a suffix of the earlier, longer string? */
if (res->fString.endsWith(suffixRes->fString)) {
assert(res->length() != suffixRes->length()); // Set strings are unique.
if (suffixRes->fNumCharsForLength == 0) {
if (suffixRes->fWritten) {
// Pool string, skip.
} else if (suffixRes->fNumCharsForLength == 0) {
/* yes, point to the earlier string */
suffixRes->fSame = res;
suffixRes->fSuffixOffset = res->length() - suffixRes->length();
if (res->fWritten) {
// Suffix-share res which is a pool string.
// Compute the resource word and collect the maximum.
suffixRes->fRes =
res->fRes + res->fNumCharsForLength + suffixRes->fSuffixOffset;
int32_t poolStringIndex = (int32_t)RES_GET_OFFSET(suffixRes->fRes);
if (poolStringIndex >= fPoolStringIndexLimit) {
fPoolStringIndexLimit = poolStringIndex + 1;
}
suffixRes->fWritten = TRUE;
}
res->fNumUnitsSaved += suffixRes->fNumCopies * suffixRes->get16BitStringsLength();
} else {
/* write the suffix by itself if we need explicit length */
}
@ -1458,23 +1593,89 @@ SRBRoot::compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode) {
if (U_FAILURE(errorCode)) {
return;
}
/* Write the non-suffix strings. */
int32_t i;
for (i = 0; i < count && array[i]->fSame == NULL; ++i) {
array[i]->writeUTF16v2(f16BitUnits);
}
if (f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
/* Write the suffix strings. Make each point to the real string. */
for (; i < count; ++i) {
StringResource *res = array[i];
StringResource *same = res->fSame;
assert(res->length() != same->length()); // Set strings are unique.
res->fRes = same->fRes + same->fNumCharsForLength + res->fSuffixOffset;
res->fSame = NULL;
res->fWritten = TRUE;
if (fIsPoolBundle) {
// Write strings that are sufficiently shared.
// Avoid writing other strings.
int32_t numStringsWritten = 0;
int32_t numUnitsSaved = 0;
int32_t numUnitsNotSaved = 0;
for (int32_t i = 0; i < count; ++i) {
StringResource *res = array[i];
// Maximum pool string index when suffix-sharing the last character.
int32_t maxStringIndex =
f16BitUnits.length() + res->fNumCharsForLength + res->length() - 1;
if (res->fNumUnitsSaved >= GENRB_MIN_16BIT_UNITS_SAVED_FOR_POOL_STRING &&
maxStringIndex < RES_MAX_OFFSET) {
res->writeUTF16v2(0, f16BitUnits);
++numStringsWritten;
numUnitsSaved += res->fNumUnitsSaved;
} else {
numUnitsNotSaved += res->fNumUnitsSaved;
res->fRes = URES_MAKE_EMPTY_RESOURCE(URES_STRING);
res->fWritten = TRUE;
}
}
if (f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
}
if (getShowWarning()) { // not quiet
printf("number of shared strings: %d\n", (int)numStringsWritten);
printf("16-bit units for strings: %6d = %6d bytes\n",
(int)f16BitUnits.length(), (int)f16BitUnits.length() * 2);
printf("16-bit units saved: %6d = %6d bytes\n",
(int)numUnitsSaved, (int)numUnitsSaved * 2);
printf("16-bit units not saved: %6d = %6d bytes\n",
(int)numUnitsNotSaved, (int)numUnitsNotSaved * 2);
}
} else {
assert(fPoolStringIndexLimit <= fUsePoolBundle->fStringIndexLimit);
/* Write the non-suffix strings. */
int32_t i;
for (i = 0; i < count && array[i]->fSame == NULL; ++i) {
StringResource *res = array[i];
if (!res->fWritten) {
int32_t localStringIndex = f16BitUnits.length();
if (localStringIndex >= fLocalStringIndexLimit) {
fLocalStringIndexLimit = localStringIndex + 1;
}
res->writeUTF16v2(fPoolStringIndexLimit, f16BitUnits);
}
}
if (f16BitUnits.isBogus()) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
return;
}
if (fWritePoolBundle != NULL && gFormatVersion >= 3) {
PseudoListResource *poolStrings =
static_cast<PseudoListResource *>(fWritePoolBundle->fRoot);
for (i = 0; i < count && array[i]->fSame == NULL; ++i) {
assert(!array[i]->fString.isEmpty());
StringResource *poolString =
new StringResource(fWritePoolBundle, array[i]->fString, errorCode);
if (poolString == NULL) {
errorCode = U_MEMORY_ALLOCATION_ERROR;
break;
}
poolStrings->add(poolString);
}
}
/* Write the suffix strings. Make each point to the real string. */
for (; i < count; ++i) {
StringResource *res = array[i];
if (res->fWritten) {
continue;
}
StringResource *same = res->fSame;
assert(res->length() != same->length()); // Set strings are unique.
res->fRes = same->fRes + same->fNumCharsForLength + res->fSuffixOffset;
int32_t localStringIndex = (int32_t)RES_GET_OFFSET(res->fRes) - fPoolStringIndexLimit;
// Suffixes of pool strings have been set already.
assert(localStringIndex >= 0);
if (localStringIndex >= fLocalStringIndexLimit) {
fLocalStringIndexLimit = localStringIndex + 1;
}
res->fWritten = TRUE;
}
}
// +1 to account for the initial zero in f16BitUnits
assert(f16BitUnits.length() <= (f16BitStringsLength + 1));

117
icu4c/source/tools/genrb/reslist.h
View file

@ -34,10 +34,14 @@
U_CDECL_BEGIN
class PseudoListResource;
struct ResFile {
ResFile()
: fBytes(NULL), fIndexes(NULL),
fKeys(NULL), fKeysLength(0), fKeysCount(0), fChecksum(0) {}
fKeys(NULL), fKeysLength(0), fKeysCount(0),
fStrings(NULL), fStringIndexLimit(0),
fChecksum(0) {}
~ResFile() { close(); }
void close();
@ -47,6 +51,10 @@ struct ResFile {
const char *fKeys;
int32_t fKeysLength;
int32_t fKeysCount;
PseudoListResource *fStrings;
int32_t fStringIndexLimit;
int32_t fChecksum;
};
@ -74,8 +82,8 @@ struct SRBRoot {
void compactKeys(UErrorCode &errorCode);
int32_t makeRes16(uint32_t resWord) const;
int32_t mapKey(int32_t oldpos) const;
uint16_t makeKey16(int32_t key) const;
private:
void compactStringsV2(UHashtable *stringSet, UErrorCode &errorCode);
@ -102,6 +110,10 @@ public:
int32_t f16BitStringsLength;
const ResFile *fUsePoolBundle;
int32_t fPoolStringIndexLimit;
int32_t fPoolStringIndex16Limit;
int32_t fLocalStringIndexLimit;
SRBRoot *fWritePoolBundle;
};
/* write a java resource file */
@ -170,8 +182,8 @@ struct SResource {
* Writes resource values into f16BitUnits
* and determines the resource item word, if possible.
*/
void write16(SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle, UErrorCode &errorCode);
void write16(SRBRoot *bundle);
virtual void handleWrite16(SRBRoot *bundle);
/**
* Calculates ("preflights") and advances the *byteOffset
@ -188,21 +200,22 @@ struct SResource {
* before actually writing the bundle contents to the file,
* which is necessary because the root item is stored at the beginning.
*/
void preWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
void preWrite(uint32_t *byteOffset);
virtual void handlePreWrite(uint32_t *byteOffset);
/**
* Writes the resource's data to mem and updates the byteOffset
* in parallel.
*/
void write(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
void write(UNewDataMemory *mem, uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
int8_t fType; /* nominal type: fRes (when != 0xffffffff) may use subtype */
UBool fWritten; /* res_write() can exit early */
uint32_t fRes; /* resource item word; RES_BOGUS=0xffffffff if not known yet */
int32_t fRes16; /* Res16 version of fRes for Table, Table16, Array16; -1 if it does not fit. */
int32_t fKey; /* Index into bundle->fKeys; -1 if no key. */
int32_t fKey16; /* Key16 version of fKey for Table & Table16; -1 if no key or it does not fit. */
int line; /* used internally to report duplicate keys in tables */
SResource *fNext; /* This is for internal chaining while building */
struct UString fComment;
@ -217,7 +230,13 @@ public:
virtual ~ContainerResource();
virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
protected:
void writeAllRes16(SRBRoot *bundle);
void preWriteAllRes(uint32_t *byteOffset);
void writeAllRes(UNewDataMemory *mem, uint32_t *byteOffset);
void writeAllRes32(UNewDataMemory *mem, uint32_t *byteOffset);
public:
// TODO: private with getter?
uint32_t fCount;
SResource *fFirst;
@ -233,10 +252,9 @@ public:
void add(SResource *res, int linenumber, UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle, UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle);
virtual void handlePreWrite(uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
int8_t fTableType; // determined by table_write16() for table_preWrite() & table_write()
SRBRoot *fRoot;
@ -252,27 +270,43 @@ public:
void add(SResource *res);
virtual void handleWrite16(SRBRoot *bundle, UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle);
virtual void handlePreWrite(uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
SResource *fLast;
};
/**
* List of resources for a pool bundle.
* Writes an empty table resource, rather than a container structure.
*/
class PseudoListResource : public ContainerResource {
public:
PseudoListResource(SRBRoot *bundle, UErrorCode &errorCode)
: ContainerResource(bundle, NULL, URES_TABLE, NULL, errorCode) {}
virtual ~PseudoListResource();
void add(SResource *res);
virtual void handleWrite16(SRBRoot *bundle);
};
class StringBaseResource : public SResource {
public:
StringBaseResource(SRBRoot *bundle, const char *tag, int8_t type,
const UChar *value, int32_t len,
const UString* comment, UErrorCode &errorCode);
StringBaseResource(SRBRoot *bundle, int8_t type,
const icu::UnicodeString &value, UErrorCode &errorCode);
StringBaseResource(int8_t type, const UChar *value, int32_t len, UErrorCode &errorCode);
virtual ~StringBaseResource();
const UChar *getBuffer() const { return fString.getBuffer(); }
int32_t length() const { return fString.length(); }
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
// TODO: private with getter?
icu::UnicodeString fString;
@ -283,16 +317,37 @@ public:
StringResource(SRBRoot *bundle, const char *tag, const UChar *value, int32_t len,
const UString* comment, UErrorCode &errorCode)
: StringBaseResource(bundle, tag, URES_STRING, value, len, comment, errorCode),
fSame(NULL), fSuffixOffset(0), fNumCharsForLength(0) {}
fSame(NULL), fSuffixOffset(0),
fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {}
StringResource(SRBRoot *bundle, const icu::UnicodeString &value, UErrorCode &errorCode)
: StringBaseResource(bundle, URES_STRING, value, errorCode),
fSame(NULL), fSuffixOffset(0),
fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(0) {}
StringResource(int32_t poolStringIndex, int8_t numCharsForLength,
const UChar *value, int32_t length,
UErrorCode &errorCode)
: StringBaseResource(URES_STRING, value, length, errorCode),
fSame(NULL), fSuffixOffset(0),
fNumCopies(0), fNumUnitsSaved(0), fNumCharsForLength(numCharsForLength) {
// v3 pool string encoded as string-v2 with low offset
fRes = URES_MAKE_RESOURCE(URES_STRING_V2, poolStringIndex);
fWritten = TRUE;
}
virtual ~StringResource();
virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle, UErrorCode &errorCode);
int32_t get16BitStringsLength() const {
return fNumCharsForLength + length() + 1; // +1 for the NUL
}
void writeUTF16v2(icu::UnicodeString &dest);
virtual void handlePreflightStrings(SRBRoot *bundle, UHashtable *stringSet, UErrorCode &errorCode);
virtual void handleWrite16(SRBRoot *bundle);
void writeUTF16v2(int32_t base, icu::UnicodeString &dest);
StringResource *fSame; // used for duplicates
int32_t fSuffixOffset; // this string is a suffix of fSame at this offset
int32_t fNumCopies; // number of equal strings represented by one stringSet element
int32_t fNumUnitsSaved; // from not writing duplicates and suffixes
int8_t fNumCharsForLength;
};
@ -322,9 +377,8 @@ public:
void add(int32_t value, UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
// TODO: UVector32
uint32_t fCount;
@ -338,9 +392,8 @@ public:
const UString* comment, UErrorCode &errorCode);
virtual ~BinaryResource();
virtual void handlePreWrite(uint32_t *byteOffset, SRBRoot *bundle, UErrorCode &errorCode);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset, SRBRoot *bundle,
UErrorCode &errorCode);
virtual void handlePreWrite(uint32_t *byteOffset);
virtual void handleWrite(UNewDataMemory *mem, uint32_t *byteOffset);
// TODO: CharString?
uint32_t fLength;
@ -357,10 +410,12 @@ UBool getIncludeCopyright(void);
void setFormatVersion(int32_t formatVersion);
int32_t getFormatVersion();
void setUsePoolBundle(UBool use);
/* in wrtxml.cpp */
uint32_t computeCRC(char *ptr, uint32_t len, uint32_t lastcrc);
uint32_t computeCRC(const char *ptr, uint32_t len, uint32_t lastcrc);
U_CDECL_END
#endif /* #ifndef RESLIST_H */

2
icu4c/source/tools/genrb/wrtxml.cpp
View file

@ -136,7 +136,7 @@ static char* getID(const char* id, const char* curKey, char* result) {
* conversion is not portable across platforms with different endianess.
*/
uint32_t computeCRC(char *ptr, uint32_t len, uint32_t lastcrc){
uint32_t computeCRC(const char *ptr, uint32_t len, uint32_t lastcrc){
int32_t crc;
uint32_t temp1;
uint32_t temp2;

6
icu4c/source/tools/toolutil/pkgitems.cpp
View file

@ -1,7 +1,7 @@
/*
*******************************************************************************
*
* Copyright (C) 2003-2014, International Business Machines
* Copyright (C) 2003-2015, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
@ -410,7 +410,8 @@ ures_enumDependencies(const char *itemName, const UDataInfo *pInfo,
fprintf(stderr, "icupkg: %s is not a pool bundle\n", poolName);
return;
}
const int32_t *poolIndexes=(const int32_t *)nativePool.getBytes()+1;
const int32_t *poolRoot=(const int32_t *)nativePool.getBytes();
const int32_t *poolIndexes=poolRoot+1;
int32_t poolIndexLength=poolIndexes[URES_INDEX_LENGTH]&0xff;
if(!(poolIndexLength>URES_INDEX_POOL_CHECKSUM &&
(poolIndexes[URES_INDEX_ATTRIBUTES]&URES_ATT_IS_POOL_BUNDLE))
@ -420,6 +421,7 @@ ures_enumDependencies(const char *itemName, const UDataInfo *pInfo,
}
if(resData.pRoot[1+URES_INDEX_POOL_CHECKSUM]==poolIndexes[URES_INDEX_POOL_CHECKSUM]) {
resData.poolBundleKeys=(const char *)(poolIndexes+poolIndexLength);
resData.poolBundleStrings=(const uint16_t *)(poolRoot+poolIndexes[URES_INDEX_KEYS_TOP]);
} else {
fprintf(stderr, "icupkg: %s has mismatched checksum for %s\n", poolName, itemName);
return;