organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-20 20:19:32 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 1

ICU-9398 avoid use of utf8_countTrailBytes[], rewrite/optimize U8_COUNT_TRAIL_BYTES() & U8_NEXT_UNSAFE(), test _UNSAFE macros only with (mostly) well-formed UTF-8 text

Browse source 
X-SVN-Rev: 32574
This commit is contained in:
Markus Scherer 2012-10-09 05:17:43 +00:00
parent 07582f8efa
commit 996422e74c
7 changed files with 391 additions and 317 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
icu4c/source/common/ucnv_u8.c
View file

@ -821,7 +821,7 @@ ucnv_UTF8FromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
if(U8_IS_TRAIL(b)) {
++i;
} else {
if(i<utf8_countTrailBytes[b]) {
if(i<U8_COUNT_TRAIL_BYTES(b)) {
/* stop converting before the lead byte if there are not enough trail bytes for it */
count-=i+1;
}
@ -887,7 +887,7 @@ ucnv_UTF8FromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
/* handle "complicated" and error cases, and continuing partial characters */
oldToULength=0;
toULength=1;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
c=b;
moreBytes:
while(toULength<toULimit) {
@ -960,7 +960,7 @@ moreBytes:
*pErrorCode=U_BUFFER_OVERFLOW_ERROR;
} else {
b=*source;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
if(toULimit>(sourceLimit-source)) {
/* collect a truncated byte sequence */
toULength=0;

4
icu4c/source/common/ucnvlat1.c
View file

@ -1,6 +1,6 @@
/*
**********************************************************************
* Copyright (C) 2000-2011, International Business Machines
* Copyright (C) 2000-2012, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* file name: ucnvlat1.cpp
@ -406,7 +406,7 @@ ucnv_Latin1FromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
if(U_SUCCESS(*pErrorCode) && source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) {
utf8->toUnicodeStatus=utf8->toUBytes[0]=b=*source++;
utf8->toULength=1;
utf8->mode=utf8_countTrailBytes[b]+1;
utf8->mode=U8_COUNT_TRAIL_BYTES(b)+1;
}
/* write back the updated pointers */

12
icu4c/source/common/ucnvmbcs.c
View file

@ -4930,7 +4930,7 @@ ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
if(U8_IS_TRAIL(b)) {
++i;
} else {
if(i<utf8_countTrailBytes[b]) {
if(i<U8_COUNT_TRAIL_BYTES(b)) {
/* exit the conversion loop before the lead byte if there are not enough trail bytes for it */
sourceLimit-=i+1;
}
@ -5023,7 +5023,7 @@ ucnv_SBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
/* handle "complicated" and error cases, and continuing partial characters */
oldToULength=0;
toULength=1;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
c=b;
moreBytes:
while(toULength<toULimit) {
@ -5148,7 +5148,7 @@ moreBytes:
source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) {
c=utf8->toUBytes[0]=b=*source++;
toULength=1;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
while(source<sourceLimit) {
utf8->toUBytes[toULength++]=b=*source++;
c=(c<<6)+b;
@ -5230,7 +5230,7 @@ ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
if(U8_IS_TRAIL(b)) {
++i;
} else {
if(i<utf8_countTrailBytes[b]) {
if(i<U8_COUNT_TRAIL_BYTES(b)) {
/* exit the conversion loop before the lead byte if there are not enough trail bytes for it */
sourceLimit-=i+1;
}
@ -5303,7 +5303,7 @@ ucnv_DBCSFromUTF8(UConverterFromUnicodeArgs *pFromUArgs,
/* handle "complicated" and error cases, and continuing partial characters */
oldToULength=0;
toULength=1;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
c=b;
moreBytes:
while(toULength<toULimit) {
@ -5458,7 +5458,7 @@ unassigned:
source<(sourceLimit=(uint8_t *)pToUArgs->sourceLimit)) {
c=utf8->toUBytes[0]=b=*source++;
toULength=1;
toULimit=utf8_countTrailBytes[b]+1;
toULimit=U8_COUNT_TRAIL_BYTES(b)+1;
while(source<sourceLimit) {
utf8->toUBytes[toULength++]=b=*source++;
c=(c<<6)+b;

59
icu4c/source/common/unicode/utf8.h
View file

@ -1,7 +1,7 @@
/*
*******************************************************************************
*
* Copyright (C) 1999-2011, International Business Machines
* Copyright (C) 1999-2012, International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
@ -60,13 +60,41 @@ U_CFUNC U_IMPORT const uint8_t /* U_IMPORT2? */ /*U_IMPORT*/
utf8_countTrailBytes[256];
/**
* Count the trail bytes for a UTF-8 lead byte.
* Counts the trail bytes for a UTF-8 lead byte.
* Returns 0 for 0..0xbf as well as for 0xfe and 0xff.
*
* This is internal since it is not meant to be called directly by external clients;
* however it is called by public macros in this file and thus must remain stable.
*
* Note: Beginning with ICU 50, the implementation uses a multi-condition expression
* which was shown in 2012 (on x86-64) to compile to fast, branch-free code.
* leadByte is evaluated multiple times.
*
* The pre-ICU 50 implementation used the exported array utf8_countTrailBytes:
* #define U8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[leadByte])
* leadByte was evaluated exactly once.
*
* @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
* @internal
*/
#define U8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[(uint8_t)leadByte])
#define U8_COUNT_TRAIL_BYTES(leadByte) \
((leadByte)<0xf0 ? \
((leadByte)>=0xc0)+((leadByte)>=0xe0) : \
(leadByte)<0xfe ? 3+((leadByte)>=0xf8)+((leadByte)>=0xfc) : 0)
/**
* Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence.
* The maximum supported lead byte is 0xf4 corresponding to U+10FFFF.
* leadByte might be evaluated multiple times.
*
* This is internal since it is not meant to be called directly by external clients;
* however it is called by public macros in this file and thus must remain stable.
*
* @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
* @internal
*/
#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \
(((leadByte)>=0xc0)+((leadByte)>=0xe0)+((leadByte)>=0xf0))
/**
* Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
@ -243,19 +271,16 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);
*/
#define U8_NEXT_UNSAFE(s, i, c) { \
(c)=(uint8_t)(s)[(i)++]; \
if((uint8_t)((c)-0xc0)<0x35) { \
uint8_t __count=U8_COUNT_TRAIL_BYTES(c); \
U8_MASK_LEAD_BYTE(c, __count); \
switch(__count) { \
/* each following branch falls through to the next one */ \
case 3: \
(c)=((c)<<6)|((s)[(i)++]&0x3f); \
case 2: \
(c)=((c)<<6)|((s)[(i)++]&0x3f); \
case 1: \
(c)=((c)<<6)|((s)[(i)++]&0x3f); \
/* no other branches to optimize switch() */ \
break; \
if((c)>=0x80) { \
if((c)<0xe0) { \
(c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \
} else if((c)<0xf0) { \
/* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \
(c)=(UChar)(((c)<<12)|(((s)[i]&0x3f)<<6)|((s)[(i)+1]&0x3f)); \
(i)+=2; \
} else { \
(c)=(((c)&7)<<18)|(((s)[i]&0x3f)<<12)|(((s)[(i)+1]&0x3f)<<6)|((s)[(i)+2]&0x3f); \
(i)+=3; \
} \
} \
}
@ -382,7 +407,7 @@ utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);
* @stable ICU 2.4
*/
#define U8_FWD_1_UNSAFE(s, i) { \
(i)+=1+U8_COUNT_TRAIL_BYTES((s)[i]); \
(i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((uint8_t)(s)[i]); \
}
/**

619
icu4c/source/test/cintltst/utf8tst.c
View file

@ -1,6 +1,6 @@
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1998-2006, International Business Machines Corporation and
* COPYRIGHT:
* Copyright (c) 1998-2012, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
/*
@ -9,7 +9,7 @@
* Modification History:
*
* Date Name Description
* 07/24/2000 Madhu Creation
* 07/24/2000 Madhu Creation
*******************************************************************************
*/
@ -18,7 +18,7 @@
#include "cmemory.h"
#include "cintltst.h"
#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0]))
#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
/* lenient UTF-8 ------------------------------------------------------------ */
@ -64,8 +64,12 @@ static void TestCodeUnitValues(void);
static void TestCharLength(void);
static void TestGetChar(void);
static void TestNextPrevChar(void);
static void TestNextPrevNonCharacters(void);
static void TestNextPrevCharUnsafe(void);
static void TestFwdBack(void);
static void TestFwdBackUnsafe(void);
static void TestSetChar(void);
static void TestSetCharUnsafe(void);
static void TestAppendChar(void);
static void TestAppend(void);
static void TestSurrogates(void);
@ -75,23 +79,27 @@ void addUTF8Test(TestNode** root);
void
addUTF8Test(TestNode** root)
{
addTest(root, &TestCodeUnitValues, "utf8tst/TestCodeUnitValues");
addTest(root, &TestCharLength, "utf8tst/TestCharLength" );
addTest(root, &TestGetChar, "utf8tst/TestGetChar" );
addTest(root, &TestNextPrevChar, "utf8tst/TestNextPrevChar" );
addTest(root, &TestFwdBack, "utf8tst/TestFwdBack" );
addTest(root, &TestSetChar, "utf8tst/TestSetChar" );
addTest(root, &TestAppendChar, "utf8tst/TestAppendChar" );
addTest(root, &TestAppend, "utf8tst/TestAppend" );
addTest(root, &TestSurrogates, "utf8tst/TestSurrogates" );
addTest(root, &TestCodeUnitValues, "utf8tst/TestCodeUnitValues");
addTest(root, &TestCharLength, "utf8tst/TestCharLength");
addTest(root, &TestGetChar, "utf8tst/TestGetChar");
addTest(root, &TestNextPrevChar, "utf8tst/TestNextPrevChar");
addTest(root, &TestNextPrevNonCharacters, "utf8tst/TestNextPrevNonCharacters");
addTest(root, &TestNextPrevCharUnsafe, "utf8tst/TestNextPrevCharUnsafe");
addTest(root, &TestFwdBack, "utf8tst/TestFwdBack");
addTest(root, &TestFwdBackUnsafe, "utf8tst/TestFwdBackUnsafe");
addTest(root, &TestSetChar, "utf8tst/TestSetChar");
addTest(root, &TestSetCharUnsafe, "utf8tst/TestSetCharUnsafe");
addTest(root, &TestAppendChar, "utf8tst/TestAppendChar");
addTest(root, &TestAppend, "utf8tst/TestAppend");
addTest(root, &TestSurrogates, "utf8tst/TestSurrogates");
}
static void TestCodeUnitValues()
{
static const uint8_t codeunit[]={0x00, 0x65, 0x7e, 0x7f, 0xc0, 0xc4, 0xf0, 0xfd, 0x80, 0x81, 0xbc, 0xbe,};
int16_t i;
for(i=0; i<sizeof(codeunit)/sizeof(codeunit[0]); i++){
for(i=0; i<LENGTHOF(codeunit); i++){
uint8_t c=codeunit[i];
log_verbose("Testing code unit value of %x\n", c);
if(i<4){
@ -129,17 +137,17 @@ static void TestCharLength()
4, 0x10402,
3, 0xd7ff,
3, 0xe000,
};
int16_t i;
UBool multiple;
for(i=0; i<sizeof(codepoint)/sizeof(codepoint[0]); i=(int16_t)(i+2)){
for(i=0; i<LENGTHOF(codepoint); i=(int16_t)(i+2)){
UChar32 c=codepoint[i+1];
if(UTF8_CHAR_LENGTH(c) != (uint16_t)codepoint[i] || U8_LENGTH(c) != (uint16_t)codepoint[i]){
log_err("The no: of code units for %lx:- Expected: %d Got: %d\n", c, codepoint[i], UTF8_CHAR_LENGTH(c));
}else{
log_verbose("The no: of code units for %lx is %d\n",c, UTF8_CHAR_LENGTH(c) );
log_verbose("The no: of code units for %lx is %d\n",c, UTF8_CHAR_LENGTH(c));
}
multiple=(UBool)(codepoint[i] == 1 ? FALSE : TRUE);
if(UTF8_NEED_MULTIPLE_UCHAR(c) != multiple){
@ -155,11 +163,11 @@ static void TestGetChar()
0x61,
0x7f,
0xe4,
0xba,
0xba,
0x8c,
0xF0,
0x90,
0x90,
0xF0,
0x90,
0x90,
0x81,
0xc0,
0x65,
@ -168,9 +176,9 @@ static void TestGetChar()
0xc9
};
static const UChar32 result[]={
/*codepoint-unsafe, codepoint-safe(not strict) codepoint-safe(strict)*/
0x61, 0x61, 0x61,
0x7f, 0x7f, 0x7f,
/* codepoint-unsafe, codepoint-safe(not strict) codepoint-safe(strict) */
0x61, 0x61, 0x61,
0x7f, 0x7f, 0x7f,
0x4e8c, 0x4e8c, 0x4e8c,
0x4e8c, 0x4e8c, 0x4e8c ,
0x4e8c, 0x4e8c, 0x4e8c,
@ -179,8 +187,8 @@ static void TestGetChar()
0x10401, 0x10401, 0x10401 ,
0x10401, 0x10401, 0x10401,
0x25, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
0x65, 0x65, 0x65,
0x31, 0x31, 0x31,
0x65, 0x65, 0x65,
0x31, 0x31, 0x31,
0x31, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
0x240, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1
};
@ -193,13 +201,13 @@ static void TestGetChar()
UTF8_GET_CHAR_UNSAFE(input, offset, c);
if(c != result[i]){
log_err("ERROR: UTF8_GET_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);
}
U8_GET_UNSAFE(input, offset, c);
if(c != result[i]){
log_err("ERROR: U8_GET_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);
}
}
@ -217,15 +225,15 @@ static void TestGetChar()
if(c != result[i+2]){
log_err("ERROR: UTF8_GET_CHAR_SAFE(strict) failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i+2], c);
}
i=(uint16_t)(i+3);
}
}
static void TestNextPrevChar(){
static void TestNextPrevChar() {
static const uint8_t input[]={0x61, 0xf0, 0x90, 0x90, 0x81, 0xc0, 0x80, 0xfd, 0xbe, 0xc2, 0x61, 0x81, 0x90, 0x90, 0xf0, 0x00};
static const UChar32 result[]={
/*next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s*/
/* next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s */
0x0061, 0x0061, 0x0061, 0x0000, 0x0000, 0x0000,
0x10401, 0x10401, 0x10401, 0xf0, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
0x90, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x2841410, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1,
@ -244,55 +252,31 @@ static void TestNextPrevChar(){
0x0000, 0x0000, 0x0000, 0x0061, 0x0061, 0x0061
};
static const int32_t movedOffset[]={
/*next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s*/
/* next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s */
1, 1, 1, 15, 15, 15,
5, 5, 5, 14, 14 , 14,
3, 3, 3, 9, 13, 13,
5, 5, 5, 14, 14 , 14,
3, 3, 3, 9, 13, 13,
4, 4, 4, 9, 12, 12,
5, 5, 5, 9, 11, 11,
7, 7, 7, 10, 10, 10,
7, 7, 7, 9, 9, 9,
8, 9, 9, 7, 7, 7,
9, 9, 9, 7, 7, 7,
11, 10, 10, 5, 5, 5,
11, 11, 11, 5, 5, 5,
12, 12, 12, 1, 1, 1,
13, 13, 13, 1, 1, 1,
14, 14, 14, 1, 1, 1,
14, 15, 15, 1, 1, 1,
14, 16, 16, 0, 0, 0,
5, 5, 5, 9, 11, 11,
7, 7, 7, 10, 10, 10,
7, 7, 7, 9, 9, 9,
8, 9, 9, 7, 7, 7,
9, 9, 9, 7, 7, 7,
11, 10, 10, 5, 5, 5,
11, 11, 11, 5, 5, 5,
12, 12, 12, 1, 1, 1,
13, 13, 13, 1, 1, 1,
14, 14, 14, 1, 1, 1,
14, 15, 15, 1, 1, 1,
14, 16, 16, 0, 0, 0,
};
/* TODO: remove unused columns for next_unsafe & prev_unsafe, and adjust the test code */
UChar32 c=0x0000;
uint32_t i=0;
uint32_t offset=0;
int32_t setOffset=0;
for(offset=0; offset<sizeof(input); offset++){
if (offset < sizeof(input) - 2) { /* Can't have it go off the end of the array based on input */
setOffset=offset;
UTF8_NEXT_CHAR_UNSAFE(input, setOffset, c);
if(setOffset != movedOffset[i]){
log_err("ERROR: UTF8_NEXT_CHAR_UNSAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
offset, movedOffset[i], setOffset);
}
if(c != result[i]){
log_err("ERROR: UTF8_NEXT_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);
}
setOffset=offset;
U8_NEXT_UNSAFE(input, setOffset, c);
if(setOffset != movedOffset[i]){
log_err("ERROR: U8_NEXT_UNSAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
offset, movedOffset[i], setOffset);
}
if(c != result[i]){
log_err("ERROR: U8_NEXT_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c);
}
}
setOffset=offset;
UTF8_NEXT_CHAR_SAFE(input, setOffset, sizeof(input), c, FALSE);
if(setOffset != movedOffset[i+1]){
@ -328,16 +312,6 @@ static void TestNextPrevChar(){
i=0;
for(offset=sizeof(input); offset > 0; --offset){
setOffset=offset;
UTF8_PREV_CHAR_UNSAFE(input, setOffset, c);
if(setOffset != movedOffset[i+3]){
log_err("ERROR: UTF8_PREV_CHAR_UNSAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
offset, movedOffset[i+3], setOffset);
}
if(c != result[i+3]){
log_err("ERROR: UTF8_PREV_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i+3], c);
}
setOffset=offset;
UTF8_PREV_CHAR_SAFE(input, 0, setOffset, c, FALSE);
if(setOffset != movedOffset[i+4]){
@ -363,77 +337,112 @@ static void TestNextPrevChar(){
if(setOffset != movedOffset[i+5]){
log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n",
offset, movedOffset[i+5], setOffset);
}
}
if(c != result[i+5]){
log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed for input=%ld. Expected:%lx Got:%lx\n", offset, result[i+5], c);
}
i=i+6;
}
}
{
/* test non-characters */
static const uint8_t nonChars[]={
0xef, 0xb7, 0x90, /* U+fdd0 */
0xef, 0xbf, 0xbf, /* U+feff */
0xf0, 0x9f, 0xbf, 0xbe, /* U+1fffe */
0xf0, 0xbf, 0xbf, 0xbf, /* U+3ffff */
0xf4, 0x8f, 0xbf, 0xbe /* U+10fffe */
};
static void TestNextPrevNonCharacters() {
/* test non-characters */
static const uint8_t nonChars[]={
0xef, 0xb7, 0x90, /* U+fdd0 */
0xef, 0xbf, 0xbf, /* U+feff */
0xf0, 0x9f, 0xbf, 0xbe, /* U+1fffe */
0xf0, 0xbf, 0xbf, 0xbf, /* U+3ffff */
0xf4, 0x8f, 0xbf, 0xbe /* U+10fffe */
};
UChar32 ch;
int32_t idx;
UChar32 ch;
int32_t idx;
for(idx=0; idx<(int32_t)sizeof(nonChars);) {
U8_NEXT(nonChars, idx, sizeof(nonChars), ch);
if(!U_IS_UNICODE_NONCHAR(ch)) {
log_err("U8_NEXT(before %d) failed to read a non-character\n", idx);
}
for(idx=0; idx<(int32_t)sizeof(nonChars);) {
U8_NEXT(nonChars, idx, sizeof(nonChars), ch);
if(!U_IS_UNICODE_NONCHAR(ch)) {
log_err("U8_NEXT(before %d) failed to read a non-character\n", idx);
}
for(idx=(int32_t)sizeof(nonChars); idx>0;) {
U8_PREV(nonChars, 0, idx, ch);
if(!U_IS_UNICODE_NONCHAR(ch)) {
log_err("U8_PREV(at %d) failed to read a non-character\n", idx);
}
}
for(idx=(int32_t)sizeof(nonChars); idx>0;) {
U8_PREV(nonChars, 0, idx, ch);
if(!U_IS_UNICODE_NONCHAR(ch)) {
log_err("U8_PREV(at %d) failed to read a non-character\n", idx);
}
}
}
static void TestFwdBack(){
static void TestNextPrevCharUnsafe() {
/*
* Use a (mostly) well-formed UTF-8 string and test at code point boundaries.
* The behavior of _UNSAFE macros for ill-formed strings is undefined.
*/
static const uint8_t input[]={
0x61,
0xf0, 0x90, 0x90, 0x81,
0xc0, 0x80, /* non-shortest form */
0xe2, 0x82, 0xac,
0xc2, 0xa1,
0xf4, 0x8f, 0xbf, 0xbf,
0x00
};
static const UChar32 codePoints[]={
0x61,
0x10401,
0,
0x20ac,
0xa1,
0x10ffff,
0
};
UChar32 c;
int32_t i;
uint32_t offset;
for(i=0, offset=0; offset<sizeof(input); ++i) {
UTF8_NEXT_CHAR_UNSAFE(input, offset, c);
if(c != codePoints[i]){
log_err("ERROR: UTF8_NEXT_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
offset, codePoints[i], c);
}
}
for(i=0, offset=0; offset<sizeof(input); ++i) {
U8_NEXT_UNSAFE(input, offset, c);
if(c != codePoints[i]){
log_err("ERROR: U8_NEXT_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
offset, codePoints[i], c);
}
}
for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){
UTF8_PREV_CHAR_UNSAFE(input, offset, c);
if(c != codePoints[i]){
log_err("ERROR: UTF8_PREV_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
offset, codePoints[i], c);
}
}
for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){
U8_PREV_UNSAFE(input, offset, c);
if(c != codePoints[i]){
log_err("ERROR: U8_PREV_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n",
offset, codePoints[i], c);
}
}
}
static void TestFwdBack() {
static const uint8_t input[]={0x61, 0xF0, 0x90, 0x90, 0x81, 0xff, 0x62, 0xc0, 0x80, 0x7f, 0x8f, 0xc0, 0x63, 0x81, 0x90, 0x90, 0xF0, 0x00};
static const uint16_t fwd_unsafe[] ={1, 5, 6, 7, 9, 10, 11, 13, 14, 15, 16, 20, };
static const uint16_t fwd_safe[] ={1, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18};
static const uint16_t back_unsafe[]={17, 16, 12, 11, 9, 7, 6, 5, 1, 0};
static const uint16_t back_safe[] ={17, 16, 15, 14, 13, 12, 11, 10, 9, 7, 6, 5, 1, 0};
static const uint16_t Nvalue[]= {0, 1, 2, 3, 1, 2, 1, 5};
static const uint16_t fwd_N_unsafe[] ={0, 1, 6, 10, 11, 14, 15};
static const uint16_t fwd_N_safe[] ={0, 1, 6, 10, 11, 13, 14, 18}; /*safe macro keeps it at the end of the string */
static const uint16_t back_N_unsafe[]={18, 17, 12, 7, 6, 1, 0};
static const uint16_t back_N_safe[] ={18, 17, 15, 12, 11, 9, 7, 0};
static const uint16_t back_N_safe[] ={18, 17, 15, 12, 11, 9, 7, 0};
uint32_t offunsafe=0, offsafe=0;
uint32_t offsafe=0;
uint32_t i=0;
while(offunsafe < sizeof(input)){
UTF8_FWD_1_UNSAFE(input, offunsafe);
if(offunsafe != fwd_unsafe[i]){
log_err("ERROR: Forward_unsafe offset expected:%d, Got:%d\n", fwd_unsafe[i], offunsafe);
}
i++;
}
i=0;
while(offunsafe < sizeof(input)){
U8_FWD_1_UNSAFE(input, offunsafe);
if(offunsafe != fwd_unsafe[i]){
log_err("ERROR: U8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", fwd_unsafe[i], offunsafe);
}
i++;
}
i=0;
while(offsafe < sizeof(input)){
UTF8_FWD_1_SAFE(input, offsafe, sizeof(input));
if(offsafe != fwd_safe[i]){
@ -451,32 +460,12 @@ static void TestFwdBack(){
i++;
}
offunsafe=sizeof(input);
i=0;
while(offunsafe > 0){
UTF8_BACK_1_UNSAFE(input, offunsafe);
if(offunsafe != back_unsafe[i]){
log_err("ERROR: Backward_unsafe offset expected:%d, Got:%d\n", back_unsafe[i], offunsafe);
}
i++;
}
offunsafe=sizeof(input);
i=0;
while(offunsafe > 0){
U8_BACK_1_UNSAFE(input, offunsafe);
if(offunsafe != back_unsafe[i]){
log_err("ERROR: U8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", back_unsafe[i], offunsafe);
}
i++;
}
i=0;
offsafe=sizeof(input);
while(offsafe > 0){
UTF8_BACK_1_SAFE(input, 0, offsafe);
if(offsafe != back_safe[i]){
log_err("ERROR: Backward_safe offset expected:%d, Got:%d\n", back_unsafe[i], offsafe);
log_err("ERROR: Backward_safe offset expected:%d, Got:%d\n", back_safe[i], offsafe);
}
i++;
}
@ -486,63 +475,31 @@ static void TestFwdBack(){
while(offsafe > 0){
U8_BACK_1(input, 0, offsafe);
if(offsafe != back_safe[i]){
log_err("ERROR: U8_BACK_1 offset expected:%d, Got:%d\n", back_unsafe[i], offsafe);
log_err("ERROR: U8_BACK_1 offset expected:%d, Got:%d\n", back_safe[i], offsafe);
}
i++;
}
offunsafe=0;
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0])-2; i++){
UTF8_FWD_N_UNSAFE(input, offunsafe, Nvalue[i]);
if(offunsafe != fwd_N_unsafe[i]){
log_err("ERROR: Forward_N_unsafe offset=%d expected:%d, Got:%d\n", i, fwd_N_unsafe[i], offunsafe);
}
}
offunsafe=0;
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0])-2; i++){
U8_FWD_N_UNSAFE(input, offunsafe, Nvalue[i]);
if(offunsafe != fwd_N_unsafe[i]){
log_err("ERROR: U8_FWD_N_UNSAFE offset=%d expected:%d, Got:%d\n", i, fwd_N_unsafe[i], offunsafe);
}
}
offsafe=0;
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0]); i++){
for(i=0; i<LENGTHOF(Nvalue); i++){
UTF8_FWD_N_SAFE(input, offsafe, sizeof(input), Nvalue[i]);
if(offsafe != fwd_N_safe[i]){
log_err("ERROR: Forward_N_safe offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe);
}
}
offsafe=0;
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0]); i++){
for(i=0; i<LENGTHOF(Nvalue); i++){
U8_FWD_N(input, offsafe, sizeof(input), Nvalue[i]);
if(offsafe != fwd_N_safe[i]){
log_err("ERROR: U8_FWD_N offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe);
}
}
offunsafe=sizeof(input);
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0])-2; i++){
UTF8_BACK_N_UNSAFE(input, offunsafe, Nvalue[i]);
if(offunsafe != back_N_unsafe[i]){
log_err("ERROR: backward_N_unsafe offset=%d expected:%d, Got:%d\n", i, back_N_unsafe[i], offunsafe);
}
}
offunsafe=sizeof(input);
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0])-2; i++){
U8_BACK_N_UNSAFE(input, offunsafe, Nvalue[i]);
if(offunsafe != back_N_unsafe[i]){
log_err("ERROR: U8_BACK_N_UNSAFE offset=%d expected:%d, Got:%d\n", i, back_N_unsafe[i], offunsafe);
}
}
offsafe=sizeof(input);
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0]); i++){
for(i=0; i<LENGTHOF(Nvalue); i++){
UTF8_BACK_N_SAFE(input, 0, offsafe, Nvalue[i]);
if(offsafe != back_N_safe[i]){
log_err("ERROR: backward_N_safe offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe);
@ -550,7 +507,7 @@ static void TestFwdBack(){
}
offsafe=sizeof(input);
for(i=0; i<sizeof(Nvalue)/sizeof(Nvalue[0]); i++){
for(i=0; i<LENGTHOF(Nvalue); i++){
U8_BACK_N(input, 0, offsafe, Nvalue[i]);
if(offsafe != back_N_safe[i]){
log_err("ERROR: U8_BACK_N offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe);
@ -558,82 +515,174 @@ static void TestFwdBack(){
}
}
static void TestSetChar(){
static void TestFwdBackUnsafe() {
/*
* Use a (mostly) well-formed UTF-8 string and test at code point boundaries.
* The behavior of _UNSAFE macros for ill-formed strings is undefined.
*/
static const uint8_t input[]={
0x61,
0xf0, 0x90, 0x90, 0x81,
0xc0, 0x80, /* non-shortest form */
0xe2, 0x82, 0xac,
0xc2, 0xa1,
0xf4, 0x8f, 0xbf, 0xbf,
0x00
};
static const int8_t boundaries[]={ 0, 1, 5, 7, 10, 12, 16, 17 };
int32_t offset;
int32_t i;
for(i=1, offset=0; offset<LENGTHOF(input); ++i) {
UTF8_FWD_1_UNSAFE(input, offset);
if(offset != boundaries[i]){
log_err("ERROR: UTF8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=1, offset=0; offset<LENGTHOF(input); ++i) {
U8_FWD_1_UNSAFE(input, offset);
if(offset != boundaries[i]){
log_err("ERROR: U8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) {
UTF8_BACK_1_UNSAFE(input, offset);
if(offset != boundaries[i]){
log_err("ERROR: UTF8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) {
U8_BACK_1_UNSAFE(input, offset);
if(offset != boundaries[i]){
log_err("ERROR: U8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=0; i<LENGTHOF(boundaries); ++i) {
offset=0;
UTF8_FWD_N_UNSAFE(input, offset, i);
if(offset != boundaries[i]) {
log_err("ERROR: UTF8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=0; i<LENGTHOF(boundaries); ++i) {
offset=0;
U8_FWD_N_UNSAFE(input, offset, i);
if(offset != boundaries[i]) {
log_err("ERROR: U8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset);
}
}
for(i=0; i<LENGTHOF(boundaries); ++i) {
int32_t j=LENGTHOF(boundaries)-1-i;
offset=LENGTHOF(input);
UTF8_BACK_N_UNSAFE(input, offset, i);
if(offset != boundaries[j]) {
log_err("ERROR: UTF8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset);
}
}
for(i=0; i<LENGTHOF(boundaries); ++i) {
int32_t j=LENGTHOF(boundaries)-1-i;
offset=LENGTHOF(input);
U8_BACK_N_UNSAFE(input, offset, i);
if(offset != boundaries[j]) {
log_err("ERROR: U8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset);
}
}
}
static void TestSetChar() {
static const uint8_t input[]
= {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0xfe, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x00 };
static const int16_t start_unsafe[]
= {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 9, 9, 12, 13 };
static const int16_t start_safe[]
= {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
static const int16_t limit_unsafe[]
= {0, 1, 4, 4, 4, 5, 6, 7, 9, 9, 10, 10, 10, 15 };
= {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 };
static const int16_t limit_safe[]
= {0, 1, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 };
= {0, 1, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 };
uint32_t i=0;
int32_t offset=0, setOffset=0;
for(offset=0; offset<(int32_t)sizeof(input); offset++){
setOffset=offset;
UTF8_SET_CHAR_START_UNSAFE(input, setOffset);
if(setOffset != start_unsafe[i]){
log_err("ERROR: UTF8_SET_CHAR_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
}
for(offset=0; offset<=LENGTHOF(input); offset++){
if (offset<LENGTHOF(input)){
setOffset=offset;
UTF8_SET_CHAR_START_SAFE(input, 0, setOffset);
if(setOffset != start_safe[i]){
log_err("ERROR: UTF8_SET_CHAR_START_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_START_UNSAFE(input, setOffset);
if(setOffset != start_unsafe[i]){
log_err("ERROR: U8_SET_CP_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_START(input, 0, setOffset);
if(setOffset != start_safe[i]){
log_err("ERROR: U8_SET_CP_START failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
}
}
setOffset=offset;
UTF8_SET_CHAR_START_SAFE(input, 0, setOffset);
if(setOffset != start_safe[i]){
log_err("ERROR: UTF8_SET_CHAR_START_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
}
setOffset=offset;
UTF8_SET_CHAR_LIMIT_SAFE(input,0, setOffset, sizeof(input));
if(setOffset != limit_safe[i]){
log_err("ERROR: UTF8_SET_CHAR_LIMIT_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_START(input, 0, setOffset);
if(setOffset != start_safe[i]){
log_err("ERROR: U8_SET_CP_START failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_LIMIT(input,0, setOffset, sizeof(input));
if(setOffset != limit_safe[i]){
log_err("ERROR: U8_SET_CP_LIMIT failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
}
if (offset != 0) { /* Can't have it go off the end of the array */
setOffset=offset;
UTF8_SET_CHAR_LIMIT_UNSAFE(input, setOffset);
if(setOffset != limit_unsafe[i]){
log_err("ERROR: UTF8_SET_CHAR_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
}
i++;
}
}
setOffset=offset;
U8_SET_CP_LIMIT_UNSAFE(input, setOffset);
if(setOffset != limit_unsafe[i]){
log_err("ERROR: U8_SET_CP_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
}
}
static void TestSetCharUnsafe() {
static const uint8_t input[]
= {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0x2e, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x80, 0x80, 0x00 };
static const int16_t start_unsafe[]
= {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 9, 9, 12, 12, 12, 15 };
static const int16_t limit_unsafe[]
= {0, 1, 4, 4, 4, 5, 6, 7, 9, 9, 10, 10, 10, 15, 15, 15, 16 };
setOffset=offset;
UTF8_SET_CHAR_LIMIT_SAFE(input,0, setOffset, sizeof(input));
if(setOffset != limit_safe[i]){
log_err("ERROR: UTF8_SET_CHAR_LIMIT_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
}
uint32_t i=0;
int32_t offset=0, setOffset=0;
for(offset=0; offset<=LENGTHOF(input); offset++){
if (offset<LENGTHOF(input)){
setOffset=offset;
UTF8_SET_CHAR_START_UNSAFE(input, setOffset);
if(setOffset != start_unsafe[i]){
log_err("ERROR: UTF8_SET_CHAR_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_LIMIT(input,0, setOffset, sizeof(input));
if(setOffset != limit_safe[i]){
log_err("ERROR: U8_SET_CP_LIMIT failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_START_UNSAFE(input, setOffset);
if(setOffset != start_unsafe[i]){
log_err("ERROR: U8_SET_CP_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset);
}
}
i++;
if (offset != 0) { /* Can't have it go off the end of the array */
setOffset=offset;
UTF8_SET_CHAR_LIMIT_UNSAFE(input, setOffset);
if(setOffset != limit_unsafe[i]){
log_err("ERROR: UTF8_SET_CHAR_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
}
setOffset=offset;
U8_SET_CP_LIMIT_UNSAFE(input, setOffset);
if(setOffset != limit_unsafe[i]){
log_err("ERROR: U8_SET_CP_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset);
}
}
i++;
}
}
static void TestAppendChar(){
static const uint8_t s[11]={0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00};
static const uint32_t test[]={
/*append-position(unsafe), CHAR to be appended */
/* append-position(unsafe), CHAR to be appended */
0, 0x10401,
2, 0x0028,
2, 0x007f,
2, 0x007f,
3, 0xd801,
1, 0x20402,
8, 0x10401,
@ -645,9 +694,9 @@ static void TestAppendChar(){
6, 0xbf,
7, 0xfe,
/*append-position(safe), CHAR to be appended */
/* append-position(safe), CHAR to be appended */
0, 0x10401,
2, 0x0028,
2, 0x0028,
3, 0x7f,
3, 0xd801, /* illegal for UTF-8 starting with Unicode 3.2 */
1, 0x20402,
@ -659,25 +708,25 @@ static void TestAppendChar(){
6, 0x81,
6, 0xbf,
7, 0xfe,
};
static const uint16_t movedOffset[]={
/*offset-moved-to(unsafe)*/
/* offset-moved-to(unsafe) */
4, /*for append-pos: 0 , CHAR 0x10401*/
3,
3,
3,
6,
5,
12,
7,
7,
7,
7,
8,
8,
8,
9,
/*offset-moved-to(safe)*/
/* offset-moved-to(safe) */
4, /*for append-pos: 0, CHAR 0x10401*/
3,
4,
@ -685,66 +734,66 @@ static void TestAppendChar(){
5,
11,
7,
7,
7,
7,
8,
8,
8,
9,
};
static const uint8_t result[][11]={
/*unsafe*/
{0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x7f, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0xed, 0xa0, 0x81, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x7f, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0xed, 0xa0, 0x81, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0xF0, 0x90, 0x90},
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00},
/*safe*/
{0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x7f, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0xef, 0xbf, 0xbf, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0xef, 0xbf, 0xbf, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xc2, 0x9f}, /*gets UTF8_ERROR_VALUE_2 which takes 2 bytes 0xc0, 0x9f*/
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00},
{0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00},
};
uint16_t i, count=0;
uint8_t str[12];
uint32_t offset;
/* UChar32 c=0;*/
uint16_t size=sizeof(s)/sizeof(s[0]);
for(i=0; i<sizeof(test)/sizeof(test[0]); i=(uint16_t)(i+2)){
uint16_t size=LENGTHOF(s);
for(i=0; i<LENGTHOF(test); i=(uint16_t)(i+2)){
uprv_memcpy(str, s, size);
offset=test[i];
offset=test[i];
if(count<13){
UTF8_APPEND_CHAR_UNSAFE(str, offset, test[i+1]);
if(offset != movedOffset[count]){
log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
count, movedOffset[count], offset);
}
if(uprv_memcmp(str, result[count], size) !=0){
log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed for count=%d. \nExpected:", count);
@ -756,9 +805,9 @@ static void TestAppendChar(){
}else{
UTF8_APPEND_CHAR_SAFE(str, offset, size, test[i+1]);
if(offset != movedOffset[count]){
log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
count, movedOffset[count], offset);
}
if(uprv_memcmp(str, result[count], size) !=0){
log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed for count=%d. \nExpected:", count);
@ -769,17 +818,17 @@ static void TestAppendChar(){
}
/*call the API instead of MACRO
uprv_memcpy(str, s, size);
offset=test[i];
offset=test[i];
c=test[i+1];
if((uint32_t)(c)<=0x7f) {
(str)[(offset)++]=(uint8_t)(c);
} else {
(offset)=utf8_appendCharSafeBody(str, (int32_t)(offset), (int32_t)(size), c);
if((uint32_t)(c)<=0x7f) {
(str)[(offset)++]=(uint8_t)(c);
} else {
(offset)=utf8_appendCharSafeBody(str, (int32_t)(offset), (int32_t)(size), c);
}
if(offset != movedOffset[count]){
log_err("ERROR: utf8_appendCharSafeBody() failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
log_err("ERROR: utf8_appendCharSafeBody() failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n",
count, movedOffset[count], offset);
}
if(uprv_memcmp(str, result[count], size) !=0){
log_err("ERROR: utf8_appendCharSafeBody() failed for count=%d. \nExpected:", count);
@ -791,8 +840,8 @@ static void TestAppendChar(){
*/
}
count++;
}
}
}

4
icu4c/source/test/perf/collationperf/collperf.cpp
View file

@ -1,6 +1,6 @@
/********************************************************************
* COPYRIGHT:
* Copyright (C) 2001-2011 IBM, Inc. All Rights Reserved.
* Copyright (C) 2001-2012 IBM, Inc. All Rights Reserved.
*
********************************************************************/
/********************************************************************************
@ -1340,7 +1340,7 @@ UChar UCharFile::get() {
// Convert the bytes from the temp array to a Unicode char.
i = 0;
uint32_t cp;
UTF8_NEXT_CHAR_UNSAFE(bytes, i, cp);
U8_NEXT_UNSAFE(bytes, i, cp);
c = (UChar)cp;
if (cp >= 0x10000) {

4
icu4c/source/test/perf/ubrkperf/ubrkperfold.cpp
View file

@ -1,6 +1,6 @@
/********************************************************************
* COPYRIGHT:
* Copyright (C) 2001-2011 IBM, Inc. All Rights Reserved.
* Copyright (C) 2001-2012 IBM, Inc. All Rights Reserved.
*
********************************************************************/
/********************************************************************************
@ -611,7 +611,7 @@ UChar UCharFile::get() {
// Convert the bytes from the temp array to a Unicode char.
i = 0;
uint32_t cp;
UTF8_NEXT_CHAR_UNSAFE(bytes, i, cp);
U8_NEXT_UNSAFE(bytes, i, cp);
c = (UChar)cp;
if (cp >= 0x10000) {