mirror of
https://github.com/unicode-org/icu.git
synced 2025-04-14 17:24:01 +00:00
ICU-4639 Initial checkin of C port of CharsetDetection.
X-SVN-Rev: 19069
This commit is contained in:
Eric Mader
parent
838f9c3535
commit
b452ca48e6
29 changed files with 4881 additions and 29 deletions
|
|
@ -75,6 +75,7 @@ nultrans.o remtrans.o casetrn.o titletrn.o tolowtrn.o toupptrn.o anytrans.o \
|
|||
name2uni.o uni2name.o nortrans.o quant.o transreg.o \
|
||||
regexcmp.o rematch.o repattrn.o regexst.o uregex.o uregexc.o \
|
||||
ulocdata.o measfmt.o currfmt.o curramt.o currunit.o measure.o utmscale.o \
|
||||
csdetect.o csmatch.o csr2022.o csrecog.o csrmbcs.o csrsbcs.o csrucode.o csrutf8.o inputext.o \
|
||||
windtfmt.o winnmfmt.o
|
||||
|
||||
## Header files to install
|
||||
|
|
|
|||
280
icu4c/source/i18n/csdetect.cpp
Normal file
280
icu4c/source/i18n/csdetect.cpp
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csdetect.h"
|
||||
#include "csmatch.h"
|
||||
|
||||
#include "cmemory.h"
|
||||
#include "umutex.h"
|
||||
#include "ucln_in.h"
|
||||
#include "inputext.h"
|
||||
#include "csrsbcs.h"
|
||||
#include "csrmbcs.h"
|
||||
#include "csrutf8.h"
|
||||
#include "csrucode.h"
|
||||
#include "csr2022.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
|
||||
|
||||
#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
|
||||
#define DELETE_ARRAY(array) uprv_free((void *) (array))
|
||||
|
||||
U_CDECL_BEGIN
|
||||
static UBool U_CALLCONV csdet_cleanup(void)
|
||||
{
|
||||
return UCharsetDetector::cleanup();
|
||||
}
|
||||
U_CDECL_END
|
||||
|
||||
CharsetRecognizer **UCharsetDetector::fCSRecognizers = NULL;
|
||||
|
||||
int32_t UCharsetDetector::fCSRecognizers_size = 0;
|
||||
|
||||
void UCharsetDetector::setRecognizers()
|
||||
{
|
||||
UBool needsInit;
|
||||
CharsetRecognizer **recognizers;
|
||||
|
||||
umtx_lock(NULL);
|
||||
needsInit = (UBool) (fCSRecognizers == NULL);
|
||||
umtx_unlock(NULL);
|
||||
|
||||
if (needsInit) {
|
||||
CharsetRecognizer *tempArray[] = {
|
||||
new CharsetRecog_UTF8(),
|
||||
|
||||
new CharsetRecog_UTF_16_BE(),
|
||||
new CharsetRecog_UTF_16_LE(),
|
||||
new CharsetRecog_UTF_32_BE(),
|
||||
new CharsetRecog_UTF_32_LE(),
|
||||
|
||||
new CharsetRecog_8859_1_en(),
|
||||
new CharsetRecog_8859_1_da(),
|
||||
new CharsetRecog_8859_1_de(),
|
||||
new CharsetRecog_8859_1_es(),
|
||||
new CharsetRecog_8859_1_fr(),
|
||||
new CharsetRecog_8859_1_it(),
|
||||
new CharsetRecog_8859_1_nl(),
|
||||
new CharsetRecog_8859_1_no(),
|
||||
new CharsetRecog_8859_1_pt(),
|
||||
new CharsetRecog_8859_1_sv(),
|
||||
new CharsetRecog_8859_2_cs(),
|
||||
new CharsetRecog_8859_2_hu(),
|
||||
new CharsetRecog_8859_2_pl(),
|
||||
new CharsetRecog_8859_2_ro(),
|
||||
new CharsetRecog_8859_5_ru(),
|
||||
new CharsetRecog_8859_6_ar(),
|
||||
new CharsetRecog_8859_7_el(),
|
||||
new CharsetRecog_8859_8_I_he(),
|
||||
new CharsetRecog_8859_8_he(),
|
||||
new CharsetRecog_windows_1251(),
|
||||
new CharsetRecog_windows_1256(),
|
||||
new CharsetRecog_KOI8_R(),
|
||||
new CharsetRecog_8859_9_tr(),
|
||||
new CharsetRecog_sjis(),
|
||||
new CharsetRecog_gb_18030(),
|
||||
new CharsetRecog_euc_jp(),
|
||||
new CharsetRecog_euc_kr(),
|
||||
|
||||
new CharsetRecog_2022JP(),
|
||||
new CharsetRecog_2022KR(),
|
||||
new CharsetRecog_2022CN()
|
||||
};
|
||||
int32_t rCount = ARRAY_SIZE(tempArray);
|
||||
int32_t r;
|
||||
|
||||
recognizers = NEW_ARRAY(CharsetRecognizer *, rCount);
|
||||
for (r = 0; r < rCount; r += 1) {
|
||||
recognizers[r] = tempArray[r];
|
||||
}
|
||||
|
||||
umtx_lock(NULL);
|
||||
if (fCSRecognizers == NULL) {
|
||||
fCSRecognizers = recognizers;
|
||||
fCSRecognizers_size = rCount;
|
||||
}
|
||||
umtx_unlock(NULL);
|
||||
|
||||
if (fCSRecognizers != recognizers) {
|
||||
for (r = 0; r < rCount; r += 1) {
|
||||
delete recognizers[r];
|
||||
recognizers[r] = NULL;
|
||||
}
|
||||
|
||||
DELETE_ARRAY(recognizers);
|
||||
}
|
||||
|
||||
recognizers = NULL;
|
||||
ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup);
|
||||
}
|
||||
}
|
||||
|
||||
UBool UCharsetDetector::cleanup()
|
||||
{
|
||||
if (fCSRecognizers != NULL) {
|
||||
for(int32_t r = 0; r < fCSRecognizers_size; r += 1) {
|
||||
delete fCSRecognizers[r];
|
||||
fCSRecognizers[r] = NULL;
|
||||
}
|
||||
|
||||
DELETE_ARRAY(fCSRecognizers);
|
||||
fCSRecognizers = NULL;
|
||||
fCSRecognizers_size = 0;
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
UCharsetDetector::UCharsetDetector()
|
||||
: textIn(new InputText()), fStripTags(FALSE), fFreshTextSet(FALSE)
|
||||
{
|
||||
setRecognizers();
|
||||
|
||||
resultArray = new UCharsetMatch *[fCSRecognizers_size];
|
||||
|
||||
for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
|
||||
resultArray[i] = new UCharsetMatch;
|
||||
}
|
||||
}
|
||||
|
||||
UCharsetDetector::~UCharsetDetector()
|
||||
{
|
||||
delete textIn;
|
||||
|
||||
for(int32_t i = 0; i < fCSRecognizers_size; i += 1) {
|
||||
delete resultArray[i];
|
||||
}
|
||||
|
||||
delete [] resultArray;
|
||||
}
|
||||
|
||||
void UCharsetDetector::setText(const char *in, int32_t len)
|
||||
{
|
||||
textIn->setText(in, len);
|
||||
fFreshTextSet = TRUE;
|
||||
}
|
||||
|
||||
UBool UCharsetDetector::setStripTagsFlag(UBool flag)
|
||||
{
|
||||
UBool temp = fStripTags;
|
||||
fStripTags = flag;
|
||||
fFreshTextSet = TRUE;
|
||||
return temp;
|
||||
}
|
||||
|
||||
UBool UCharsetDetector::getStripTagsFlag() const
|
||||
{
|
||||
return fStripTags;
|
||||
}
|
||||
|
||||
void UCharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const
|
||||
{
|
||||
textIn->setDeclaredEncoding(encoding,len);
|
||||
}
|
||||
|
||||
int32_t UCharsetDetector::getDetectableCount()
|
||||
{
|
||||
setRecognizers();
|
||||
|
||||
return fCSRecognizers_size;
|
||||
}
|
||||
|
||||
const UCharsetMatch* UCharsetDetector::detect(UErrorCode &status)
|
||||
{
|
||||
int32_t maxMatchesFound = 0;
|
||||
|
||||
detectAll(maxMatchesFound, status);
|
||||
|
||||
if(maxMatchesFound > 0) {
|
||||
return resultArray[0];
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// this sort of conversion is explicitly mentioned in the C++ standard
|
||||
// in section 4.4:
|
||||
|
||||
// [Note: if a program could assign a pointer of type T** to a pointer of type
|
||||
// const T** (that is, if line //1 below was allowed), a program could
|
||||
// inadvertently modify a const object (as it is done on line //2). For
|
||||
// example,
|
||||
|
||||
// int32_t main() {
|
||||
// const char c = 'c';
|
||||
// char* pc;
|
||||
// const char** pcc = &pc;//1: not allowed
|
||||
// *pcc = &c;
|
||||
// *pc = 'C';//2: modifies a const object
|
||||
// }
|
||||
const UCharsetMatch * const *UCharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status)
|
||||
{
|
||||
int32_t resultCount = 0;
|
||||
|
||||
if(!textIn->isSet()) {
|
||||
status = U_MISSING_RESOURCE_ERROR;// TODO: Need to set proper status code for input text not set
|
||||
|
||||
return 0;
|
||||
} else if(fFreshTextSet) {
|
||||
CharsetRecognizer *csr;
|
||||
int32_t detectResults;
|
||||
int32_t confidence;
|
||||
|
||||
textIn->MungeInput(fStripTags);
|
||||
|
||||
// Iterate over all possible charsets, remember all that
|
||||
// give a match quality > 0.
|
||||
for (int32_t i = 0; i < fCSRecognizers_size; i += 1) {
|
||||
csr = fCSRecognizers[i];
|
||||
detectResults = csr->match(textIn);
|
||||
confidence = detectResults;
|
||||
|
||||
if (confidence > 0) {
|
||||
resultArray[resultCount++]->set(textIn,csr,confidence);
|
||||
}
|
||||
}
|
||||
|
||||
for(int32_t i = resultCount; i < fCSRecognizers_size; i += 1) {
|
||||
resultArray[i]->set(textIn,0,0);
|
||||
}
|
||||
|
||||
//Bubble sort
|
||||
for(int32_t i = resultCount; i > 1; i -= 1) {
|
||||
for(int32_t j = 0; j < i-1; j += 1) {
|
||||
if(resultArray[j]->getConfidence() < resultArray[j+1]->getConfidence()) {
|
||||
UCharsetMatch *temp = resultArray[j];
|
||||
resultArray[j]= resultArray[j+1];
|
||||
resultArray[j+1]=temp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fFreshTextSet = FALSE;
|
||||
}
|
||||
|
||||
maxMatchesFound = resultCount;
|
||||
|
||||
return resultArray;
|
||||
}
|
||||
|
||||
const char *UCharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const
|
||||
{
|
||||
if( index > fCSRecognizers_size-1 || index < 0) {
|
||||
status = U_INDEX_OUTOFBOUNDS_ERROR;
|
||||
|
||||
return 0;
|
||||
} else {
|
||||
return fCSRecognizers[index]->getName();
|
||||
}
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
57
icu4c/source/i18n/csdetect.h
Normal file
57
icu4c/source/i18n/csdetect.h
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSDETECT_H
|
||||
#define __CSDETECT_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uobject.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class InputText;
|
||||
class CharsetRecognizer;
|
||||
struct UCharsetMatch;
|
||||
|
||||
struct UCharsetDetector : public UMemory
|
||||
{
|
||||
private:
|
||||
InputText *textIn;
|
||||
UCharsetMatch **resultArray;
|
||||
UBool fStripTags; // If true, setText() will strip tags from input text.
|
||||
UBool fFreshTextSet;
|
||||
static CharsetRecognizer **fCSRecognizers;
|
||||
static int32_t fCSRecognizers_size;
|
||||
static void setRecognizers();
|
||||
|
||||
public:
|
||||
UCharsetDetector();
|
||||
|
||||
~UCharsetDetector();
|
||||
|
||||
void setText(const char *in, int32_t len);
|
||||
|
||||
const UCharsetMatch * const *detectAll(int32_t &maxMatchesFound, UErrorCode &status);
|
||||
|
||||
const UCharsetMatch *detect(UErrorCode& status);
|
||||
|
||||
void setDeclaredEncoding(const char *encoding, int32_t len) const;
|
||||
|
||||
UBool setStripTagsFlag(UBool flag);
|
||||
|
||||
UBool getStripTagsFlag() const;
|
||||
|
||||
const char *getCharsetName(int32_t index, UErrorCode& status) const;
|
||||
|
||||
static int32_t getDetectableCount();
|
||||
|
||||
static UBool cleanup();
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSDETECT_H */
|
||||
57
icu4c/source/i18n/csmatch.cpp
Normal file
57
icu4c/source/i18n/csmatch.cpp
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/ucnv.h"
|
||||
|
||||
#include "csmatch.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
#include "inputext.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
UCharsetMatch::UCharsetMatch()
|
||||
: csr(0), confidence(0)
|
||||
{
|
||||
// nothing else to do.
|
||||
}
|
||||
|
||||
void UCharsetMatch::set(InputText *input, CharsetRecognizer *cr, int32_t conf)
|
||||
{
|
||||
textIn = input;
|
||||
csr = cr;
|
||||
confidence = conf;
|
||||
}
|
||||
|
||||
const char* UCharsetMatch::getName()const
|
||||
{
|
||||
return csr->getName();
|
||||
}
|
||||
|
||||
const char* UCharsetMatch::getLanguage()const
|
||||
{
|
||||
return csr->getLanguage();
|
||||
}
|
||||
|
||||
int32_t UCharsetMatch::getConfidence()const
|
||||
{
|
||||
return confidence;
|
||||
}
|
||||
|
||||
int32_t UCharsetMatch::getUChars(UChar *buf, int32_t cap, UErrorCode *status) const
|
||||
{
|
||||
UConverter *conv = ucnv_open(getName(), status);
|
||||
int32_t result = ucnv_toUChars(conv, buf, cap, (const char *) textIn->fRawInput, textIn->fRawLength, status);
|
||||
|
||||
ucnv_close(conv);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
42
icu4c/source/i18n/csmatch.h
Normal file
42
icu4c/source/i18n/csmatch.h
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSMATCH_H
|
||||
#define __CSMATCH_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uobject.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class InputText;
|
||||
class CharsetRecognizer;
|
||||
|
||||
struct UCharsetMatch : public UMemory
|
||||
{
|
||||
private:
|
||||
CharsetRecognizer *csr;
|
||||
InputText *textIn;
|
||||
int32_t confidence;
|
||||
|
||||
public:
|
||||
UCharsetMatch();
|
||||
|
||||
void set(InputText *input, CharsetRecognizer *cr, int32_t conf);
|
||||
|
||||
const char *getName()const;
|
||||
|
||||
const char *getLanguage()const;
|
||||
|
||||
int32_t getConfidence()const;
|
||||
|
||||
int32_t getUChars(UChar *buf, int32_t cap, UErrorCode *status) const;
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSMATCH_H */
|
||||
176
icu4c/source/i18n/csr2022.cpp
Normal file
176
icu4c/source/i18n/csr2022.cpp
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "cstring.h"
|
||||
|
||||
#include "csr2022.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
/**
|
||||
* Matching function shared among the 2022 detectors JP, CN and KR
|
||||
* Counts up the number of legal an unrecognized escape sequences in
|
||||
* the sample of text, and computes a score based on the total number &
|
||||
* the proportion that fit the encoding.
|
||||
*
|
||||
*
|
||||
* @param text the byte buffer containing text to analyse
|
||||
* @param textLen the size of the text in the byte.
|
||||
* @param escapeSequences the byte escape sequences to test for.
|
||||
* @return match quality, in the range of 0-100.
|
||||
*/
|
||||
int32_t CharsetRecog_2022::match_2022(const uint8_t *text, int32_t textLen, const uint8_t escapeSequences[][5], int32_t escapeSequences_length)
|
||||
{
|
||||
int32_t i, j;
|
||||
int32_t escN;
|
||||
int32_t hits = 0;
|
||||
int32_t misses = 0;
|
||||
int32_t shifts = 0;
|
||||
int32_t quality;
|
||||
|
||||
i = 0;
|
||||
while(i < textLen) {
|
||||
if(text[i] == 0x1B) {
|
||||
escN = 0;
|
||||
while(escN < escapeSequences_length) {
|
||||
int32_t seq_length = uprv_strlen((const char *) escapeSequences[escN]);
|
||||
const uint8_t *seq = escapeSequences[escN];
|
||||
|
||||
j = 1;
|
||||
while(j < seq_length) {
|
||||
if(seq[j] != text[i+j]) {
|
||||
goto checkEscapes;
|
||||
}
|
||||
|
||||
j += 1;
|
||||
}
|
||||
|
||||
hits += 1;
|
||||
i += seq_length-1;
|
||||
goto scanInput;
|
||||
checkEscapes:
|
||||
escN += 1;
|
||||
}
|
||||
|
||||
misses += 1;
|
||||
}
|
||||
|
||||
if( text[i]== 0x0e || text[i] == 0x0f){
|
||||
shifts += 1;
|
||||
}
|
||||
|
||||
scanInput:
|
||||
i += 1;
|
||||
}
|
||||
|
||||
if (hits == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//
|
||||
// Initial quality is based on relative proportion of recongized vs.
|
||||
// unrecognized escape sequences.
|
||||
// All good: quality = 100;
|
||||
// half or less good: quality = 0;
|
||||
// linear inbetween.
|
||||
quality = (100*hits - 100*misses) / (hits + misses);
|
||||
|
||||
// Back off quality if there were too few escape sequences seen.
|
||||
// Include shifts in this computation, so that KR does not get penalized
|
||||
// for having only a single Escape sequence, but many shifts.
|
||||
if (hits+shifts < 5) {
|
||||
quality -= (5-(hits+shifts))*10;
|
||||
}
|
||||
|
||||
if (quality < 0) {
|
||||
quality = 0;
|
||||
}
|
||||
|
||||
return quality;
|
||||
}
|
||||
|
||||
|
||||
const uint8_t CharsetRecog_2022JP::escapeSequences[][5] = {
|
||||
{0x1b, 0x24, 0x28, 0x43, 0x00}, // KS X 1001:1992
|
||||
{0x1b, 0x24, 0x28, 0x44, 0x00}, // JIS X 212-1990
|
||||
{0x1b, 0x24, 0x40, 0x00, 0x00}, // JIS C 6226-1978
|
||||
{0x1b, 0x24, 0x41, 0x00, 0x00}, // GB 2312-80
|
||||
{0x1b, 0x24, 0x42, 0x00, 0x00}, // JIS X 208-1983
|
||||
{0x1b, 0x26, 0x40, 0x00, 0x00}, // JIS X 208 1990, 1997
|
||||
{0x1b, 0x28, 0x42, 0x00, 0x00}, // ASCII
|
||||
{0x1b, 0x28, 0x48, 0x00, 0x00}, // JIS-Roman
|
||||
{0x1b, 0x28, 0x49, 0x00, 0x00}, // Half-width katakana
|
||||
{0x1b, 0x28, 0x4a, 0x00, 0x00}, // JIS-Roman
|
||||
{0x1b, 0x2e, 0x41, 0x00, 0x00}, // ISO 8859-1
|
||||
{0x1b, 0x2e, 0x46, 0x00, 0x00} // ISO 8859-7
|
||||
};
|
||||
|
||||
const uint8_t CharsetRecog_2022KR::escapeSequences[][5] = {
|
||||
{0x1b, 0x24, 0x29, 0x43, 0x00}
|
||||
};
|
||||
|
||||
const uint8_t CharsetRecog_2022CN::escapeSequences[][5] = {
|
||||
{0x1b, 0x24, 0x29, 0x41, 0x00}, // GB 2312-80
|
||||
{0x1b, 0x24, 0x29, 0x47, 0x00}, // CNS 11643-1992 Plane 1
|
||||
{0x1b, 0x24, 0x2A, 0x48, 0x00}, // CNS 11643-1992 Plane 2
|
||||
{0x1b, 0x24, 0x29, 0x45, 0x00}, // ISO-IR-165
|
||||
{0x1b, 0x24, 0x2B, 0x49, 0x00}, // CNS 11643-1992 Plane 3
|
||||
{0x1b, 0x24, 0x2B, 0x4A, 0x00}, // CNS 11643-1992 Plane 4
|
||||
{0x1b, 0x24, 0x2B, 0x4B, 0x00}, // CNS 11643-1992 Plane 5
|
||||
{0x1b, 0x24, 0x2B, 0x4C, 0x00}, // CNS 11643-1992 Plane 6
|
||||
{0x1b, 0x24, 0x2B, 0x4D, 0x00}, // CNS 11643-1992 Plane 7
|
||||
{0x1b, 0x4e, 0x00, 0x00, 0x00}, // SS2
|
||||
{0x1b, 0x4f, 0x00, 0x00, 0x00}, // SS3
|
||||
};
|
||||
|
||||
const int32_t CharsetRecog_2022JP::escapeSequences_length =
|
||||
sizeof(CharsetRecog_2022JP::escapeSequences) / sizeof(CharsetRecog_2022JP::escapeSequences[0]);
|
||||
|
||||
const int32_t CharsetRecog_2022KR::escapeSequences_length =
|
||||
sizeof(CharsetRecog_2022KR::escapeSequences) / sizeof(CharsetRecog_2022KR::escapeSequences[0]);
|
||||
|
||||
const int32_t CharsetRecog_2022CN::escapeSequences_length =
|
||||
sizeof(CharsetRecog_2022CN::escapeSequences) / sizeof(CharsetRecog_2022CN::escapeSequences[0]);
|
||||
|
||||
const char *CharsetRecog_2022JP::getName() const
|
||||
{
|
||||
return "ISO-2022-JP";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_2022JP::match(InputText *textIn)
|
||||
{
|
||||
return match_2022(textIn->fInputBytes, textIn->fInputLen, escapeSequences, escapeSequences_length);
|
||||
}
|
||||
|
||||
const char *CharsetRecog_2022KR::getName() const
|
||||
{
|
||||
return "ISO-2022-KR";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_2022KR::match(InputText *textIn)
|
||||
{
|
||||
return match_2022(textIn->fInputBytes, textIn->fInputLen, escapeSequences, escapeSequences_length);
|
||||
}
|
||||
|
||||
const char *CharsetRecog_2022CN::getName() const
|
||||
{
|
||||
return "ISO-2022-CN";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_2022CN::match(InputText *textIn)
|
||||
{
|
||||
return match_2022(textIn->fInputBytes, textIn->fInputLen, escapeSequences, escapeSequences_length);
|
||||
}
|
||||
|
||||
CharsetRecog_2022::~CharsetRecog_2022()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
95
icu4c/source/i18n/csr2022.h
Normal file
95
icu4c/source/i18n/csr2022.h
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSR2022_H
|
||||
#define __CSR2022_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
/**
|
||||
* class CharsetRecog_2022 part of the ICU charset detection imlementation.
|
||||
* This is a superclass for the individual detectors for
|
||||
* each of the detectable members of the ISO 2022 family
|
||||
* of encodings.
|
||||
*
|
||||
* The separate classes are nested within this class.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
class CharsetRecog_2022 : public CharsetRecognizer
|
||||
{
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_2022() = 0;
|
||||
|
||||
protected:
|
||||
|
||||
/**
|
||||
* Matching function shared among the 2022 detectors JP, CN and KR
|
||||
* Counts up the number of legal an unrecognized escape sequences in
|
||||
* the sample of text, and computes a score based on the total number &
|
||||
* the proportion that fit the encoding.
|
||||
*
|
||||
*
|
||||
* @param text the byte buffer containing text to analyse
|
||||
* @param textLen the size of the text in the byte.
|
||||
* @param escapeSequences the byte escape sequences to test for.
|
||||
* @return match quality, in the range of 0-100.
|
||||
*/
|
||||
int32_t match_2022(const uint8_t *text, int32_t textLen, const uint8_t escapeSequences[][5], int32_t escapeSequences_length);
|
||||
|
||||
};
|
||||
|
||||
class CharsetRecog_2022JP :public CharsetRecog_2022
|
||||
{
|
||||
private:
|
||||
static const uint8_t escapeSequences[][5];
|
||||
static const int32_t escapeSequences_length;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_2022JP() {}
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_2022KR :public CharsetRecog_2022 {
|
||||
private:
|
||||
static const uint8_t escapeSequences[][5];
|
||||
static const int32_t escapeSequences_length;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_2022KR() {}
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
|
||||
};
|
||||
|
||||
class CharsetRecog_2022CN :public CharsetRecog_2022
|
||||
{
|
||||
private:
|
||||
static const uint8_t escapeSequences[][5];
|
||||
static const int32_t escapeSequences_length;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_2022CN() {}
|
||||
|
||||
const char* getName() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSR2022_H */
|
||||
24
icu4c/source/i18n/csrecog.cpp
Normal file
24
icu4c/source/i18n/csrecog.cpp
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
CharsetRecognizer::~CharsetRecognizer()
|
||||
{
|
||||
// nothing to do.
|
||||
}
|
||||
|
||||
const char *CharsetRecognizer::getLanguage() const
|
||||
{
|
||||
return "";
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
40
icu4c/source/i18n/csrecog.h
Normal file
40
icu4c/source/i18n/csrecog.h
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSRECOG_H
|
||||
#define __CSRECOG_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uobject.h"
|
||||
|
||||
#include "inputext.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class CharsetRecognizer : public UMemory
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Get the IANA name of this charset.
|
||||
* @return the charset name.
|
||||
*/
|
||||
virtual const char *getName() const = 0;
|
||||
|
||||
/**
|
||||
* Get the ISO language code for this charset.
|
||||
* @return the language code, or <code>null</code> if the language cannot be determined.
|
||||
*/
|
||||
virtual const char *getLanguage() const;
|
||||
|
||||
virtual int32_t match(InputText *textIn) = 0;
|
||||
|
||||
virtual ~CharsetRecognizer();
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSRECOG_H */
|
||||
438
icu4c/source/i18n/csrmbcs.cpp
Normal file
438
icu4c/source/i18n/csrmbcs.cpp
Normal file
|
|
@ -0,0 +1,438 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrmbcs.h"
|
||||
|
||||
#include <math.h>
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
#define min(x,y) ((x<y)?(x):(y))
|
||||
|
||||
const int32_t CharsetRecog_sjis::commonChars []=
|
||||
// TODO: This set of data comes from the character frequency-
|
||||
// of-occurence analysis tool. The data needs to be moved
|
||||
// into a resource and loaded from there.
|
||||
{0x82cc, 0x82c5, 0x82a2, 0x815b, 0x8141, 0x82b5, 0x82c9, 0x82b7, 0x8142, 0x82c4,
|
||||
0x82c6, 0x82cd, 0x82dc, 0x82f0, 0x82e9, 0x82c8, 0x82aa, 0x8393, 0x82bd, 0x8358,
|
||||
0x82ea, 0x8343, 0x82a4, 0x82a9, 0x8367, 0x82b1, 0x8145, 0x82e0, 0x838b, 0x834e,
|
||||
0x82e8, 0x82e7, 0x8140, 0x8362, 0x8389, 0x82c1, 0x838a, 0x82ab, 0x8376, 0x82b3,
|
||||
0x82f1, 0x82a0, 0x8368, 0x93fa, 0x8175, 0x8176, 0x835e, 0x82e6, 0x8357, 0x82ad,
|
||||
0x8381, 0x82a6, 0x82b9, 0x82bb, 0x82be, 0x8341, 0x8374, 0x82af, 0x9056, 0x82a8,
|
||||
0x82c2, 0x8354, 0x8e67, 0x8375, 0x82c7, 0x95f1, 0x8356, 0x967b, 0x92e8, 0x8345,
|
||||
0x82ce, 0x8385, 0x9770, 0x82df, 0x82dd, 0x836f, 0x8342, 0x8ca7, 0x8352, 0x837d,
|
||||
0x838d, 0x8346, 0x834f, 0x8380, 0x82ed, 0x8d73, 0x8349, 0x8365, 0x8fee, 0x95b6,
|
||||
0x8169, 0x816a, 0x836a, 0x8dec, 0x82bf, 0x834c, 0x8366, 0x82e2, 0x838c, 0x945c};
|
||||
|
||||
const int32_t CharsetRecog_euc_jp::commonChars[] =
|
||||
// TODO: This set of data comes from the character frequency-
|
||||
// of-occurence analysis tool. The data needs to be moved
|
||||
// into a resource and loaded from there.
|
||||
{0xa4ce, 0xa4c7, 0xa4a4, 0xa1bc, 0xa1a2, 0xa4b7, 0xa4cb, 0xa4b9, 0xa1a3, 0xa4c6,
|
||||
0xa4c8, 0xa4cf, 0xa4de, 0xa4f2, 0xa4eb, 0xa4ca, 0xa4ac, 0xa5f3, 0xa4bf, 0xa5b9,
|
||||
0xa4ec, 0xa5a4, 0xa4a6, 0xa4ab, 0xa5c8, 0xa4b3, 0xa1a6, 0xa4e2, 0xa5eb, 0xa5af,
|
||||
0xa4ea, 0xa4e9, 0xa1a1, 0xa5c3, 0xa5e9, 0xa4c3, 0xa5ea, 0xa4ad, 0xa5d7, 0xa4b5,
|
||||
0xa4f3, 0xa4a2, 0xa5c9, 0xc6fc, 0xa1d6, 0xa1d7, 0xa5bf, 0xa4e8, 0xa5b8, 0xa4af,
|
||||
0xa5e1, 0xa4a8, 0xa4bb, 0xa4bd, 0xa4c0, 0xa5a2, 0xa5d5, 0xa4b1, 0xbfb7, 0xa4aa,
|
||||
0xa4c4, 0xa5b5, 0xbbc8, 0xa5d6, 0xa4c9, 0xcaf3, 0xa5b7, 0xcbdc, 0xc4ea, 0xa5a6,
|
||||
0xa4d0, 0xa5e5, 0xcdd1, 0xa4e1, 0xa4df, 0xa5d0, 0xa5a3, 0xb8a9, 0xa5b3, 0xa5de,
|
||||
0xa5ed, 0xa5a7, 0xa5b0, 0xa5e0, 0xa4ef, 0xb9d4, 0xa5aa, 0xa5c6, 0xbef0, 0xcab8,
|
||||
0xa1ca, 0xa1cb, 0xa5cb, 0xbaee, 0xa4c1, 0xa5ad, 0xa5c7, 0xa4e4, 0xa5ec, 0xc7bd};
|
||||
|
||||
const int32_t CharsetRecog_euc_kr::commonChars[] =
|
||||
// TODO: This set of data comes from the character frequency-
|
||||
// of-occurence analysis tool. The data needs to be moved
|
||||
// into a resource and loaded from there.
|
||||
{0xc0cc, 0xb4d9, 0xb4c2, 0xc0c7, 0xbfa1, 0xc7cf, 0xb0a1, 0xb0ed, 0xc7d1, 0xc1f6,
|
||||
0xc0bb, 0xb7ce, 0xb1e2, 0xbcad, 0xc0ba, 0xbbe7, 0xc1a4, 0xc0da, 0xb5b5, 0xb8a6,
|
||||
0xbeee, 0xb4cf, 0xbcf6, 0xbdc3, 0xb1d7, 0xb4eb, 0xb8ae, 0xc0ce, 0xb3aa, 0xbec6,
|
||||
0xc0d6, 0xbab8, 0xb5e9, 0xb6f3, 0xc7d8, 0xb0cd, 0xc0cf, 0xbdba, 0xc0b8, 0xb1b9,
|
||||
0xc1a6, 0xb9fd, 0xbbf3, 0xb0d4, 0xb8e9, 0xb8b8, 0xb0fa, 0xc0fb, 0xbace, 0xc1d6,
|
||||
0xbfa9, 0xc0fc, 0xbfeb, 0xb9ae, 0xc6ae, 0xbbfd, 0xbcba, 0xc0a7, 0xbff8, 0xb5c7,
|
||||
0xbfe4, 0xbfec, 0xbdc5, 0xc7d2, 0xc7e5, 0xb0fc, 0xb1b8, 0xbaf1, 0xbedf, 0xc5cd,
|
||||
0xb8b6, 0xbdc0, 0xb7af, 0xb5bf, 0xb3bb, 0xc8ad, 0xc0bd, 0xb0b3, 0xc4a1, 0xb7c2,
|
||||
0xb9ab, 0xc0af, 0xbef8, 0xb5a5, 0xbcd2, 0xb9ce, 0xc1df, 0xbfc0, 0xc1f8, 0xb0e6,
|
||||
0xb1c7, 0xbad0, 0xbefa, 0xc0e5, 0xbec8, 0xc1b6, 0xb8bb, 0xb0f8, 0xb9cc, 0xb0c5};
|
||||
|
||||
const int32_t CharsetRecog_gb_18030::commonChars[] =
|
||||
// TODO: This set of data comes from the character frequency-
|
||||
// of-occurence analysis tool. The data needs to be moved
|
||||
// into a resource and loaded from there.
|
||||
{0xa3ac, 0xb5c4, 0xa1a1, 0xa1a4, 0xa1a3, 0xcac7, 0xd2bb, 0xb4f3, 0xd4da, 0xd6d0,
|
||||
0xcafd, 0xd3d0, 0xa1f3, 0xb2bb, 0xa3ba, 0xbbfa, 0xc8cb, 0xa1a2, 0xd3c3, 0xd1a7,
|
||||
0xc8d5, 0xbedd, 0xb8f6, 0xd0c2, 0xcdf8, 0xd2aa, 0xb9fa, 0xc1cb, 0xc9cf, 0xa1b0,
|
||||
0xa1b1, 0xced2, 0xbcfe, 0xcec4, 0xd2d4, 0xc4dc, 0xc0b4, 0xd4c2, 0xcab1, 0xd0d0,
|
||||
0xbdcc, 0xbfc9, 0xb6d4, 0xbcdb, 0xb1be, 0xb3f6, 0xb8b4, 0xc9fa, 0xb1b8, 0xbcbc,
|
||||
0xcfc2, 0xbacd, 0xbecd, 0xb3c9, 0xd5e2, 0xb8df, 0xb7d6, 0xc5cc, 0xbfc6, 0xbbe1,
|
||||
0xceaa, 0xc8e7, 0xcfb5, 0xa1f1, 0xc4ea, 0xb1a8, 0xb6af, 0xc0ed, 0xd3fd, 0xb7a2,
|
||||
0xc8ab, 0xb7bd, 0xcee5, 0xc2db, 0xbba7, 0xd0d4, 0xb9c9, 0xc3c7, 0xb9fd, 0xcad0,
|
||||
0xb5e3, 0xbbd6, 0xcfd6, 0xcab5, 0xd2b2, 0xbfb4, 0xb6e0, 0xccec, 0xc7f8, 0xd0c5,
|
||||
0xcad6, 0xb9d8, 0xb5bd, 0xb7dd, 0xc6f7, 0xcaf5, 0xa3a1, 0xb7a8, 0xb9ab, 0xd2b5,
|
||||
0xcbf9, 0xcdbc, 0xc6e4, 0xd3da, 0xd0a1, 0xd1a1, 0xd3ce, 0xbfaa, 0xb4e6, 0xc4bf,
|
||||
0xd7f7, 0xb5e7, 0xcdb3, 0xc7e9, 0xd7ee, 0xc6c0, 0xcfdf, 0xb5d8, 0xb5c0, 0xbead,
|
||||
0xb4c5, 0xc6b7, 0xc4da, 0xd0c4, 0xb9a4, 0xd4aa, 0xc2bc, 0xc3c0, 0xbaf3, 0xcabd,
|
||||
0xbcd2, 0xcef1, 0xbdab, 0xa3ad, 0xa3bf, 0xb3a4, 0xb9fb, 0xd6ae, 0xc1bf, 0xbbd8,
|
||||
0xb8f1, 0xb6f8, 0xb6a8, 0xcde2, 0xbac3, 0xb3cc, 0xccd8, 0xd7d4, 0xcbb5};
|
||||
|
||||
const int32_t CharsetRecog_sjis::commonCharsLen =
|
||||
sizeof(CharsetRecog_sjis::commonChars) / sizeof(CharsetRecog_sjis::commonChars[0]);
|
||||
|
||||
const int32_t CharsetRecog_euc_jp::commonCharsLen =
|
||||
sizeof(CharsetRecog_euc_jp::commonChars) / sizeof(CharsetRecog_euc_jp::commonChars[0]);
|
||||
|
||||
const int32_t CharsetRecog_euc_kr::commonCharsLen =
|
||||
sizeof(CharsetRecog_euc_kr::commonChars) / sizeof(CharsetRecog_euc_kr::commonChars[0]);
|
||||
|
||||
const int32_t CharsetRecog_gb_18030::commonCharsLen =
|
||||
sizeof(CharsetRecog_gb_18030::commonChars) / sizeof(CharsetRecog_gb_18030::commonChars[0]);
|
||||
|
||||
static int32_t binarySearch(const int32_t *array, int32_t len, int32_t value)
|
||||
{
|
||||
int32_t start = 0, end = len-1;
|
||||
int32_t mid = (start+end)/2;
|
||||
|
||||
while(start <= end) {
|
||||
if(array[mid] == value) {
|
||||
return mid;
|
||||
}
|
||||
|
||||
if(array[mid] < value){
|
||||
start = mid+1;
|
||||
} else {
|
||||
end = mid-1;
|
||||
}
|
||||
|
||||
mid = (start+end)/2;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
iteratedChar::iteratedChar():charValue(0), index(0), nextIndex(0), error(FALSE), done(FALSE)
|
||||
{
|
||||
// nothing else to do.
|
||||
}
|
||||
|
||||
void iteratedChar::reset()
|
||||
{
|
||||
charValue = 0;
|
||||
index = -1;
|
||||
nextIndex = 0;
|
||||
error = FALSE;
|
||||
done = FALSE;
|
||||
}
|
||||
|
||||
int32_t iteratedChar::nextByte(InputText *det)
|
||||
{
|
||||
if (nextIndex >= det->fRawLength) {
|
||||
done = TRUE;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
return det->fRawInput[nextIndex++];
|
||||
}
|
||||
|
||||
CharsetRecog_mbcs::~CharsetRecog_mbcs()
|
||||
{
|
||||
// nothing to do.
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_mbcs::match_mbcs(InputText *det, const int32_t commonChars[], int32_t commonCharsLen) {
|
||||
int singleByteCharCount = 0;
|
||||
int doubleByteCharCount = 0;
|
||||
int commonCharCount = 0;
|
||||
int badCharCount = 0;
|
||||
int totalCharCount = 0;
|
||||
int confidence = 0;
|
||||
iteratedChar *iter = new iteratedChar();
|
||||
|
||||
// {
|
||||
for (iter->reset(); nextChar(iter, det);) {
|
||||
totalCharCount += 1;
|
||||
|
||||
if (iter->error) {
|
||||
badCharCount++;
|
||||
} else {
|
||||
if (iter->charValue <= 0xFF) {
|
||||
singleByteCharCount += 1;
|
||||
} else {
|
||||
doubleByteCharCount += 1;
|
||||
|
||||
if (commonChars != 0) {
|
||||
if (binarySearch(commonChars, commonCharsLen, iter->charValue) >= 0){
|
||||
commonCharCount += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (badCharCount >= 2 && badCharCount*5 >= doubleByteCharCount) {
|
||||
// Bail out early if the byte data is not matching the encoding scheme.
|
||||
// break detectBlock;
|
||||
delete iter;
|
||||
return confidence;
|
||||
}
|
||||
}
|
||||
|
||||
delete iter;
|
||||
|
||||
if (doubleByteCharCount == 0 && badCharCount == 0) {
|
||||
// No multi-byte chars.
|
||||
// ASCII file? It's probably not our encoding,
|
||||
// but is not incompatible with our encoding, so don't give it a zero.
|
||||
confidence = 10;
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
//
|
||||
// No match if there are too many characters that don't fit the encoding scheme.
|
||||
// (should we have zero tolerance for these?)
|
||||
//
|
||||
if (doubleByteCharCount < 20*badCharCount) {
|
||||
confidence = 0;
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
if (commonChars == 0) {
|
||||
// We have no statistics on frequently occuring characters.
|
||||
// Assess confidence purely on having a reasonable number of
|
||||
// multi-byte characters (the more the better
|
||||
confidence = 30 + doubleByteCharCount - 20*badCharCount;
|
||||
if (confidence > 100) {
|
||||
confidence = 100;
|
||||
}
|
||||
} else {
|
||||
//
|
||||
// Frequency of occurence statistics exist.
|
||||
//
|
||||
double maxVal = log10((float)doubleByteCharCount / 4);
|
||||
double scaleFactor = 90.0 / maxVal;
|
||||
confidence = (int32_t)(log10((double)commonCharCount+1) * scaleFactor + 10.0);
|
||||
confidence = min(confidence, 100);
|
||||
if(confidence<0)confidence=0;
|
||||
}
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
CharsetRecog_sjis::~CharsetRecog_sjis()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
UBool CharsetRecog_sjis::nextChar(iteratedChar* it, InputText* det) {
|
||||
it->index = it->nextIndex;
|
||||
it->error = FALSE;
|
||||
|
||||
int32_t firstByte = it->charValue = it->nextByte(det);
|
||||
|
||||
if (firstByte < 0) {
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
if (firstByte <= 0x7F || (firstByte > 0xA0 && firstByte <= 0xDF)) {
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
int32_t secondByte = it->nextByte(det);
|
||||
|
||||
if (secondByte < 0) {
|
||||
return FALSE;
|
||||
}
|
||||
it->charValue = (firstByte << 8) | secondByte;
|
||||
if (! ((secondByte >= 0x40 && secondByte <= 0x7F) || (secondByte >= 0x80 && secondByte <= 0xFF))) {
|
||||
// Illegal second byte value.
|
||||
it->error = TRUE;
|
||||
}
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_sjis::match(InputText* det)
|
||||
{
|
||||
return match_mbcs(det, commonChars, commonCharsLen);
|
||||
}
|
||||
|
||||
const char *CharsetRecog_sjis::getName() const
|
||||
{
|
||||
return "Shift_JIS";
|
||||
}
|
||||
|
||||
CharsetRecog_euc::~CharsetRecog_euc()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
UBool CharsetRecog_euc::nextChar(iteratedChar* it, InputText* det) {
|
||||
int32_t firstByte = 0;
|
||||
int32_t secondByte = 0;
|
||||
int32_t thirdByte = 0;
|
||||
// int32_t fourthByte = 0;
|
||||
|
||||
it->index = it->nextIndex;
|
||||
it->error = FALSE;
|
||||
firstByte = it->charValue = it->nextByte(det);
|
||||
|
||||
if (firstByte < 0) {
|
||||
// Ran off the end of the input data
|
||||
it->done = TRUE;
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
if (firstByte <= 0x8D) {
|
||||
// single byte char
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
secondByte = it->nextByte(det);
|
||||
it->charValue = (it->charValue << 8) | secondByte;
|
||||
|
||||
if (firstByte >= 0xA1 && firstByte <= 0xFE) {
|
||||
// Two byte Char
|
||||
if (secondByte < 0xA1) {
|
||||
it->error = TRUE;
|
||||
}
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
if (firstByte == 0x8E) {
|
||||
// Code Set 2.
|
||||
// In EUC-JP, total char size is 2 bytes, only one byte of actual char value.
|
||||
// In EUC-TW, total char size is 4 bytes, three bytes contribute to char value.
|
||||
// We don't know which we've got.
|
||||
// Treat it like EUC-JP. If the data really was EUC-TW, the following two
|
||||
// bytes will look like a well formed 2 byte char.
|
||||
if (secondByte < 0xA1) {
|
||||
it->error = TRUE;
|
||||
}
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
if (firstByte == 0x8F) {
|
||||
// Code set 3.
|
||||
// Three byte total char size, two bytes of actual char value.
|
||||
thirdByte = it->nextByte(det);
|
||||
it->charValue = (it->charValue << 8) | thirdByte;
|
||||
|
||||
if (thirdByte < 0xa1) {
|
||||
it->error = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
return (! it->done);
|
||||
|
||||
}
|
||||
|
||||
CharsetRecog_euc_jp::~CharsetRecog_euc_jp()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_euc_jp::getName() const
|
||||
{
|
||||
return "EUC-JP";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_euc_jp::match(InputText *det)
|
||||
{
|
||||
return match_mbcs(det, commonChars, commonCharsLen);
|
||||
}
|
||||
|
||||
CharsetRecog_euc_kr::~CharsetRecog_euc_kr()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_euc_kr::getName() const
|
||||
{
|
||||
return "EUC-KR";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_euc_kr::match(InputText *det)
|
||||
{
|
||||
return match_mbcs(det, commonChars, commonCharsLen);
|
||||
}
|
||||
|
||||
CharsetRecog_gb_18030::~CharsetRecog_gb_18030()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
UBool CharsetRecog_gb_18030::nextChar(iteratedChar* it, InputText* det) {
|
||||
int32_t firstByte = 0;
|
||||
int32_t secondByte = 0;
|
||||
int32_t thirdByte = 0;
|
||||
int32_t fourthByte = 0;
|
||||
|
||||
it->index = it->nextIndex;
|
||||
it->error = FALSE;
|
||||
firstByte = it->charValue = it->nextByte(det);
|
||||
|
||||
if (firstByte < 0) {
|
||||
// Ran off the end of the input data
|
||||
it->done = TRUE;
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
if (firstByte <= 0x80) {
|
||||
// single byte char
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
secondByte = it->nextByte(det);
|
||||
it->charValue = (it->charValue << 8) | secondByte;
|
||||
|
||||
if (firstByte >= 0x81 && firstByte <= 0xFE) {
|
||||
// Two byte Char
|
||||
if ((secondByte >= 0x40 && secondByte <= 0x7E) || (secondByte >=80 && secondByte <= 0xFE)) {
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
// Four byte char
|
||||
if (secondByte >= 0x30 && secondByte <= 0x39) {
|
||||
thirdByte = it->nextByte(det);
|
||||
|
||||
if (thirdByte >= 0x81 && thirdByte <= 0xFE) {
|
||||
fourthByte = it->nextByte(det);
|
||||
|
||||
if (fourthByte >= 0x30 && fourthByte <= 0x39) {
|
||||
it->charValue = (it->charValue << 16) | (thirdByte << 8) | fourthByte;
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it->error = TRUE;
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
return (! it->done);
|
||||
}
|
||||
|
||||
const char *CharsetRecog_gb_18030::getName() const
|
||||
{
|
||||
return "GB18030";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_gb_18030::match(InputText *det)
|
||||
{
|
||||
return match_mbcs(det, commonChars, commonCharsLen);
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
201
icu4c/source/i18n/csrmbcs.h
Normal file
201
icu4c/source/i18n/csrmbcs.h
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSRMBCS_H
|
||||
#define __CSRMBCS_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
// "Character" iterated character class.
|
||||
// Recognizers for specific mbcs encodings make their "characters" available
|
||||
// by providing a nextChar() function that fills in an instance of iteratedChar
|
||||
// with the next char from the input.
|
||||
// The returned characters are not converted to Unicode, but remain as the raw
|
||||
// bytes (concatenated into an int) from the codepage data.
|
||||
//
|
||||
// For Asian charsets, use the raw input rather than the input that has been
|
||||
// stripped of markup. Detection only considers multi-byte chars, effectively
|
||||
// stripping markup anyway, and double byte chars do occur in markup too.
|
||||
//
|
||||
class iteratedChar
|
||||
{
|
||||
public:
|
||||
int32_t charValue; // 1-4 bytes from the raw input data
|
||||
int32_t index;
|
||||
int32_t nextIndex;
|
||||
UBool error;
|
||||
UBool done;
|
||||
|
||||
public:
|
||||
iteratedChar();
|
||||
void reset();
|
||||
int32_t nextByte(InputText* det);
|
||||
};
|
||||
|
||||
|
||||
class CharsetRecog_mbcs : public CharsetRecognizer {
|
||||
|
||||
protected:
|
||||
/**
|
||||
* Test the match of this charset with the input text data
|
||||
* which is obtained via the CharsetDetector object.
|
||||
*
|
||||
* @param det The CharsetDetector, which contains the input text
|
||||
* to be checked for being in this charset.
|
||||
* @return Two values packed into one int (Damn java, anyhow)
|
||||
* <br/>
|
||||
* bits 0-7: the match confidence, ranging from 0-100
|
||||
* <br/>
|
||||
* bits 8-15: The match reason, an enum-like value.
|
||||
*/
|
||||
int32_t match_mbcs(InputText* det, const int32_t commonChars[], int32_t commonCharsLen);
|
||||
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_mbcs();
|
||||
|
||||
/**
|
||||
* Get the IANA name of this charset.
|
||||
* @return the charset name.
|
||||
*/
|
||||
|
||||
const char *getName() const = 0 ;
|
||||
int32_t match(InputText* det) = 0;
|
||||
|
||||
/**
|
||||
* Get the next character (however many bytes it is) from the input data
|
||||
* Subclasses for specific charset encodings must implement this function
|
||||
* to get characters according to the rules of their encoding scheme.
|
||||
*
|
||||
* This function is not a method of class iteratedChar only because
|
||||
* that would require a lot of extra derived classes, which is awkward.
|
||||
* @param it The iteratedChar "struct" into which the returned char is placed.
|
||||
* @param det The charset detector, which is needed to get at the input byte data
|
||||
* being iterated over.
|
||||
* @return True if a character was returned, false at end of input.
|
||||
*/
|
||||
virtual UBool nextChar(iteratedChar *it, InputText *textIn) = 0;
|
||||
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Shift-JIS charset recognizer.
|
||||
*
|
||||
*/
|
||||
class CharsetRecog_sjis : public CharsetRecog_mbcs {
|
||||
private:
|
||||
static const int32_t commonChars [];
|
||||
static const int32_t commonCharsLen;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_sjis();
|
||||
|
||||
UBool nextChar(iteratedChar *it, InputText *det);
|
||||
|
||||
int32_t match(InputText *det);
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* EUC charset recognizers. One abstract class that provides the common function
|
||||
* for getting the next character according to the EUC encoding scheme,
|
||||
* and nested derived classes for EUC_KR, EUC_JP, EUC_CN.
|
||||
*
|
||||
*/
|
||||
class CharsetRecog_euc : public CharsetRecog_mbcs
|
||||
{
|
||||
public:
|
||||
virtual ~CharsetRecog_euc();
|
||||
|
||||
const char* getName() const = 0;
|
||||
|
||||
int32_t match(InputText* det) = 0;
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* Get the next character value for EUC based encodings.
|
||||
* Character "value" is simply the raw bytes that make up the character
|
||||
* packed into an int.
|
||||
*/
|
||||
UBool nextChar(iteratedChar *it, InputText *det);
|
||||
};
|
||||
|
||||
/**
|
||||
* The charset recognize for EUC-JP. A singleton instance of this class
|
||||
* is created and kept by the public CharsetDetector class
|
||||
*/
|
||||
class CharsetRecog_euc_jp : public CharsetRecog_euc
|
||||
{
|
||||
private:
|
||||
static const int32_t commonChars[];
|
||||
static const int32_t commonCharsLen;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_euc_jp();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText *det);
|
||||
};
|
||||
|
||||
/**
|
||||
* The charset recognize for EUC-KR. A singleton instance of this class
|
||||
* is created and kept by the public CharsetDetector class
|
||||
*/
|
||||
class CharsetRecog_euc_kr : public CharsetRecog_euc
|
||||
{
|
||||
private:
|
||||
static const int32_t commonChars[];
|
||||
static const int32_t commonCharsLen;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_euc_kr();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText *det);
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* GB-18030 recognizer. Uses simplified Chinese statistics.
|
||||
*
|
||||
*/
|
||||
class CharsetRecog_gb_18030 : public CharsetRecog_mbcs
|
||||
{
|
||||
private:
|
||||
|
||||
static const int32_t commonChars[];
|
||||
static const int32_t commonCharsLen;
|
||||
/*
|
||||
* (non-Javadoc)
|
||||
* Get the next character value for EUC based encodings.
|
||||
* Character "value" is simply the raw bytes that make up the character
|
||||
* packed into an int.
|
||||
*/
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_gb_18030();
|
||||
|
||||
UBool nextChar(iteratedChar* it, InputText* det);
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText *det);
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSRMBCS_H */
|
||||
1083
icu4c/source/i18n/csrsbcs.cpp
Normal file
1083
icu4c/source/i18n/csrsbcs.cpp
Normal file
File diff suppressed because it is too large
Load diff
455
icu4c/source/i18n/csrsbcs.h
Normal file
455
icu4c/source/i18n/csrsbcs.h
Normal file
|
|
@ -0,0 +1,455 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSRSBCS_H
|
||||
#define __CSRSBCS_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uobject.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class NGramParser : public UMemory
|
||||
{
|
||||
private:
|
||||
static const int32_t N_GRAM_SIZE = 3;
|
||||
static const int32_t N_GRAM_MASK = 0xFFFFFF;
|
||||
|
||||
int32_t byteIndex;
|
||||
int32_t ngram;
|
||||
|
||||
const int32_t *ngramList;
|
||||
const uint8_t *charMap;
|
||||
|
||||
int32_t ngramCount;
|
||||
int32_t hitCount;
|
||||
|
||||
public:
|
||||
NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap);
|
||||
|
||||
private:
|
||||
/*
|
||||
* Binary search for value in table, which must have exactly 64 entries.
|
||||
*/
|
||||
int32_t search(const int32_t *table, int32_t value);
|
||||
|
||||
void lookup(int32_t thisNgram);
|
||||
void addByte(int32_t b);
|
||||
int32_t nextByte(InputText *det);
|
||||
|
||||
public:
|
||||
int32_t parse(InputText *det);
|
||||
|
||||
};
|
||||
|
||||
class CharsetRecog_sbcs : public CharsetRecognizer
|
||||
{
|
||||
protected:
|
||||
UBool haveC1Bytes;
|
||||
|
||||
public:
|
||||
CharsetRecog_sbcs();
|
||||
|
||||
virtual ~CharsetRecog_sbcs();
|
||||
|
||||
virtual const char *getName() const = 0;
|
||||
|
||||
virtual int32_t match(InputText *det) = 0;
|
||||
|
||||
int32_t match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t charMap[]);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_2 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_2();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_5 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_5();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_6 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_6();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_7 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_7();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_8 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_8();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_9 : public CharsetRecog_sbcs
|
||||
{
|
||||
protected:
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_9();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_en : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_en();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_da : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_da();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_de : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_de();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_es : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_es();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_fr : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_fr();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_it : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_it();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_nl : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_nl();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_no : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_no();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_pt : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_pt();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_1_sv : public CharsetRecog_8859_1
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_1_sv();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_2_cs : public CharsetRecog_8859_2
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_2_cs();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_2_hu : public CharsetRecog_8859_2
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_2_hu();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_2_pl : public CharsetRecog_8859_2
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_2_pl();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_2_ro : public CharsetRecog_8859_2
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_2_ro();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_5_ru : public CharsetRecog_8859_5
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_5_ru();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_6_ar : public CharsetRecog_8859_6
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_6_ar();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_7_el : public CharsetRecog_8859_7
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_7_el();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_8_I_he : public CharsetRecog_8859_8
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_8_I_he();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_8_he : public CharsetRecog_8859_8
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_8_he ();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_8859_9_tr : public CharsetRecog_8859_9
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_8859_9_tr ();
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_windows_1256 : public CharsetRecog_sbcs
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_windows_1256();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_windows_1251 : public CharsetRecog_sbcs
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_windows_1251();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
|
||||
class CharsetRecog_KOI8_R : public CharsetRecog_sbcs
|
||||
{
|
||||
private:
|
||||
static const int32_t ngrams[];
|
||||
static const uint8_t charMap[];
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_KOI8_R();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
const char *getLanguage() const;
|
||||
|
||||
int32_t match(InputText *textIn);
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSRSBCS_H */
|
||||
142
icu4c/source/i18n/csrucode.cpp
Normal file
142
icu4c/source/i18n/csrucode.cpp
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrucode.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
CharsetRecog_Unicode::~CharsetRecog_Unicode()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_UTF_16_BE::getName() const
|
||||
{
|
||||
return "UTF-16BE";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF_16_BE::match(InputText* textIn)
|
||||
{
|
||||
const uint8_t *input = textIn->fRawInput;
|
||||
|
||||
if ((input[0] & 0xFF) == 0xFE && (input[1] & 0xFF) == 0xFF) {
|
||||
return 100;
|
||||
}
|
||||
|
||||
// TODO: Do some statastics to check for unsigned UTF-16BE
|
||||
return 0;
|
||||
}
|
||||
|
||||
CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_UTF_16_LE::getName() const
|
||||
{
|
||||
return "UTF-16LE";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF_16_LE::match(InputText* textIn)
|
||||
{
|
||||
const uint8_t *input = textIn->fRawInput;
|
||||
|
||||
if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) {
|
||||
return 100;
|
||||
}
|
||||
|
||||
// TODO: Do some statastics to check for unsigned UTF-16LE
|
||||
return 0;
|
||||
}
|
||||
|
||||
CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF_32::match(InputText* textIn)
|
||||
{
|
||||
const uint8_t *input = textIn->fRawInput;
|
||||
int32_t limit = (textIn->fRawLength / 4) * 4;
|
||||
int32_t numValid = 0;
|
||||
int32_t numInvalid = 0;
|
||||
bool hasBOM = FALSE;
|
||||
int32_t confidence = 0;
|
||||
|
||||
if (input[0] == 0x00 && input[1] == 0x00 && input[2] == 0xFE && input[3] == 0xFF) {
|
||||
hasBOM = TRUE;
|
||||
}
|
||||
|
||||
for(int32_t i = 0; i < limit; i += 4) {
|
||||
int32_t ch = getChar(input, i);
|
||||
|
||||
if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
|
||||
numInvalid += 1;
|
||||
} else {
|
||||
numValid += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Cook up some sort of confidence score, based on presense of a BOM
|
||||
// and the existence of valid and/or invalid multi-byte sequences.
|
||||
if (hasBOM && numInvalid==0) {
|
||||
confidence = 100;
|
||||
} else if (hasBOM && numValid > numInvalid*10) {
|
||||
confidence = 80;
|
||||
} else if (numValid > 3 && numInvalid == 0) {
|
||||
confidence = 100;
|
||||
} else if (numValid > 0 && numInvalid == 0) {
|
||||
confidence = 80;
|
||||
} else if (numValid > numInvalid*10) {
|
||||
// Probably corruput UTF-32BE data. Valid sequences aren't likely by chance.
|
||||
confidence = 25;
|
||||
}
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_UTF_32_BE::getName() const
|
||||
{
|
||||
return "UTF-32BE";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
|
||||
{
|
||||
return input[index + 0] << 24 | input[index + 1] << 16 |
|
||||
input[index + 2] << 8 | input[index + 3];
|
||||
}
|
||||
|
||||
CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_UTF_32_LE::getName() const
|
||||
{
|
||||
return "UTF-32LE";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
|
||||
{
|
||||
return input[index + 3] << 24 | input[index + 2] << 16 |
|
||||
input[index + 1] << 8 | input[index + 0];
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
103
icu4c/source/i18n/csrucode.h
Normal file
103
icu4c/source/i18n/csrucode.h
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSRUCODE_H
|
||||
#define __CSRUCODE_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
/**
|
||||
* This class matches UTF-16 and UTF-32, both big- and little-endian. The
|
||||
* BOM will be used if it is present.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
class CharsetRecog_Unicode : public CharsetRecognizer
|
||||
{
|
||||
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_Unicode();
|
||||
/* (non-Javadoc)
|
||||
* @see com.ibm.icu.text.CharsetRecognizer#getName()
|
||||
*/
|
||||
const char* getName() const = 0;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
|
||||
*/
|
||||
int32_t match(InputText* textIn) = 0;
|
||||
};
|
||||
|
||||
|
||||
class CharsetRecog_UTF_16_BE : public CharsetRecog_Unicode
|
||||
{
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_UTF_16_BE();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText* textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_UTF_16_LE : public CharsetRecog_Unicode
|
||||
{
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_UTF_16_LE();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
int32_t match(InputText* textIn);
|
||||
};
|
||||
|
||||
class CharsetRecog_UTF_32 : public CharsetRecog_Unicode
|
||||
{
|
||||
protected:
|
||||
virtual int32_t getChar(const uint8_t *input, int32_t index) const = 0;
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_UTF_32();
|
||||
|
||||
const char* getName() const = 0;
|
||||
|
||||
int32_t match(InputText* textIn);
|
||||
};
|
||||
|
||||
|
||||
class CharsetRecog_UTF_32_BE : public CharsetRecog_UTF_32
|
||||
{
|
||||
protected:
|
||||
int32_t getChar(const uint8_t *input, int32_t index) const;
|
||||
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_UTF_32_BE();
|
||||
|
||||
const char *getName() const;
|
||||
};
|
||||
|
||||
|
||||
class CharsetRecog_UTF_32_LE : public CharsetRecog_UTF_32
|
||||
{
|
||||
protected:
|
||||
int32_t getChar(const uint8_t *input, int32_t index) const;
|
||||
|
||||
public:
|
||||
virtual ~CharsetRecog_UTF_32_LE();
|
||||
|
||||
const char* getName() const;
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSRUCODE_H */
|
||||
108
icu4c/source/i18n/csrutf8.cpp
Normal file
108
icu4c/source/i18n/csrutf8.cpp
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrutf8.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
CharsetRecog_UTF8::~CharsetRecog_UTF8()
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
const char *CharsetRecog_UTF8::getName() const
|
||||
{
|
||||
return "UTF-8";
|
||||
}
|
||||
|
||||
int32_t CharsetRecog_UTF8::match(InputText* det) {
|
||||
bool hasBOM = FALSE;
|
||||
int32_t numValid = 0;
|
||||
int32_t numInvalid = 0;
|
||||
const uint8_t *input = det->fRawInput;
|
||||
int32_t i;
|
||||
int32_t trailBytes = 0;
|
||||
int32_t confidence;
|
||||
|
||||
if (det->fRawLength >= 3 &&
|
||||
input[0] == 0xEF && input[1] == 0xBB && input[2] == 0xBF) {
|
||||
hasBOM = TRUE;
|
||||
}
|
||||
|
||||
// Scan for multi-byte sequences
|
||||
for (i=0; i < det->fRawLength; i += 1) {
|
||||
int32_t b = input[i];
|
||||
|
||||
if ((b & 0x80) == 0) {
|
||||
continue; // ASCII
|
||||
}
|
||||
|
||||
// Hi bit on char found. Figure out how long the sequence should be
|
||||
if ((b & 0x0E0) == 0x0C0) {
|
||||
trailBytes = 1;
|
||||
} else if ((b & 0x0F0) == 0x0E0) {
|
||||
trailBytes = 2;
|
||||
} else if ((b & 0x0F8) == 0xF0) {
|
||||
trailBytes = 3;
|
||||
} else {
|
||||
numInvalid += 1;
|
||||
|
||||
if (numInvalid > 5) {
|
||||
break;
|
||||
}
|
||||
|
||||
trailBytes = 0;
|
||||
}
|
||||
|
||||
// Verify that we've got the right number of trail bytes in the sequence
|
||||
for (;;) {
|
||||
i += 1;
|
||||
|
||||
if (i >= det->fRawLength) {
|
||||
break;
|
||||
}
|
||||
|
||||
b = input[i];
|
||||
|
||||
if ((b & 0xC0) != 0x080) {
|
||||
numInvalid += 1;
|
||||
break;
|
||||
}
|
||||
|
||||
if (--trailBytes == 0) {
|
||||
numValid += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Cook up some sort of confidence score, based on presense of a BOM
|
||||
// and the existence of valid and/or invalid multi-byte sequences.
|
||||
confidence = 0;
|
||||
if (hasBOM && numInvalid == 0) {
|
||||
confidence = 100;
|
||||
} else if (hasBOM && numValid > numInvalid*10) {
|
||||
confidence = 80;
|
||||
} else if (numValid > 3 && numInvalid == 0) {
|
||||
confidence = 100;
|
||||
} else if (numValid > 0 && numInvalid == 0) {
|
||||
confidence = 80;
|
||||
} else if (numValid == 0 && numInvalid == 0) {
|
||||
// Plain ASCII.
|
||||
confidence = 10;
|
||||
} else if (numValid > numInvalid*10) {
|
||||
// Probably corruput utf-8 data. Valid sequences aren't likely by chance.
|
||||
confidence = 25;
|
||||
}
|
||||
|
||||
return confidence;
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
39
icu4c/source/i18n/csrutf8.h
Normal file
39
icu4c/source/i18n/csrutf8.h
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSRUTF8_H
|
||||
#define __CSRUTF8_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "csrecog.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
/**
|
||||
* Charset recognizer for UTF-8
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
class CharsetRecog_UTF8: public CharsetRecognizer {
|
||||
|
||||
public:
|
||||
|
||||
virtual ~CharsetRecog_UTF8();
|
||||
|
||||
const char *getName() const;
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
|
||||
*/
|
||||
int32_t match(InputText *det);
|
||||
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __CSRUTF8_H */
|
||||
|
|
@ -1429,6 +1429,60 @@
|
|||
<Filter
|
||||
Name="charset detect"
|
||||
Filter="">
|
||||
<File
|
||||
RelativePath=".\csdetect.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csdetect.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csmatch.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csmatch.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csr2022.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csr2022.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrecog.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrecog.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrmbcs.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrmbcs.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrsbcs.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrsbcs.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrucode.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrucode.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrutf8.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csrutf8.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\inputext.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\inputext.h">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\ucsdet.cpp">
|
||||
</File>
|
||||
|
|
|
|||
150
icu4c/source/i18n/inputext.cpp
Normal file
150
icu4c/source/i18n/inputext.cpp
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "inputext.h"
|
||||
|
||||
#include "cmemory.h"
|
||||
#include "cstring.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
|
||||
|
||||
#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
|
||||
#define DELETE_ARRAY(array) uprv_free((void *) (array))
|
||||
|
||||
InputText::InputText()
|
||||
: fInputBytes(NEW_ARRAY(uint8_t, kBufSize)), // The text to be checked. Markup will have been
|
||||
// removed if appropriate.
|
||||
fByteStats(NEW_ARRAY(int16_t, 256)), // byte frequency statistics for the input text.
|
||||
// Value is percent, not absolute.
|
||||
fDeclaredEncoding(0),
|
||||
fRawInput(0),
|
||||
fRawLength(0)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
InputText::~InputText()
|
||||
{
|
||||
DELETE_ARRAY(fDeclaredEncoding);
|
||||
DELETE_ARRAY(fByteStats);
|
||||
DELETE_ARRAY(fInputBytes);
|
||||
}
|
||||
|
||||
void InputText::setText(const char *in, int32_t len)
|
||||
{
|
||||
fInputLen = 0;
|
||||
fC1Bytes = FALSE;
|
||||
fRawInput = (const uint8_t *) in;
|
||||
fRawLength = len;
|
||||
}
|
||||
|
||||
void InputText::setDeclaredEncoding(const char* encoding, int32_t len)
|
||||
{
|
||||
if(encoding) {
|
||||
len += 1; // to make place for the \0 at the end.
|
||||
delete[] fDeclaredEncoding;
|
||||
fDeclaredEncoding = NEW_ARRAY(char, len);
|
||||
uprv_strncpy(fDeclaredEncoding, encoding, len);
|
||||
}
|
||||
}
|
||||
|
||||
UBool InputText::isSet() const
|
||||
{
|
||||
return fRawInput? TRUE : FALSE;
|
||||
}
|
||||
|
||||
/**
|
||||
* MungeInput - after getting a set of raw input data to be analyzed, preprocess
|
||||
* it by removing what appears to be html markup.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
void InputText::MungeInput(UBool fStripTags) {
|
||||
int srci = 0;
|
||||
int dsti = 0;
|
||||
uint8_t b;
|
||||
bool inMarkup = FALSE;
|
||||
int32_t openTags = 0;
|
||||
int32_t badTags = 0;
|
||||
|
||||
//
|
||||
// html / xml markup stripping.
|
||||
// quick and dirty, not 100% accurate, but hopefully good enough, statistically.
|
||||
// discard everything within < brackets >
|
||||
// Count how many total '<' and illegal (nested) '<' occur, so we can make some
|
||||
// guess as to whether the input was actually marked up at all.
|
||||
if (fStripTags) {
|
||||
for (srci = 0; srci < fRawLength; srci += 1) {
|
||||
b = fRawInput[srci];
|
||||
|
||||
if (b == (uint8_t)'<') {
|
||||
if (inMarkup) {
|
||||
badTags += 1;
|
||||
}
|
||||
|
||||
inMarkup = TRUE;
|
||||
openTags += 1;
|
||||
}
|
||||
|
||||
if (! inMarkup) {
|
||||
fInputBytes[dsti++] = b;
|
||||
}
|
||||
|
||||
if (b == (uint8_t)'>') {
|
||||
inMarkup = FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
fInputLen = dsti;
|
||||
}
|
||||
|
||||
//
|
||||
// If it looks like this input wasn't marked up, or if it looks like it's
|
||||
// essentially nothing but markup abandon the markup stripping.
|
||||
// Detection will have to work on the unstripped input.
|
||||
//
|
||||
if (openTags<5 || openTags/5 < badTags ||
|
||||
(fInputLen < 100 && fRawLength>600)) {
|
||||
int32_t limit = fRawLength;
|
||||
|
||||
if (limit > kBufSize) {
|
||||
limit = kBufSize;
|
||||
}
|
||||
|
||||
for (srci=0; srci<limit; srci++) {
|
||||
fInputBytes[srci] = fRawInput[srci];
|
||||
}
|
||||
|
||||
fInputLen = srci;
|
||||
}
|
||||
|
||||
//
|
||||
// Tally up the byte occurence statistics.
|
||||
// These are available for use by the various detectors.
|
||||
//
|
||||
|
||||
uprv_memset(fByteStats, 0, (sizeof fByteStats[0]) * 256);
|
||||
|
||||
for (srci = 0; srci < fInputLen; srci += 1) {
|
||||
fByteStats[fInputBytes[srci]] += 1;
|
||||
}
|
||||
|
||||
for (int32_t i = 0x80; i <= 0x9F; i += 1) {
|
||||
if (fByteStats[i] != 0) {
|
||||
fC1Bytes = TRUE;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
50
icu4c/source/i18n/inputext.h
Normal file
50
icu4c/source/i18n/inputext.h
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __INPUTEXT_H
|
||||
#define __INPUTEXT_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uobject.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
class InputText : public UMemory
|
||||
{
|
||||
public:
|
||||
InputText();
|
||||
~InputText();
|
||||
|
||||
void setText(const char *in, int32_t len);
|
||||
void setDeclaredEncoding(const char *encoding, int32_t len);
|
||||
UBool isSet() const;
|
||||
void MungeInput(UBool fStripTags);
|
||||
|
||||
|
||||
static const int32_t kBufSize = 8192;
|
||||
// The text to be checked. Markup will have been
|
||||
// removed if appropriate.
|
||||
uint8_t *fInputBytes;
|
||||
int32_t fInputLen; // Length of the byte data in fInputText.
|
||||
// byte frequency statistics for the input text.
|
||||
// Value is percent, not absolute.
|
||||
// Value is rounded up, so zero really means zero occurences.
|
||||
int16_t *fByteStats;
|
||||
UBool fC1Bytes; // True if any bytes in the range 0x80 - 0x9F are in the input;false by default
|
||||
char *fDeclaredEncoding;
|
||||
|
||||
const uint8_t *fRawInput; // Original, untouched input bytes.
|
||||
// If user gave us a byte array, this is it.
|
||||
// If user gave us a stream, it's read to a
|
||||
// buffer here.
|
||||
int32_t fRawLength; // Length of data in fRawInput array.
|
||||
|
||||
};
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
||||
#endif /* __INPUTEXT_H */
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
/*
|
||||
******************************************************************************
|
||||
* *
|
||||
* Copyright (C) 2001-2004, International Business Machines *
|
||||
* Copyright (C) 2001-2006, International Business Machines *
|
||||
* Corporation and others. All Rights Reserved. *
|
||||
* *
|
||||
******************************************************************************
|
||||
|
|
@ -38,6 +38,7 @@ typedef enum ECleanupI18NType {
|
|||
UCLN_I18N_COLLATOR,
|
||||
UCLN_I18N_UCOL,
|
||||
UCLN_I18N_UCOL_BLD,
|
||||
UCLN_I18N_CSDET,
|
||||
UCLN_I18N_COUNT /* This must be last */
|
||||
} ECleanupI18NType;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,262 @@
|
|||
/*
|
||||
*******************************************************************************
|
||||
* Copyright (C) 2005, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
*******************************************************************************
|
||||
* file name: ucsdet.cpp
|
||||
*/
|
||||
********************************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
********************************************************************************
|
||||
*/
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
|
||||
#include "unicode/ucsdet.h"
|
||||
#include "csdetect.h"
|
||||
#include "csmatch.h"
|
||||
|
||||
// Place Holder file for Char Set Detection Implementation.
|
||||
#include "unicode/putil.h"
|
||||
|
||||
#include "cmemory.h"
|
||||
#include "uenumimp.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
||||
#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
|
||||
|
||||
#define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type))
|
||||
#define DELETE_ARRAY(array) uprv_free((void *) (array))
|
||||
|
||||
|
||||
U_DRAFT UCharsetDetector * U_EXPORT2
|
||||
ucsdet_open(UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
UCharsetDetector* ucsd = new UCharsetDetector();
|
||||
return ucsd;
|
||||
}
|
||||
|
||||
U_DRAFT void U_EXPORT2
|
||||
ucsdet_close(UCharsetDetector *csd)
|
||||
{
|
||||
delete csd;
|
||||
}
|
||||
|
||||
U_DRAFT void U_EXPORT2
|
||||
ucsdet_setText(UCharsetDetector *csd, const char *textIn, int32_t len, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return;
|
||||
}
|
||||
|
||||
csd->setText(textIn, len);
|
||||
}
|
||||
|
||||
U_DRAFT const char * U_EXPORT2
|
||||
ucsdet_getName(const UCharsetMatch *csm, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return csm->getName();
|
||||
}
|
||||
|
||||
U_DRAFT int32_t U_EXPORT2
|
||||
ucsdet_getConfidence(const UCharsetMatch *csm, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return csm->getConfidence();
|
||||
}
|
||||
|
||||
U_DRAFT const char * U_EXPORT2
|
||||
ucsdet_getLanguage(const UCharsetMatch *csm, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return csm->getLanguage();
|
||||
}
|
||||
|
||||
U_DRAFT const UCharsetMatch * U_EXPORT2
|
||||
ucsdet_detect(UCharsetDetector *csd, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return csd->detect(*status);
|
||||
}
|
||||
|
||||
U_DRAFT void U_EXPORT2
|
||||
ucsdet_setDeclaredEncoding(UCharsetDetector *csd, const char *encoding, int32_t length, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return;
|
||||
}
|
||||
|
||||
csd->setDeclaredEncoding(encoding,length);
|
||||
}
|
||||
|
||||
U_DRAFT UCharsetMatch**
|
||||
ucsdet_detectAll(UCharsetDetector *csd,
|
||||
int32_t *maxMatchesFound, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return (UCharsetMatch**)csd->detectAll(*maxMatchesFound,*status);
|
||||
}
|
||||
|
||||
// U_DRAFT const char * U_EXPORT2
|
||||
// ucsdet_getDetectableCharsetName(const UCharsetDetector *csd, int32_t index, UErrorCode *status)
|
||||
// {
|
||||
// if(U_FAILURE(*status)) {
|
||||
// return 0;
|
||||
// }
|
||||
// return csd->getCharsetName(index,*status);
|
||||
// }
|
||||
|
||||
// U_DRAFT int32_t U_EXPORT2
|
||||
// ucsdet_getDetectableCharsetsCount(const UCharsetDetector *csd, UErrorCode *status)
|
||||
// {
|
||||
// if(U_FAILURE(*status)) {
|
||||
// return -1;
|
||||
// }
|
||||
// return UCharsetDetector::getDetectableCount();
|
||||
// }
|
||||
|
||||
typedef struct {
|
||||
int32_t currIndex;
|
||||
int32_t maxIndex;
|
||||
char *currChar;
|
||||
UChar *currUChar;
|
||||
char **array;
|
||||
} Context;
|
||||
|
||||
#define cont ((Context *)en->context)
|
||||
|
||||
static void U_CALLCONV
|
||||
enumClose(UEnumeration *en) {
|
||||
if(cont->currUChar != NULL) {
|
||||
DELETE_ARRAY(cont->currUChar);
|
||||
cont->currUChar = NULL;
|
||||
}
|
||||
|
||||
delete en;
|
||||
}
|
||||
|
||||
static int32_t U_CALLCONV
|
||||
enumCount(UEnumeration *en, UErrorCode *status) {
|
||||
return cont->maxIndex;
|
||||
}
|
||||
|
||||
static const UChar* U_CALLCONV
|
||||
enumUNext(UEnumeration *en, int32_t *resultLength, UErrorCode *status) {
|
||||
if(cont->currIndex >= cont->maxIndex) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if(cont->currUChar == NULL) {
|
||||
cont->currUChar = NEW_ARRAY(UChar, 1024);
|
||||
}
|
||||
|
||||
cont->currChar = (cont->array)[cont->currIndex];
|
||||
*resultLength = (int32_t)strlen(cont->currChar);
|
||||
u_charsToUChars(cont->currChar, cont->currUChar, *resultLength);
|
||||
cont->currIndex++;
|
||||
|
||||
return cont->currUChar;
|
||||
}
|
||||
|
||||
static const char* U_CALLCONV
|
||||
enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode *status) {
|
||||
if(cont->currIndex >= cont->maxIndex) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
cont->currChar = (cont->array)[cont->currIndex];
|
||||
*resultLength = (int32_t)strlen(cont->currChar);
|
||||
cont->currIndex++;
|
||||
|
||||
return cont->currChar;
|
||||
}
|
||||
|
||||
static void U_CALLCONV
|
||||
enumReset(UEnumeration *en, UErrorCode *status) {
|
||||
cont->currIndex = 0;
|
||||
}
|
||||
|
||||
Context enumContext = {
|
||||
0, 0,
|
||||
NULL, NULL,
|
||||
NULL
|
||||
};
|
||||
|
||||
UEnumeration enumeration = {
|
||||
NULL,
|
||||
&enumContext,
|
||||
enumClose,
|
||||
enumCount,
|
||||
enumUNext,
|
||||
enumNext,
|
||||
enumReset
|
||||
};
|
||||
|
||||
|
||||
static UEnumeration *getEnum(const char** source, int32_t size) {
|
||||
UEnumeration *en = new UEnumeration;
|
||||
memcpy(en, &enumeration, sizeof(UEnumeration));
|
||||
cont->array = (char **)source;
|
||||
cont->maxIndex = size;
|
||||
return en;
|
||||
}
|
||||
|
||||
static const char** charsets = NEW_ARRAY(const char *, UCharsetDetector::getDetectableCount());
|
||||
|
||||
U_DRAFT UEnumeration * U_EXPORT2
|
||||
ucsdet_getAllDetectableCharsets(const UCharsetDetector *csd, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t size = UCharsetDetector::getDetectableCount();
|
||||
|
||||
for(int32_t index=0;index<size;index++) {
|
||||
charsets[index] = csd->getCharsetName(index, *status);
|
||||
}
|
||||
|
||||
return getEnum(charsets,size);
|
||||
}
|
||||
|
||||
U_DRAFT UBool U_EXPORT2
|
||||
ucsdet_isInputFilterEnabled(const UCharsetDetector *csd)
|
||||
{
|
||||
return csd->getStripTagsFlag();
|
||||
}
|
||||
|
||||
U_DRAFT UBool U_EXPORT2
|
||||
ucsdet_enableInputFilter(UCharsetDetector *csd, UBool filter)
|
||||
{
|
||||
UBool prev = csd->getStripTagsFlag();
|
||||
csd->setStripTagsFlag(filter);
|
||||
|
||||
return prev;
|
||||
}
|
||||
|
||||
U_DRAFT int32_t U_EXPORT2
|
||||
ucsdet_getUChars(const UCharsetMatch *csm,
|
||||
UChar *buf, int32_t cap, UErrorCode *status)
|
||||
{
|
||||
if(U_FAILURE(*status)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return csm->getUChars(buf, cap, status);
|
||||
}
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
|
|
|||
|
|
@ -1,22 +1,23 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
* file name: ucsdet.h
|
||||
* encoding: US-ASCII
|
||||
* indentation:4
|
||||
*
|
||||
* created on: 2005Aug04
|
||||
* created by: Andy Heninger
|
||||
*
|
||||
* ICU Character Set Detection, API for C
|
||||
*
|
||||
* Draft version 18 Oct 2005
|
||||
*
|
||||
*/
|
||||
#ifndef CSDET_H
|
||||
#define CSDET_H
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
* file name: ucsdet.h
|
||||
* encoding: US-ASCII
|
||||
* indentation:4
|
||||
*
|
||||
* created on: 2005Aug04
|
||||
* created by: Andy Heninger
|
||||
*
|
||||
* ICU Character Set Detection, API for C
|
||||
*
|
||||
* Draft version 18 Oct 2005
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef __UCSDET_H
|
||||
#define __UCSDET_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uenum.h"
|
||||
|
|
@ -344,6 +345,6 @@ ucsdet_enableInputFilter(UCharsetDetector *csd, UBool filter);
|
|||
|
||||
|
||||
#endif /* U_HIDE_DRAFT_API */
|
||||
#endif /* CSDET_H */
|
||||
#endif /* __UCSDET_H */
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ jamotest.o srchtest.o reptest.o regextst.o \
|
|||
itrbnf.o itrbnfrt.o itrbnfp.o ucaconf.o icusvtst.o \
|
||||
uobjtest.o idnaref.o idnaconf.o nptrans.o punyref.o testidn.o testidna.o incaltst.o \
|
||||
calcasts.o v32test.o uvectest.o textfile.o tokiter.o utxttest.o wbnf.o \
|
||||
windttst.o winnmtst.o winutil.o
|
||||
windttst.o winnmtst.o winutil.o csdetest.o
|
||||
|
||||
|
||||
DEPS = $(OBJECTS:.o=.d)
|
||||
|
|
|
|||
394
icu4c/source/test/intltest/csdetest.cpp
Normal file
394
icu4c/source/test/intltest/csdetest.cpp
Normal file
|
|
@ -0,0 +1,394 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/ucsdet.h"
|
||||
#include "unicode/ucnv.h"
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/putil.h"
|
||||
|
||||
#include "intltest.h"
|
||||
#include "csdetest.h"
|
||||
|
||||
//#include "cmemory.h"
|
||||
#include "xmlparser.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define ARRAY_SIZE(array) (sizeof array / sizeof array[0])
|
||||
|
||||
#define NEW_ARRAY(type,count) (type *) /*uprv_*/malloc((count) * sizeof(type))
|
||||
#define DELETE_ARRAY(array) /*uprv_*/free((void *) (array))
|
||||
|
||||
#define CH_SPACE 0x0020
|
||||
#define CH_SLASH 0x002F
|
||||
|
||||
//---------------------------------------------------------------------------
|
||||
//
|
||||
// Test class boilerplate
|
||||
//
|
||||
//---------------------------------------------------------------------------
|
||||
CharsetDetectionTest::CharsetDetectionTest()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
CharsetDetectionTest::~CharsetDetectionTest()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
void CharsetDetectionTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
|
||||
{
|
||||
if (exec) logln("TestSuite CharsetDetectionTest: ");
|
||||
switch (index) {
|
||||
case 0: name = "ConstructionTest";
|
||||
if (exec) ConstructionTest();
|
||||
break;
|
||||
|
||||
case 1: name = "UTF8Test";
|
||||
if (exec) UTF8Test();
|
||||
break;
|
||||
|
||||
case 2: name = "UTF16Test";
|
||||
if (exec) UTF16Test();
|
||||
break;
|
||||
|
||||
case 3: name = "C1BytesTest";
|
||||
if (exec) C1BytesTest();
|
||||
break;
|
||||
|
||||
case 4: name = "InputFilterTest";
|
||||
if (exec) InputFilterTest();
|
||||
break;
|
||||
|
||||
case 5: name = "DetectionTest";
|
||||
if (exec) DetectionTest();
|
||||
break;
|
||||
|
||||
default: name = "";
|
||||
break; //needed to end loop
|
||||
}
|
||||
}
|
||||
|
||||
static UnicodeString *split(const UnicodeString &src, UChar ch, int32_t &splits)
|
||||
{
|
||||
int32_t offset = -1;
|
||||
|
||||
splits = 1;
|
||||
while((offset = src.indexOf(ch, offset + 1)) >= 0) {
|
||||
splits += 1;
|
||||
}
|
||||
|
||||
UnicodeString *result = new UnicodeString[splits];
|
||||
|
||||
int32_t start = 0;
|
||||
int32_t split = 0;
|
||||
int32_t end;
|
||||
|
||||
while((end = src.indexOf(ch, start)) >= 0) {
|
||||
src.extractBetween(start, end, result[split++]);
|
||||
start = end + 1;
|
||||
}
|
||||
|
||||
src.extractBetween(start, src.length(), result[split]);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static char *extractBytes(const UnicodeString &source, const char *codepage, int32_t &length)
|
||||
{
|
||||
int32_t sLength = source.length();
|
||||
char *bytes;
|
||||
|
||||
length = source.extract(0, sLength, NULL, codepage);
|
||||
bytes = NEW_ARRAY(char, length + 1);
|
||||
source.extract(0, sLength, bytes, codepage);
|
||||
|
||||
return bytes;
|
||||
}
|
||||
|
||||
static void freeBytes(char *bytes)
|
||||
{
|
||||
DELETE_ARRAY(bytes);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::checkEncoding(const UnicodeString &testString, const UnicodeString &encoding, const UnicodeString &id)
|
||||
{
|
||||
int32_t splits = 0;
|
||||
int32_t testLength = testString.length();
|
||||
UnicodeString *eSplit = split(encoding, CH_SLASH, splits);
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
int32_t cpLength = eSplit[0].length();
|
||||
char codepage[64];
|
||||
|
||||
u_UCharsToChars(eSplit[0].getBuffer(), codepage, cpLength);
|
||||
codepage[cpLength] = '\0';
|
||||
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
|
||||
int32_t byteLength = 0;
|
||||
char *bytes = extractBytes(testString, codepage, byteLength);
|
||||
|
||||
ucsdet_setText(csd, bytes, byteLength, &status);
|
||||
|
||||
const UCharsetMatch *csm = ucsdet_detect(csd, &status);
|
||||
UnicodeString name(ucsdet_getName(csm, &status));
|
||||
UnicodeString lang(ucsdet_getLanguage(csm, &status));
|
||||
UChar *decoded = NULL;
|
||||
int32_t dLength = 0;
|
||||
|
||||
if (name.compare(eSplit[0]) != 0) {
|
||||
errln("Encoding detection failure for " + id + ": expected " + eSplit[0] + ", got " + name);
|
||||
goto bail;
|
||||
}
|
||||
|
||||
if (splits > 1 && lang.compare(eSplit[1]) != 0) {
|
||||
errln("Language detection failure for " + id + ", " + eSplit[0] + ": expected " + eSplit[1] + ", got " + lang);
|
||||
goto bail;
|
||||
}
|
||||
|
||||
decoded = NEW_ARRAY(UChar, testLength);
|
||||
dLength = ucsdet_getUChars(csm, decoded, testLength, &status);
|
||||
|
||||
if (testString.compare(decoded, dLength) != 0) {
|
||||
errln("Round-trip error for " + id + ", " + eSplit[0] + ": getUChars() didn't yeild the original string.");
|
||||
|
||||
//for(int32_t i = 0; i < testLength; i += 1) {
|
||||
// if(testString[i] != decoded[i]) {
|
||||
// printf("Strings differ at byte %d\n", i);
|
||||
// break;
|
||||
// }
|
||||
//}
|
||||
}
|
||||
|
||||
DELETE_ARRAY(decoded);
|
||||
|
||||
bail:
|
||||
freeBytes(bytes);
|
||||
ucsdet_close(csd);
|
||||
delete[] eSplit;
|
||||
}
|
||||
|
||||
const char *CharsetDetectionTest::getPath(char buffer[2048], const char *filename) {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
const char *testDataDirectory = IntlTest::getSourceTestData(status);
|
||||
|
||||
if (U_FAILURE(status)) {
|
||||
errln("ERROR: getPath() failed - %s", u_errorName(status));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
strcpy(buffer, testDataDirectory);
|
||||
strcat(buffer, filename);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::ConstructionTest()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
UEnumeration *e = ucsdet_getAllDetectableCharsets(csd, &status);
|
||||
int32_t count = uenum_count(e, &status);
|
||||
|
||||
for(int32_t i = 0; i < count; i += 1) {
|
||||
int32_t length;
|
||||
const char *name = uenum_next(e, &length, &status);
|
||||
|
||||
if(name == NULL || length <= 0) {
|
||||
errln("ucsdet_getAllDetectableCharsets() returned an null or empty name!");
|
||||
}
|
||||
}
|
||||
|
||||
uenum_close(e);
|
||||
ucsdet_close(csd);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::UTF8Test()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UnicodeString ss = "This is a string with some non-ascii characters that will "
|
||||
"be converted to UTF-8, then shoved through the detection process. "
|
||||
"\\u0391\\u0392\\u0393\\u0394\\u0395"
|
||||
"Sure would be nice if our source could contain Unicode directly!";
|
||||
UnicodeString s = ss.unescape();
|
||||
int32_t byteLength = 0, sLength = s.length();
|
||||
char *bytes = extractBytes(s, "UTF-8", byteLength);
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
const UCharsetMatch *match;
|
||||
UChar *detected = NEW_ARRAY(UChar, sLength);
|
||||
|
||||
ucsdet_setText(csd, bytes, byteLength, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
|
||||
ucsdet_getUChars(match, detected, sLength, &status);
|
||||
|
||||
if (s.compare(detected, sLength) != 0) {
|
||||
errln("Round-trip test failed!");
|
||||
}
|
||||
|
||||
DELETE_ARRAY(detected);
|
||||
freeBytes(bytes);
|
||||
ucsdet_close(csd);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::UTF16Test()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
/* Notice the BOM on the start of this string */
|
||||
UChar chars[] = {
|
||||
0xFEFF, 0x0623, 0x0648, 0x0631, 0x0648, 0x0628, 0x0627, 0x002C,
|
||||
0x0020, 0x0628, 0x0631, 0x0645, 0x062c, 0x064a, 0x0627, 0x062a,
|
||||
0x0020, 0x0627, 0x0644, 0x062d, 0x0627, 0x0633, 0x0648, 0x0628,
|
||||
0x0020, 0x002b, 0x0020, 0x0627, 0x0646, 0x062a, 0x0631, 0x0646,
|
||||
0x064a, 0x062a, 0x0000};
|
||||
UnicodeString s(chars);
|
||||
int32_t beLength = 0, leLength = 0;
|
||||
char *beBytes = extractBytes(s, "UTF-16BE", beLength);
|
||||
char *leBytes = extractBytes(s, "UTF-16LE", leLength);
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
const UCharsetMatch *match;
|
||||
const char *name;
|
||||
|
||||
ucsdet_setText(csd, beBytes, beLength, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
name = ucsdet_getName(match, &status);
|
||||
|
||||
if (strcmp(name, "UTF-16BE") != 0) {
|
||||
errln("Encoding detection failure for UTF-16BE: got %s", name);
|
||||
}
|
||||
|
||||
ucsdet_setText(csd, leBytes, leLength, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
name = ucsdet_getName(match, &status);
|
||||
|
||||
if (strcmp(name, "UTF-16LE") != 0) {
|
||||
errln("Enconding detection failure for UTF-16LE: got %s", name);
|
||||
}
|
||||
|
||||
freeBytes(leBytes);
|
||||
freeBytes(beBytes);
|
||||
ucsdet_close(csd);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::InputFilterTest()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UnicodeString ss = "<a> <lot> <of> <English> <inside> <the> <markup> Un tr\\u00E8s petit peu de Fran\\u00E7ais. <to> <confuse> <the> <detector>";
|
||||
UnicodeString s = ss.unescape();
|
||||
int32_t byteLength = 0;
|
||||
char *bytes = extractBytes(s, "ISO-8859-1", byteLength);
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
const UCharsetMatch *match;
|
||||
const char *lang;
|
||||
|
||||
ucsdet_enableInputFilter(csd, TRUE);
|
||||
|
||||
if (!ucsdet_isInputFilterEnabled(csd)) {
|
||||
errln("ucsdet_enableInputFilter(csd, TRUE) did not enable input filter!");
|
||||
}
|
||||
|
||||
ucsdet_setText(csd, bytes, byteLength, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
lang = ucsdet_getLanguage(match, &status);
|
||||
|
||||
if (strcmp(lang, "fr") != 0) {
|
||||
errln("Input filter did not strip markup!");
|
||||
}
|
||||
|
||||
ucsdet_enableInputFilter(csd, FALSE);
|
||||
ucsdet_setText(csd, bytes, byteLength, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
lang = ucsdet_getLanguage(match, &status);
|
||||
|
||||
if (strcmp(lang, "en") != 0) {
|
||||
errln("Unfiltered input did not detect as English!");
|
||||
}
|
||||
|
||||
freeBytes(bytes);
|
||||
ucsdet_close(csd);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::C1BytesTest()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UnicodeString sISO = "This is a small sample of some English text. Just enough to be sure that it detects correctly.";
|
||||
UnicodeString ssWindows = "This is another small sample of some English text. Just enough to be sure that it detects correctly. It also includes some \\u201CC1\\u201D bytes.";
|
||||
UnicodeString sWindows = ssWindows.unescape();
|
||||
int32_t lISO = 0, lWindows = 0;
|
||||
char *bISO = extractBytes(sISO, "ISO-8859-1", lISO);
|
||||
char *bWindows = extractBytes(sWindows, "windows-1252", lWindows);
|
||||
UCharsetDetector *csd = ucsdet_open(&status);
|
||||
const UCharsetMatch *match;
|
||||
const char *name;
|
||||
|
||||
ucsdet_setText(csd, bWindows, lWindows, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
name = ucsdet_getName(match, &status);
|
||||
|
||||
if (strcmp(name, "windows-1252") != 0) {
|
||||
errln("English text with C1 bytes does not detect as windows-1252!");
|
||||
}
|
||||
|
||||
ucsdet_setText(csd, bISO, lISO, &status);
|
||||
match = ucsdet_detect(csd, &status);
|
||||
name = ucsdet_getName(match, &status);
|
||||
|
||||
if (strcmp(name, "ISO-8859-1") != 0) {
|
||||
errln("English text without C1 bytes does not detect as ISO-8859-1!");
|
||||
}
|
||||
|
||||
freeBytes(bWindows);
|
||||
freeBytes(bISO);
|
||||
|
||||
ucsdet_close(csd);
|
||||
}
|
||||
|
||||
void CharsetDetectionTest::DetectionTest()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
char path[2048];
|
||||
const char *testFilePath = getPath(path, "csdetest.xml");
|
||||
|
||||
if (testFilePath == NULL) {
|
||||
return; /* Couldn't get path: error message already output. */
|
||||
}
|
||||
|
||||
UXMLParser *parser = UXMLParser::createParser(status);
|
||||
UXMLElement *root = parser->parseFile(testFilePath, status);
|
||||
|
||||
UnicodeString test_case = UNICODE_STRING_SIMPLE("test-case");
|
||||
UnicodeString id_attr = UNICODE_STRING_SIMPLE("id");
|
||||
UnicodeString enc_attr = UNICODE_STRING_SIMPLE("encodings");
|
||||
|
||||
const UXMLElement *testCase;
|
||||
int32_t tc = 0;
|
||||
|
||||
while((testCase = root->nextChildElement(tc)) != NULL) {
|
||||
if (testCase->getTagName().compare(test_case) == 0) {
|
||||
const UnicodeString *id = testCase->getAttribute(id_attr);
|
||||
const UnicodeString *encodings = testCase->getAttribute(enc_attr);
|
||||
const UnicodeString text = testCase->getText(TRUE);
|
||||
int32_t encodingCount;
|
||||
UnicodeString *encodingList = split(*encodings, CH_SPACE, encodingCount);
|
||||
|
||||
for(int32_t e = 0; e < encodingCount; e += 1) {
|
||||
checkEncoding(text, encodingList[e], *id);
|
||||
}
|
||||
|
||||
delete[] encodingList;
|
||||
}
|
||||
}
|
||||
|
||||
delete root;
|
||||
delete parser;
|
||||
}
|
||||
|
||||
|
||||
39
icu4c/source/test/intltest/csdetest.h
Normal file
39
icu4c/source/test/intltest/csdetest.h
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
**********************************************************************
|
||||
* Copyright (C) 2005-2006, International Business Machines
|
||||
* Corporation and others. All Rights Reserved.
|
||||
**********************************************************************
|
||||
*/
|
||||
|
||||
#ifndef __CSDETEST_H
|
||||
#define __CSDETEST_H
|
||||
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/unistr.h"
|
||||
|
||||
#include "intltest.h"
|
||||
|
||||
class CharsetDetectionTest: public IntlTest {
|
||||
public:
|
||||
|
||||
CharsetDetectionTest();
|
||||
virtual ~CharsetDetectionTest();
|
||||
|
||||
virtual void runIndexedTest(int32_t index, UBool exec, const char* &name, char* par = NULL );
|
||||
|
||||
virtual void ConstructionTest();
|
||||
virtual void UTF8Test();
|
||||
virtual void UTF16Test();
|
||||
virtual void C1BytesTest();
|
||||
virtual void InputFilterTest();
|
||||
virtual void DetectionTest();
|
||||
|
||||
private:
|
||||
void checkEncoding(const UnicodeString &testString,
|
||||
const UnicodeString &encoding, const UnicodeString &id);
|
||||
|
||||
virtual const char *getPath(char buffer[2048], const char *filename);
|
||||
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
@ -913,6 +913,16 @@
|
|||
RelativePath=".\convtest.h">
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="charset detection"
|
||||
Filter="">
|
||||
<File
|
||||
RelativePath=".\csdetest.cpp">
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\csdetest.h">
|
||||
</File>
|
||||
</Filter>
|
||||
</Files>
|
||||
<Globals>
|
||||
</Globals>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/********************************************************************
|
||||
* COPYRIGHT:
|
||||
* Copyright (c) 1998-2004, International Business Machines Corporation and
|
||||
* Copyright (c) 1998-2006, International Business Machines Corporation and
|
||||
* others. All Rights Reserved.
|
||||
********************************************************************/
|
||||
|
||||
|
|
@ -32,6 +32,7 @@
|
|||
#include "icusvtst.h"
|
||||
#include "testidna.h"
|
||||
#include "convtest.h"
|
||||
#include "csdetest.h"
|
||||
|
||||
#define CASE_SUITE(id, suite) case id: \
|
||||
name = #suite; \
|
||||
|
|
@ -169,6 +170,15 @@ void MajorTestLevel::runIndexedTest( int32_t index, UBool exec, const char* &nam
|
|||
#endif
|
||||
break;
|
||||
|
||||
case 13: name = "csdet";
|
||||
if (exec) {
|
||||
logln("TestSuite CharsetDetection---"); logln();
|
||||
CharsetDetectionTest test;
|
||||
callTest(test, par);
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
default: name = ""; break;
|
||||
}
|
||||
|
||||
|
|
|
|||
553
icu4c/source/test/testdata/csdetest.xml
vendored
Normal file
553
icu4c/source/test/testdata/csdetest.xml
vendored
Normal file
|
|
@ -0,0 +1,553 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- Copyright (c) 2005-2006 IBM Corporation and others. All rights reserved -->
|
||||
<!-- See individual test cases for their specific copyright. -->
|
||||
|
||||
<charset-detection-tests>
|
||||
<test-case id="IUC10-ar" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-6/ar windows-1256/ar">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
أوروبا, برمجيات الحاسوب + انترنيت :
|
||||
تصبح عالميا مع يونيكود
|
||||
IUC10
|
||||
تسجّل الآن لحضور المؤتمر الدولي العاشر ليونيكود, الذي سيعقد في 10-12 آذار 1997 بمدينة ماينتس,
|
||||
ألمانيا. وسيجمع المؤتمر بين خبراء من كافة قطاعات الصناعة على الشبكة العالمية انترنيت ويونيكود, حيث ستتم,
|
||||
على الصعيدين الدولي والمحلي على حد سواء مناقشة سبل استخدام يونكود في النظم القائمة وفيما يخص التطبيقات
|
||||
الحاسوبية, الخطوط, تصميم النصوص والحوسبة متعددة اللغات.
|
||||
|
||||
Unicode
|
||||
عندما يريد العالم أن يتكلّم, فهو يتحدّث بلغة يونيكود
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-da-Q" encodings="UTF-8 UTF-32BE UTF-32LE windows-1252/da">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software + Internet:
|
||||
Bliv global med Unicode
|
||||
IUC10
|
||||
Indskriv dig nu til den tiende internationale Unicode-konference, der holdes den 10-12
|
||||
marts 1997 i Mainz, Tyskland. Konferencen samler eksperter fra hele verden inden for det
|
||||
globale Internet og Unicode, internationalisering og lokalisering, implementering af
|
||||
Unicode i styresystemer og programmer, skrifttyper, tekst-layout og flersproget databehandling.
|
||||
|
||||
Unicode
|
||||
Når verden vil tale, taler den “Unicode.”
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-da" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/da">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software + Internet:
|
||||
Bliv global med Unicode
|
||||
IUC10
|
||||
Indskriv dig nu til den tiende internationale Unicode-konference, der holdes den 10-12
|
||||
marts 1997 i Mainz, Tyskland. Konferencen samler eksperter fra hele verden inden for det
|
||||
globale Internet og Unicode, internationalisering og lokalisering, implementering af
|
||||
Unicode i styresystemer og programmer, skrifttyper, tekst-layout og flersproget databehandling.
|
||||
|
||||
Unicode
|
||||
Når verden vil tale, taler den Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-de" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/de">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software + das Internet:
|
||||
International mit Unicode
|
||||
IUC10
|
||||
Melden Sie sich jetzt für die 10. Internationale Unicode Konferenz an, die in der Zeit vom 10.-12. März 1997 in
|
||||
Mainz stattfinden wird. Die Konferenz ist ein Treffpunkt für Betriebsexperten aus den Bereichen globales
|
||||
Internet und Unicode, Internationalisierung und Lokalisierung, die Implementierung von Unicode in
|
||||
Betriebssystemen und Programmen, sowie für Schriftarten, Textlayout und mehrsprachige Computeranwendungen.
|
||||
|
||||
Unicode
|
||||
Wenn die Welt miteinander spricht, spricht sie Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<!-- No UTF-8 in this test because there are no non-ASCII characters. -->
|
||||
<test-case id="IUC10-en" encodings="UTF-32BE UTF-32LE ISO-8859-1/en">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europe, Software + the Internet:
|
||||
Going Global with Unicode
|
||||
IUC10
|
||||
Register now for the Tenth International Unicode Conference, to be held on March 10-12, 1997,
|
||||
in Mainz, Germany. The Conference will bring together industry-wide experts on the global Internet and
|
||||
Unicode, internationalization and localization, implementation of Unicode in operating systems and applications,
|
||||
fonts, text layout, and multilingual computing.
|
||||
|
||||
Unicode
|
||||
When the world wants to talk, it speaks Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-es" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/es">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software + el Internet:
|
||||
Mundializando con Unicode
|
||||
IUC10
|
||||
Inscríbase ahora para la Décima Conferencia Internacional Unicode, que tendrá lugar del 10 al 12 de marzo de
|
||||
1997 en Maguncia, Alemania. La Conferencia reunirá expertos de los sectores de la mundialización del Internet y
|
||||
Unicode, la internacionalización y localización, implementación de Unicode en sistemas operativos y aplicaciones,
|
||||
tipos, composición de texto e informática multilingüe.
|
||||
|
||||
Unicode
|
||||
Cuando el mundo quiere conversar, habla Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-fr" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/fr">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
L'Europe, le logiciel et l'Internet :
|
||||
la mondialisation avec Unicode
|
||||
IUC10
|
||||
Inscrivez-vous dès maintenant à la dixième Conférence internationale sur Unicode, qui se tiendra du 10 au 12
|
||||
mars 1997 à Mayence, en Allemagne. Cette conférence rassemblera des experts de tous les horizons industriels
|
||||
sur les sujets suivants : l'Internet mondial et Unicode, l'internationalisation et l'adaptation locale,
|
||||
l'implémentation d'Unicode dans les systèmes d'exploitation et les applications, les polices de caractères,
|
||||
la disposition de texte, l'informatique plurilingue.
|
||||
|
||||
Unicode
|
||||
Quand le monde veut communiquer, il parle en Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-he" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-8/he">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
אירופה, תוכנה והאינטרנט:
|
||||
Unicode יוצא לשוק העולמי
|
||||
IUC10
|
||||
הירשמו כעת לכנס Unicode הבינלאומי העשירי, שייערך בין התאריכים 12-10 במרץ 1997, במיינץ שבגרמניה. בכנס
|
||||
ישתתפו מומחים מכל ענפי התעשייה בנושא האינטרנט העולמי וה-Unicode, בהתאמה לשוק הבינלאומי והמקומי, ביישום
|
||||
Unicode במערכות הפעלה וביישומים, בגופנים, בפריסת טקסט ובמחשוב רב-לשוני.
|
||||
|
||||
Unicode
|
||||
כאשר העולם רוצה לדבר, הוא מדבר ב-Unicode
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-he-Q" encodings="UTF-8 UTF-32BE UTF-32LE windows-1255/he">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
אירופה, תוכנה והאינטרנט:
|
||||
Unicode יוצא לשוק העולמי
|
||||
IUC10
|
||||
הירשמו כעת לכנס Unicode הבינלאומי העשירי, שייערך בין התאריכים 12-10 במרץ 1997, במיינץ שבגרמניה. בכנס
|
||||
ישתתפו מומחים מכל ענפי התעשייה בנושא האינטרנט העולמי וה-Unicode, בהתאמה לשוק הבינלאומי והמקומי, ביישום
|
||||
Unicode במערכות הפעלה וביישומים, בגופנים, בפריסת טקסט ובמחשוב רב-לשוני.
|
||||
|
||||
Unicode
|
||||
כאשר העולם רוצה לדבר, הוא מדבר -“Unicode.”
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-hu" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-2/hu">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Európa, a Szoftver s az Internet -
|
||||
Globálissá Változik a Unicode Által
|
||||
IUC10
|
||||
Iratkozzon már most a Tizedik Nemzetközi Unicode Konferenciára, amely Március 10-12 1997
|
||||
lesz megtartva, Meinz-be, Németországba. Ebben a Konferencián az iparág szerte sok szakértő
|
||||
fog részt venni: a globális Internet s Unicode nemzetközisítése s lokalizálása, a
|
||||
Unicode beteljesítése a működő rendszerekben s alkalmazásokban, fontokba, szöveg
|
||||
térbeosztásba s többnyelvű computerekben.
|
||||
|
||||
Unicode
|
||||
Ha a világ beszélni akar, azt Unicode-ul mondja.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-hu-Q" encodings="UTF-8 UTF-32BE UTF-32LE windows-1250/hu">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Európa, a Szoftver s az Internet -
|
||||
Globálissá Változik a Unicode Által
|
||||
IUC10
|
||||
Iratkozzon már most a Tizedik Nemzetközi Unicode Konferenciára, amely Március 10-12 1997
|
||||
lesz megtartva, Meinz-be, Németországba. Ebben a Konferencián az iparág szerte sok szakértő
|
||||
fog részt venni: a globális Internet s Unicode nemzetközisítése s lokalizálása, a
|
||||
Unicode beteljesítése a működő rendszerekben s alkalmazásokban, fontokba, szöveg
|
||||
térbeosztásba s többnyelvű computerekben.
|
||||
|
||||
Unicode
|
||||
Ha a világ beszélni akar, azt “Unicode”-ul mondja.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-it" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/it">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, software e Internet:
|
||||
Globalizzazione con Unicode
|
||||
IUC10
|
||||
Iscrivetevi subito alla X Conferenza Internazionale su Unicode, che si terrà dal 10 al 12 marzo 1997 a
|
||||
Mainz in Germania. Alla Conferenza parteciperanno esperti di tutti i settori per discutere di Internet globale e
|
||||
Unicode, internazionalizzazione e localizzazione, implementazione di Unicode in sistemi operativi e applicazioni,
|
||||
caratteri, composizione dei testi ed elaborazione multilingue.
|
||||
|
||||
Unicode
|
||||
Quando il mondo vuole comunicare, parla Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<!-- No EUC-JP in this test because it detects as GB18030 -->
|
||||
<test-case id="IUC10-jp" encodings="UTF-8 UTF-32BE UTF-32LE Shift_JIS ISO-2022-JP">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
ヨーロッパ、ソフトウェア、そしてインターネット:
|
||||
Unicode でグローバル化を狙って
|
||||
IUC10
|
||||
第 10 回の Unicode 会議は 1997 年 3 月 10-12日、ドイツのマインツで開かれます。参加希望の方は今すぐ登録してください。
|
||||
この会議では、グローバルなインタネット、Unicode、ソフトウェアの国際化およびローカリゼーション、OS およびアプリケーションでの
|
||||
Unicode のインプリメンテーション、フォント、テキスト表示、マルチ言語コンピューティングにおける業界の専門家が集まります。
|
||||
|
||||
Unicode
|
||||
世界的に話すなら、Unicode です。
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-ko" encodings="UTF-8 UTF-32BE UTF-32LE EUC-KR ISO-2022-KR">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
유럽, 소프트웨어 그리고 인터넷:
|
||||
유니코드와 함께 세계로
|
||||
IUC10
|
||||
제10회 유니코드 국제 회의가 1997년 3월 10일부터 12일까지 독일의 마인즈에서 열립니다. 지금 등록하십시오.
|
||||
이 회의에서는 업계 전반의 전문가들이 함께 모여 다음과 같은 분야를 다룹니다. - 인터넷과 유니코드, 국제화와 지역화,
|
||||
운영 체제와 응용 프로그램에서 유니코드의 구현, 글꼴, 문자 배열, 다국어 컴퓨팅.
|
||||
|
||||
Unicode
|
||||
세계를 향한 대화, 유니코드로 하십시오
|
||||
|
||||
</test-case>
|
||||
|
||||
<!-- No UTF-8 in this test because there are no non-ASCII characters. -->
|
||||
<test-case id="IUC10-nl" encodings="UTF-32BE UTF-32LE ISO-8859-1/nl">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software + het Internet:
|
||||
wereldwijd met Unicode
|
||||
IUC10
|
||||
Meld u nu aan voor de Tiende Internationale Unicode-conferentie, die van 10 tot 12 maart 1997 in
|
||||
Mainz (Duitsland) wordt gehouden. De Conferentie is een ontmoetingsplaats voor experts uit de industrie op het
|
||||
gebied van het wereldwijde Internet en Unicode, internationalisatie en localisatie, implementatie van Unicode in
|
||||
besturingssystemen en applicaties, lettertypes, tekstopmaak en meertalig computergebruik.
|
||||
|
||||
Unicode
|
||||
Als de wereld wil praten, spreekt hij Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<!-- No language for ISO-8859-1 in this test because no-BO is recogonized as Danish... -->
|
||||
<test-case id="IUC10-no-BO" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Programvare og Internet:
|
||||
Global fokus med Unicode
|
||||
IUC10
|
||||
Registrer deg som deltager på den tiende inernasjonale Unicode konferansen i Mainz, Tyskland, fra 10. til 12. mars,
|
||||
1997. Konferansen vil samle eksperter på Internet, Unicode, internasjonalisering og integrasjon av Unicode i
|
||||
operativsystemer og programmer, fonter, tekst layout og flerspråklig databehandling.
|
||||
|
||||
Unicode
|
||||
Når verden vil snakke, snakker den Unicode
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-no-NY" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/no">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, programvare og Internett:
|
||||
Femn verda med Unicode
|
||||
IUC10
|
||||
Meld deg på den 10. internasjonale Unicode-konferansen. Han går føre seg i Mainz i Tyskland i dagane 10.--12. mars
|
||||
1997, og samlar fagkunnige innan konferansetemaet frå heile databransjen. Tema: Det globale Internettet og
|
||||
Unicode, internasjonalisering og nasjonal tilpassing, implementering av Unicode i operativsystem og brukarprogram,
|
||||
skriftsnitt (fontar), tekstutlegg, og fleirspråkleg databehandling.
|
||||
|
||||
Unicode
|
||||
Når verda ønskjer å snakke, talar ho Unicode
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-pt-BR" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/pt">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software e a Internet:
|
||||
Globalização com o Unicode
|
||||
IUC10
|
||||
Inscreva-se agora para a Décima Conferência Internacional Sobre O Unicode, realizada entre os dias 10 e 12 de
|
||||
março de 1997 em Mainz na Alemanha. A Conferência reunirá peritos de todas as áreas da indústria especializados
|
||||
em assuntos relacionados com a Internet global e o Unicode, internacionalização e localização de software,
|
||||
implementação do Unicode em sistemas operacionais e aplicativos, fontes, layout de texto e informática multilíngüe.
|
||||
|
||||
Unicode
|
||||
Quando o mundo quer falar, fala Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-pt-PT" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/pt">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software e a Internet:
|
||||
Globalização com o Unicode
|
||||
IUC10
|
||||
Inscreva-se agora para a Décima Conferência Internacional Sobre O Unicode, a ser realizada entre os dias 10 e 12
|
||||
de Março de 1997 em Mainz na Alemanha. A Conferência reunirá peritos de todas as áreas da indústria
|
||||
especializados em assuntos relacionados com a Internet global e o Unicode, internacionalização e localização de
|
||||
software, implementação do Unicode em sistemas operativos e aplicações, tipos de letra, esquematização de
|
||||
texto e informática multilíngue.
|
||||
|
||||
Unicode
|
||||
Quando o mundo quer falar, fala Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-ro" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-2/ro">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, Software şi Internet:
|
||||
Globalizarea cu Unicode
|
||||
IUC10
|
||||
Inscrieţi-vă acum la a Zecea Conferinţă Internaţională "Unicode" ce va avea loc in
|
||||
perioada de 10-12 martie, 1997 în Mainz, Germania. Conferinţa va întruni experţi din
|
||||
variate domenii: Internet global şi Unicode, internaţionalizare şi localizare,
|
||||
implementarede Unicode în sisteme de operare şi aplicaţii, fonturi, aranjare de text în
|
||||
pagină, computerizare multilinguală.
|
||||
|
||||
Unicode
|
||||
Când lumea vrea să comunice, vorbeşte Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-ru" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-5/ru windows-1251/ru KOI8-R/ru">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Европа, Программное обеспечение + Интернет:
|
||||
Unicode в глобальных масштабах
|
||||
IUC10
|
||||
Зарегистрируйтесь сейчас на Десятую Международную Конференцию по Unicode, которая состоится
|
||||
10-12 марта 1997 года в Майнце в Германии. Конференция соберет широкий круг экспертов по и
|
||||
вопросам глобального Интернета и Unicode, локализации и интернационализации, воплощению
|
||||
применению Unicode в различных операционных системах и программных приложениях,
|
||||
шрифтах, верстке и многоязычных компьютерных системах.
|
||||
|
||||
Unicode
|
||||
Когда мир желает общаться, он общается на Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-sv" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-1/sv">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Europa, programvara och Internet:
|
||||
globalisera med Unicode
|
||||
IUC10
|
||||
Anmäl Dig till den tionde internationella Unicode-konferensen, som hålls den 10-12 mars 1997 i Mainz,
|
||||
Tyskland. Vid konferensen kommer experter inom följande områden att delta: det globala Internet och Unicode,
|
||||
internationalisering och lokalisering, implementering av Unicode i operativsystem, tillämpningar, typsnitt,
|
||||
textlayout och mångspråklig datoranvändning.
|
||||
|
||||
Unicode
|
||||
När världen vill tala, så talar den Unicode.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="IUC10-yi" encodings="UTF-8 UTF-32BE UTF-32LE">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
אײראָפּע: פּראָגראַמװאַרג און די װעלטנעץ:
|
||||
אוניקאָד איבער דער גאָרער װעלט
|
||||
IUC10
|
||||
פֿאַרשרײַבט זיך שױן אױף דער צענטער אינטערנאַציאָנאַלער אוניקאָד-קאָנפֿערענץ, װאָס װעט פֿאָרקומען דעם
|
||||
10טן ביזן 12טן מאַרץ, 1997, אין מײַנץ, דײַטשלאַנד. די קאָנפֿערענץ װעט צוזאַמענברענגן מבֿינים פֿון װעלטנעץ,
|
||||
אוניקאָד, אי אַלװעלטלעכן אי סבֿיבֿהדיקן פּראָגראַמװאַרג, אַרײַנשטעלן אוניקאָד אין אָפּעריר-סיסטעמען און
|
||||
אָנװענדונגען, שריפֿטן, טעקסט-אױסשטעל, און מערשפּראַכיקע קאָמפּיוטערײַ.
|
||||
|
||||
Unicode
|
||||
אַז די װעלט װיל רעדן, רעדט זי אוניקאָד
|
||||
|
||||
</test-case>
|
||||
|
||||
<!-- test-case id="IUC10-zh-Hant" encodings="UTF-8 UTF-32BE UTF-32LE ISO-2022-CN GB18030" -->
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
<!--
|
||||
歐洲，軟體及網際網路：
|
||||
讓統一碼（Unicode）領你進入全世界
|
||||
IUC10
|
||||
你現在就應報名將在１９９７年３月１０至１２日於德國美姿城（Mainz）召開的第十屆國際統一碼研討會。
|
||||
本次研討會將邀請多位業界專家研討關於全球網際網路及統一碼發展、國際化及本土化、
|
||||
支援統一碼的作業系統及應用程式、字型、文字排版、電腦多國語文化等多項課題。
|
||||
|
||||
Unicode
|
||||
當世界需要溝通時，請用統一碼（Unicode）
|
||||
-->
|
||||
|
||||
<!-- /test-case -->
|
||||
|
||||
<!-- No ISO-2022-CN in this test because Java doesn't support it in both directions :-( -->
|
||||
<test-case id="IUC10-zh-Hans" encodings="UTF-8 UTF-32BE UTF-32LE ISO-2022-CN GB18030">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
欧洲，软件＋互联网
|
||||
用统一码 (Unicode) 走遍世界
|
||||
IUC10
|
||||
将于1997年 3 月10日－12日在德国 Mainz 市举行的第十届统一码国际研讨会现在开始注册。
|
||||
本次会议将汇集各方面的专家。涉及的领域包括：国际互联网和统一码，国际化和本地化，
|
||||
统一码在操作系统和应用软件中的实现，字型，文本格式以及多文种计算等。
|
||||
|
||||
Unicode
|
||||
当世界需要沟通时，请用Unicode！
|
||||
|
||||
Conference Program
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-cz" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-2/cs">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Co je Unicode?
|
||||
|
||||
Unicode přiřazuje každému znaku jedinečné číslo,
|
||||
nezávisle na platformě,
|
||||
nezávisle na programu,
|
||||
nezávisle na jazyce.
|
||||
|
||||
Počítače, ze své podstaty, pracují pouze s čísly. Písmena a další znaky ukládají tak, že každému z nich
|
||||
přiřadí číslo. Před vznikem Unicode existovaly stovky rozdílných kódovacích systémů pro přiřazování těchto
|
||||
čísel. Žádné z těchto kódování nemohlo obsahovat dostatek znaků: například Evropská unie sama potřebuje
|
||||
několik různých kódování, aby pokryla všechny své jazyky. Dokonce i pro jeden jediný jazyk, jako je angličtina,
|
||||
nevyhovovalo žádné kódování pro všechny písmena, interpunkci a běžně používané technické symboly.
|
||||
|
||||
Tyto kódovací systémy také byly v konfliktu jeden s druhým. To znamená, že dvě kódování mohou používat
|
||||
stejné číslo pro dva různé znaky, nebo používat různá čísla pro stejný znak. Jakýkoli počítač (zvláště servery)
|
||||
musí podporovat mnoho různých kódování; přesto, kdykoli jsou data předávána mezi různými kódováními nebo
|
||||
platformami, hrozí, že tato data budou poškozena.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-el" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-7/el">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Τι είναι το Unicode;
|
||||
|
||||
Η κωδικοσελίδα Unicode προτείνει έναν και μοναδικό αριθμό για κάθε χαρακτήρα,
|
||||
ανεξάρτητα από το λειτουργικό σύστημα,
|
||||
ανεξάρτητα από το λογισμικό,
|
||||
ανεξάρτητα από την γλώσσα.
|
||||
|
||||
Οι ηλεκτρονικοί υπολογιστές, σε τελική ανάλυση, χειρίζονται απλώς αριθμούς. Αποθηκεύουν γράμματα και
|
||||
άλλους χαρακτήρες αντιστοιχώντας στο καθένα τους από έναν αριθμό (ονομάζουμε μία τέτοια αντιστοιχία
|
||||
κωδικοσελίδα). Πριν την εφεύρεση του Unicode, υπήρχαν εκατοντάδες διαφορετικές κωδικοσελίδες. Λόγω
|
||||
περιορισμών μεγέθους όμως, σε καμία κωδικοσελίδα δεν χωρούσαν αρκετοί χαρακτήρες: λόγου χάριν,
|
||||
η Ευρωπαϊκή Ένωση χρειαζόταν πλήθος διαφορετικών κωδικοσελίδων για να καλύψει όλες τις γλώσσες των
|
||||
χωρών-μελών της. Ακόμα και για μία και μόνη γλώσσα, όπως π.χ. τα Αγγλικά, μία κωδικοσελίδα δεν
|
||||
επαρκούσε για να καλύψει όλα τα γράμματα, σημεία στίξης και τεχνικά σύμβολα ευρείας χρήσης.
|
||||
|
||||
Εκτός αυτού, οι κωδικοσελίδες αυτές διαφωνούσαν μεταξύ τους. Έτσι, δύο κωδικοσελίδες μπορούσαν κάλλιστα να
|
||||
χρησιμοποιούν τον ίδιο αριθμό για δύο διαφορετικούς χαρακτήρες, ή να χρησιμοποιούν διαφορετικούς αριθμούς για
|
||||
τον ίδιο χαρακτήρα. Κάθε υπολογιστής (και ιδίως εάν ήταν διακομιστής) έπρεπε να υποστηρίζει πλήθος διαφορετικών
|
||||
κωδικοσελίδων· ταυτόχρονα κάθε φορά που δεδομένα μεταφέρονταν μεταξύ διαφορετικών κωδικοσελίδων ή
|
||||
λειτουργικών συστημάτων, τα δεδομένα αυτά κινδύνευαν να αλλοιωθούν.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-el-Q" encodings="UTF-8 UTF-32BE UTF-32LE windows-1253/el">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Τι είναι το “Unicode”;
|
||||
|
||||
Η κωδικοσελίδα Unicode προτείνει έναν και μοναδικό αριθμό για κάθε χαρακτήρα,
|
||||
ανεξάρτητα από το λειτουργικό σύστημα,
|
||||
ανεξάρτητα από το λογισμικό,
|
||||
ανεξάρτητα από την γλώσσα.
|
||||
|
||||
Οι ηλεκτρονικοί υπολογιστές, σε τελική ανάλυση, χειρίζονται απλώς αριθμούς. Αποθηκεύουν γράμματα και
|
||||
άλλους χαρακτήρες αντιστοιχώντας στο καθένα τους από έναν αριθμό (ονομάζουμε μία τέτοια αντιστοιχία
|
||||
κωδικοσελίδα). Πριν την εφεύρεση του Unicode, υπήρχαν εκατοντάδες διαφορετικές κωδικοσελίδες. Λόγω
|
||||
περιορισμών μεγέθους όμως, σε καμία κωδικοσελίδα δεν χωρούσαν αρκετοί χαρακτήρες: λόγου χάριν,
|
||||
η Ευρωπαϊκή Ένωση χρειαζόταν πλήθος διαφορετικών κωδικοσελίδων για να καλύψει όλες τις γλώσσες των
|
||||
χωρών-μελών της. Ακόμα και για μία και μόνη γλώσσα, όπως π.χ. τα Αγγλικά, μία κωδικοσελίδα δεν
|
||||
επαρκούσε για να καλύψει όλα τα γράμματα, σημεία στίξης και τεχνικά σύμβολα ευρείας χρήσης.
|
||||
|
||||
Εκτός αυτού, οι κωδικοσελίδες αυτές διαφωνούσαν μεταξύ τους. Έτσι, δύο κωδικοσελίδες μπορούσαν κάλλιστα να
|
||||
χρησιμοποιούν τον ίδιο αριθμό για δύο διαφορετικούς χαρακτήρες, ή να χρησιμοποιούν διαφορετικούς αριθμούς για
|
||||
τον ίδιο χαρακτήρα. Κάθε υπολογιστής (και ιδίως εάν ήταν διακομιστής) έπρεπε να υποστηρίζει πλήθος διαφορετικών
|
||||
κωδικοσελίδων· ταυτόχρονα κάθε φορά που δεδομένα μεταφέρονταν μεταξύ διαφορετικών κωδικοσελίδων ή
|
||||
λειτουργικών συστημάτων, τα δεδομένα αυτά κινδύνευαν να αλλοιωθούν.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-pl" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-2/pl">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Czym jest Unikod ?
|
||||
|
||||
Unikod przypisuje unikalny numer każdemu znakowi, niezależny od używanej platformy, programu czy języka.
|
||||
|
||||
Zasadniczo, komputery rozumieją tylko liczby. Zapisują litery i inne znaki przypisując każdemu z nich liczbę.
|
||||
Nim powstał Unikod, było wiele różnych systemów kodowania przypisujących te liczby. Brakowało jednego,
|
||||
który mógłby pomieścić wystarczająco dużą liczbę znaków. Przykładowo, sama Unia Europejska potrzebowała
|
||||
kilku różnych kodowań, by móc używać wszystkich używanych w niej języków. Nawet dla pojedynczego języka
|
||||
takiego jak np. angielski brakowało jednego kodowania, które byłoby odpowiednie dla zaprezentowania
|
||||
wszystkich liter, znaków przestankowych i popularnych symboli technicznych.
|
||||
|
||||
Innym problemem było, że kodowania te kolidowały ze sobą. Dwa, różne kodowania używały jednej liczby dla dwu
|
||||
różnych znaków lub różnych liczb dla tego samego znaku. Wszystkie komputery (między innymi serwery) muszą
|
||||
wspierać wszystkie te kodowania, gdyż dane przesyłane między różnymi systemami operacyjnymi zawsze
|
||||
narażone są na uszkodzenie.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-tr" encodings="UTF-8 UTF-32BE UTF-32LE ISO-8859-9/tr">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
Evrensel Kod Nedir?
|
||||
|
||||
Evrensel Kod her yazı karakteri için bir ve yalnız bir sayı şart koşar,
|
||||
hangi altyapı,
|
||||
hangi yazılım,
|
||||
hangi dil olursa olsun.
|
||||
|
||||
İlke olarak, bilgisayarlar sadece sayılarla işlem yaparlar. Kelimelerin ve yazı karakterlerinin her biri için
|
||||
birer sayı atarlar ve böyle saklarlar. Evrensel Kod keşfedilmeden önce, bu sayıları atamak için birçok şifreleme
|
||||
yöntemi vardı. Ancak, tüm bu dilleri gösterebilecek, örneğin; Avrupa Topluluğu bünyesindeki tüm ülkelerin dillerini
|
||||
kapsayacak bir tek şifreleme yöntemi yoktu. Bunun yanısıra, sadece İngilizcedeki harfleri, noktalama
|
||||
işaretlerini ve teknik sembolleri kapsayan tek bir şifreleme yöntemi de bulunmamaktaydı.
|
||||
|
||||
Bu şifreleme yöntemleri kendi aralarında çelişmektedir. İki farklı şifreleme, aynı sayıyı iki farklı karaktere
|
||||
vermiş olabilir ya da farklı sayılar aynı karekteri kodlayabilir. Bilgisayarlar, özellikle sunucular, birçok
|
||||
şifrelemeyi desteklemek zorundadırlar; veriler, farklı şifreleme ve altyapılardan geçerken bozulma riski taşırlar.
|
||||
|
||||
</test-case>
|
||||
|
||||
<test-case id="WIU-tr-Q" encodings="UTF-8 UTF-32BE UTF-32LE windows-1254/tr">
|
||||
<!-- Copyright © 1991-2005 Unicode, Inc. All rights reserved. -->
|
||||
|
||||
“Evrensel Kod” Nedir?
|
||||
|
||||
Evrensel Kod her yazı karakteri için bir ve yalnız bir sayı şart koşar,
|
||||
hangi altyapı,
|
||||
hangi yazılım,
|
||||
hangi dil olursa olsun.
|
||||
|
||||
İlke olarak, bilgisayarlar sadece sayılarla işlem yaparlar. Kelimelerin ve yazı karakterlerinin her biri için
|
||||
birer sayı atarlar ve böyle saklarlar. Evrensel Kod keşfedilmeden önce, bu sayıları atamak için birçok şifreleme
|
||||
yöntemi vardı. Ancak, tüm bu dilleri gösterebilecek, örneğin; Avrupa Topluluğu bünyesindeki tüm ülkelerin dillerini
|
||||
kapsayacak bir tek şifreleme yöntemi yoktu. Bunun yanısıra, sadece İngilizcedeki harfleri, noktalama
|
||||
işaretlerini ve teknik sembolleri kapsayan tek bir şifreleme yöntemi de bulunmamaktaydı.
|
||||
|
||||
Bu şifreleme yöntemleri kendi aralarında çelişmektedir. İki farklı şifreleme, aynı sayıyı iki farklı karaktere
|
||||
vermiş olabilir ya da farklı sayılar aynı karekteri kodlayabilir. Bilgisayarlar, özellikle sunucular, birçok
|
||||
şifrelemeyi desteklemek zorundadırlar; veriler, farklı şifreleme ve altyapılardan geçerken bozulma riski taşırlar.
|
||||
|
||||
</test-case>
|
||||
</charset-detection-tests>
|
||||
Loading…
Add table
Reference in a new issue