organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-06 14:05:32 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 8

ICU-7530 Alphabetic index, move from branch to trunk.

Browse source 
X-SVN-Rev: 29482
This commit is contained in:
Andy Heninger 2011-02-25 22:21:30 +00:00
parent 0e2bf7da98
commit 7b7f48eef8
13 changed files with 2560 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
icu4c/source/common/uvector.cpp
View file

@ -1,6 +1,6 @@
/*
******************************************************************************
* Copyright (C) 1999-2010, International Business Machines Corporation and *
* Copyright (C) 1999-2011, International Business Machines Corporation and *
* others. All Rights Reserved. *
******************************************************************************
* Date Name Description
@ -549,5 +549,16 @@ void UVector::sort(USortComparator *compare, UErrorCode &ec) {
}
}
/**
* Sort with a user supplied comparator of type UComparator.
*/
void UVector::sortWithUComparator(UComparator *compare, const void *context, UErrorCode &ec) {
if (U_SUCCESS(ec)) {
uprv_sortArray(elements, count, sizeof(UHashTok),
compare, context, FALSE, &ec);
}
}
U_NAMESPACE_END

11
icu4c/source/common/uvector.h
View file

@ -1,6 +1,6 @@
/*
**********************************************************************
* Copyright (C) 1999-2009, International Business Machines
* Copyright (C) 1999-2011, International Business Machines
* Corporation and others. All Rights Reserved.
**********************************************************************
* Date Name Description
@ -14,6 +14,7 @@
#include "unicode/utypes.h"
#include "unicode/uobject.h"
#include "uarrsort.h"
#include "uhash.h"
U_NAMESPACE_BEGIN
@ -259,6 +260,14 @@ public:
*/
void sort(USortComparator *compare, UErrorCode &ec);
/**
* Sort the contents of this vector using a caller-supplied function
* of type UComparator to do the comparison. Provides more flexibility
* than uvector::sort() because an additional user-parameter can be passed to
* the comparison function.
*/
void sortWithUComparator(UComparator *compare, const void *context, UErrorCode &ec);
/**
* ICU "poor man's RTTI", returns a UClassID for this class.
*/

4
icu4c/source/i18n/Makefile.in
View file

@ -1,6 +1,6 @@
#******************************************************************************
#
# Copyright (C) 1998-2010, International Business Machines
# Copyright (C) 1998-2011, International Business Machines
# Corporation and others. All Rights Reserved.
#
#******************************************************************************
@ -84,7 +84,7 @@ zonemeta.o zstrfmt.o plurrule.o plurfmt.o selfmt.o dtitvfmt.o dtitvinf.o \
tmunit.o tmutamt.o tmutfmt.o colldata.o bmsearch.o bms.o currpinf.o \
uspoof.o uspoof_impl.o uspoof_build.o uspoof_conf.o uspoof_wsconf.o \
ztrans.o zrule.o vzone.o fphdlimp.o fpositer.o locdspnm.o \
decNumber.o decContext.o
decNumber.o decContext.o alphaindex.o
## Header files to install
HEADERS = $(srcdir)/unicode/*.h

1319
icu4c/source/i18n/alphaindex.cpp Normal file
View file

File diff suppressed because it is too large Load diff

3
icu4c/source/i18n/ucln_in.h
View file

@ -1,7 +1,7 @@
/*
******************************************************************************
* *
* Copyright (C) 2001-2009, International Business Machines *
* Copyright (C) 2001-2011, International Business Machines *
* Corporation and others. All Rights Reserved. *
* *
******************************************************************************
@ -46,6 +46,7 @@ typedef enum ECleanupI18NType {
UCLN_I18N_UCOL_BLD,
UCLN_I18N_CSDET,
UCLN_I18N_COLL_DATA,
UCLN_I18N_INDEX_CHARACTERS,
UCLN_I18N_COUNT /* This must be last */
} ECleanupI18NType;

6
icu4c/source/i18n/ulocdata.c
View file

@ -1,7 +1,7 @@
/*
******************************************************************************
* *
* Copyright (C) 2003-2009, International Business Machines *
* Copyright (C) 2003-2011, International Business Machines *
* Corporation and others. All Rights Reserved. *
* *
******************************************************************************
@ -100,7 +100,9 @@ U_CAPI USet* U_EXPORT2
ulocdata_getExemplarSet(ULocaleData *uld, USet *fillIn,
uint32_t options, ULocaleDataExemplarSetType extype, UErrorCode *status){
static const char* const exemplarSetTypes[] = { "ExemplarCharacters", "AuxExemplarCharacters" };
static const char* const exemplarSetTypes[] = { "ExemplarCharacters",
"AuxExemplarCharacters",
"ExemplarCharactersIndex"};
const UChar *exemplarChars = NULL;
int32_t len = 0;
UErrorCode localStatus = U_ZERO_ERROR;

717
icu4c/source/i18n/unicode/alphaindex.h Normal file
View file

@ -0,0 +1,717 @@
/*
*******************************************************************************
*
* Copyright (C) 2011 International Business Machines
* Corporation and others. All Rights Reserved.
*
*******************************************************************************
*/
#ifndef INDEXCHARS_H
#define INDEXCHARS_H
#include "unicode/utypes.h"
#include "unicode/uobject.h"
#include "unicode/locid.h"
/**
* \file
* \brief C++ API: Index Characters
*/
U_CDECL_BEGIN
/**
* Constants for Alphabetic Index Label Types.
* The form of these enum constants anticipates having a plain C API
* for Alphabetic Indexes that will also use them.
* @draft ICU 4.8
*/
typedef enum UAlphabeticIndexLabelType {
/**
* Normal Label, typically the starting letter of the names
* in the bucket with this label.
* @draft ICU 4.8
*/
U_ALPHAINDEX_NORMAL = 0,
/**
* Undeflow Label. The bucket with this label contains names
* in scripts that sort before any of the bucket labels in this index.
* @draft ICU 4.8
*/
U_ALPHAINDEX_UNDERFLOW = 1,
/**
* Inflow Label. The bucket with this label contains names
* in scripts that sort between two of the bucket labels in this index.
* Inflow labels are created when an index contains normal labels for
* multiple scripts, and skips other scripts that sort between some of the
* included scripts.
* @draft ICU 4.8
*/
U_ALPHAINDEX_INFLOW = 2,
/**
* Overflow Label. Te bucket with this label contains names in scripts
* that sort after all of the bucket labels in this index.
* @draft ICU 4.8
*/
U_ALPHAINDEX_OVERFLOW = 3
} UAlphabeticIndexLabelType;
struct UHashtable;
U_CDECL_END
U_NAMESPACE_BEGIN
// Forward Declarations
class Collator;
class RuleBasedCollator;
class StringEnumeration;
class UnicodeSet;
class UVector;
/**
* class AlphabeticIndex supports the creation of a UI index appropriate for a given language, such as:
*
* <pre>
* <b>… A B C D E F G H I J K L M N O P Q R S T U V W X Y Z Æ Ø Å …</b>
*
* <b>A</b>
* Addison
* Albertson
* Azensky
* <b>B</b>
* Baker
* ...
* </pre>
*
* The class can generate a list of labels for use as a UI "index", that is, a list of
* clickable characters (or character sequences) that allow the user to see a segment
* (bucket) of a larger "target" list. That is, each label corresponds to a bucket in
* the target list, where everything in the bucket is greater than or equal to the character
* (according to the locale's collation). Strings can be added to the index;
* they will be in sorted order in the right bucket.
* <p>
* The class also supports having buckets for strings before the first (underflow),
* after the last (overflow), and between scripts (inflow). For example, if the index
* is constructed with labels for Russian and English, Greek characters would fall
* into an inflow bucket between the other two scripts.
* <p>
* The AlphabeticIndex class is not intended for public subclassing.
* <p>
* <i>Example</i>
* <p>
* The "show..." methods below are just to illustrate usage.
*
* <pre>
* // Create a simple index. "Item" is assumed to be an application
* // defined type that the application's UI and other processing knows about,
* // and that has a name.
*
* UErrorCode status = U_ZERO_ERROR;
* AlphabeticIndex index = new AlphabeticIndex(desiredLocale, status);
* index->addLabels(additionalLocale, status);
* for (Item *item in some source of Items ) {
* index->addRecord(item->name(), item, status);
* }
* ...
* // Show index at top. We could skip or gray out empty buckets
*
* while (index->nextBucket(status)) {
* if (showAll || index->getBucketRecordCount() != 0) {
* showLabelAtTop(UI, index->getBucketLabel());
* }
* }
* ...
* // Show the buckets with their contents, skipping empty buckets
*
* index->resetBucketIterator(status);
* while (index->nextBucket(status)) {
* if (index->getBucketRecordCount() != 0) {
* showLabelInList(UI, index->getBucketLabel());
* while (index->nextRecord(status)) {
* showIndexedItem(UI, static_cast<Item *>(index->getRecordData()))
* </pre>
*
* The caller can build different UIs using this class.
* For example, an index character could be omitted or grayed-out
* if its bucket is empty. Small buckets could also be combined based on size, such as:
*
* <pre>
* <b>… A-F G-N O-Z …</b>
* </pre>
*
* <p>
* <b>Notes:</b>
* <ul>
* <li>Additional collation parameters can be passed in as part of the locale name.
* For example, German plus numeric
* sorting would be "de@kn-true".
*
* @draft ICU 4.8
* @provisional This API might change or be removed in a future release.
*/
class U_I18N_API AlphabeticIndex: public UObject {
public:
/**
* Construct an AlphabeticIndex object for the specified locale. If the locale's
* data does not include index characters, a set of them will be
* synthesized based on the locale's exemplar characters. The locale
* determines the sorting order for both the index characters and the
* user item names appearing under each Index character.
*
* @param locale the desired locale.
* @param status Error code, will be set with the reason if the construction
* of the AlphabeticIndex object fails.
* @draft ICU 4.8
*/
AlphabeticIndex(const Locale &locale, UErrorCode &status);
/**
* Add Labels to this Index. The labels are additions to those
* that are already in the index; they do not replace the existing
* ones.
* @param additions The additional characters to add to the index, such as A-Z.
* @param status Error code, will be set with the reason if the
* operation fails.
* @return this, for chaining
* @draft ICU 4.8
*/
virtual AlphabeticIndex &addLabels(const UnicodeSet &additions, UErrorCode &status);
/**
* Add the index characters from a Locale to the index. The labels
* are added to those that are already in the index; they do not replace the
* existing index characters. The collation order for this index is not
* changed; it remains that of the locale that was originally specified
* when creating this Index.
*
* @param locale The locale whose index characters are to be added.
* @param status Error code, will be set with the reason if the
* operation fails.
* @return this, for chaining
* @draft ICU 4.8
*/
virtual AlphabeticIndex &addLabels(const Locale &locale, UErrorCode &status);
/**
* Destructor
*/
virtual ~AlphabeticIndex();
/**
* Get the Collator that establishes the ordering of the items in this index.
* Ownership of the collator remains with the AlphabeticIndex instance.
*
* The returned collator is a reference to the internal collator used by this
* index. It may be safely used to compare the names of items or to get
* sort keys for names. However if any settings need to be changed,
* or other non-const methods called, a cloned copy must be made first.
*
* @return The collator
* @draft ICU 4.8
*/
virtual const RuleBasedCollator &getCollator() const;
/**
* Get the default label used for abbreviated buckets <i>between</i> other index characters.
* For example, consider the labels when Latin and Greek are used:
* X Y Z ... &#x0391; &#x0392; &#x0393;.
*
* @return inflow label
* @draft ICU 4.8
*/
virtual const UnicodeString &getInflowLabel() const;
/**
* Set the default label used for abbreviated buckets <i>between</i> other index characters.
* An inflow label will be automatically inserted if two otherwise-adjacent label characters
* are from different scripts, e.g. Latin and Cyrillic, and a third script, e.g. Greek,
* sorts between the two. The default inflow character is an ellipsis (...)
*
* @param inflowLabel the new Inflow label.
* @param status Error code, will be set with the reason if the operation fails.
* @return this
* @draft ICU 4.8
*/
virtual AlphabeticIndex &setInflowLabel(const UnicodeString &inflowLabel, UErrorCode &status);
/**
* Get the special label used for items that sort after the last normal label,
* and that would not otherwise have an appropriate label.
*
* @return the overflow label
* @draft ICU 4.8
*/
virtual const UnicodeString &getOverflowLabel() const;
/**
* Set the label used for items that sort after the last normal label,
* and that would not otherwise have an appropriate label.
*
* @param overflowLabel the new overflow label.
* @param status Error code, will be set with the reason if the operation fails.
* @return this
* @draft ICU 4.8
*/
virtual AlphabeticIndex &setOverflowLabel(const UnicodeString &overflowLabel, UErrorCode &status);
/**
* Get the special label used for items that sort before the first normal label,
* and that would not otherwise have an appropriate label.
*
* @return underflow label
* @draft ICU 4.8
*/
virtual const UnicodeString &getUnderflowLabel() const;
/**
* Set the label used for items that sort before the first normal label,
* and that would not otherwise have an appropriate label.
*
* @param underflowLabel the new underflow label.
* @param status Error code, will be set with the reason if the operation fails.
* @return this
* @draft ICU 4.8
*/
virtual AlphabeticIndex &setUnderflowLabel(const UnicodeString &underflowLabel, UErrorCode &status);
/**
* Get the limit on the number of labels permitted in the index.
* The number does not include over, under and inflow labels.
*
* @return maxLabelCount maximum number of labels.
* @draft ICU 4.8
*/
virtual int32_t getMaxLabelCount() const;
/**
* Set a limit on the number of labels permitted in the index.
* The number does not include over, under and inflow labels.
* Currently, if the number is exceeded, then every
* nth item is removed to bring the count down.
* A more sophisticated mechanism may be available in the future.
*
* @param maxLabelCount the maximum number of labels.
* @return This, for chaining
* @draft ICU 4.8
*/
virtual AlphabeticIndex &setMaxLabelCount(int32_t maxLabelCount, UErrorCode &status);
/**
* Get the Unicode character (or tailored string) that defines an overflow bucket;
* that is anything greater than or equal to that string should go in that bucket,
* instead of with the last character. Normally that is the first character of the script
* after lowerLimit. Thus in X Y Z ... <i>Devanagari-ka</i>, the overflow character for Z
* would be the <i>Greek-alpha</i>.
*
* @param lowerLimit The character below the overflow (or inflow) bucket
* @return string that defines top of the overflow buck for lowerLimit, or an empty string if there is none
* @internal
*/
virtual const UnicodeString &getOverflowComparisonString(const UnicodeString &lowerLimit,
UErrorCode &status);
/**
* Add a record to the index. Each record will be associated with an index Bucket
* based on the record's name. The list of records for each bucket will be sorted
* based on the collation ordering of the names in the index's locale.
* Records with duplicate names are permitted; they will be kept in the order
* that they were added.
*
* @param name The display name for the Record. The Record will be placed in
* a bucket based on this name.
* @param data An optional pointer to user data associated with this
* item. When iterating the contents of a bucket, both the
* data pointer the name will be available for each Record.
* @param status Error code, will be set with the reason if the operation fails.
* @return This, for chaining.
* @draft ICU 4.8
*/
virtual AlphabeticIndex &addRecord(const UnicodeString &name, const void *data, UErrorCode &status);
/**
* Remove all Records from the Index. The set of Buckets, which define the headings under
* which records are classified, is not altered.
*
* @param status Error code, will be set with the reason if the operation fails.
* @return This, for chaining.
* @draft ICU 4.8
*/
virtual AlphabeticIndex &clearRecords(UErrorCode &status);
/** Get the number of labels in this index.
* Note: may trigger lazy index construction.
*
* @param status Error code, will be set with the reason if the operation fails.
* @return The number of labels in this index, including any under, over or
* in-flow labels.
* @draft ICU 4.8
*/
virtual int32_t getBucketCount(UErrorCode &status);
/** Get the total number of Records in this index, that is, the number
* of <name, data> pairs added.
*
* @param status Error code, will be set with the reason if the operation fails.
* @return The number of records in this index, that is, the total number
* of (name, data) items added with addRecord().
* @draft ICU 4.8
*/
virtual int32_t getRecordCount(UErrorCode &status);
/**
* Given the name of a record, return the zero-based index of the Bucket
* in which the item should appear. The name need not be in the index.
* A Record will not be added to the index by this function.
* Bucket numbers are zero-based, in Bucket iteration order.
*
* @param name The name whose bucket position in the index is to be determined.
* @param status Error code, will be set with the reason if the operation fails.
* @return The bucket number for this name.
* @draft ICU 4.8
*
*/
virtual int32_t getBucketIndex(const UnicodeString &itemName, UErrorCode &status);
/**
* Get the zero based index of the current Bucket from an iteration
* over the Buckets of this index. Return -1 if no iteration is in process.
* @return the index of the current Bucket
* @draft ICU 4.8
*/
virtual int32_t getBucketIndex() const;
/**
* Advance the iteration over the Buckets of this index. Return FALSE if
* there are no more Buckets.
*
* @param status Error code, will be set with the reason if the operation fails.
* U_ENUM_OUT_OF_SYNC_ERROR will be reported if the index is modified while
* an enumeration of its contents are in process.
*
* @return TRUE if success, FALSE if at end of iteration
* @draft ICU 4.8
*/
virtual UBool nextBucket(UErrorCode &status);
/**
* Return the name of the Label of the current bucket from an iteration over the buckets.
* If the iteration is before the first Bucket (nextBucket() has not been called),
* or after the last, return an empty string.
*
* @return the bucket label.
* @draft ICU 4.8
*/
virtual const UnicodeString &getBucketLabel() const;
/**
* Return the type of the label for the current Bucket (selected by the
* iteration over Buckets.)
*
* @return the label type.
* @draft ICU 4.8
*/
virtual UAlphabeticIndexLabelType getBucketLabelType() const;
/**
* Get the number of <name, data> Records in the current Bucket.
* If the current bucket iteration position is before the first label or after the
* last, return 0.
*
* @return the number of Records.
* @draft ICU 4.8
*/
virtual int32_t getBucketRecordCount() const;
/**
* Reset the Bucket iteration for this index. The next call to nextBucket()
* will restart the iteration at the first label.
*
* @param status Error code, will be set with the reason if the operation fails.
* @return this, for chaining.
* @draft ICU 4.8
*/
virtual AlphabeticIndex &resetBucketIterator(UErrorCode &status);
/**
* Advance to the next record in the current Bucket.
* When nextBucket() is called, Record iteration is reset to just before the
* first Record in the new Bucket.
*
* @param status Error code, will be set with the reason if the operation fails.
* U_ENUM_OUT_OF_SYNC_ERROR will be reported if the index is modified while
* an enumeration of its contents are in process.
* @return TRUE if successful, FALSE when the iteration advances past the last item.
* @draft ICU 4.8
*/
virtual UBool nextRecord(UErrorCode &status);
/**
* Get the name of the current Record.
* Return an empty string if the Record iteration position is before first
* or after the last.
*
* @return The name of the current index item.
* @draft ICU 4.8
*/
virtual const UnicodeString &getRecordName() const;
/**
* Return the data pointer of the Record currently being iterated over.
* Return NULL if the current iteration position before the first item in this Bucket,
* or after the last.
*
* @return The current Record's data pointer.
* @draft ICU 4.8
*/
virtual const void *getRecordData() const;
/**
* Reset the Record iterator position to before the first Record in the current Bucket.
*
* @return This, for chaining.
* @draft ICU 4.8
*/
virtual AlphabeticIndex &resetRecordIterator();
private:
// No ICU "poor man's RTTI" for this class nor its subclasses.
virtual UClassID getDynamicClassID() const;
/**
* No Copy constructor.
* @internal
*/
AlphabeticIndex(const AlphabeticIndex &other);
/**
* No assignment.
*/
AlphabeticIndex &operator =(const AlphabeticIndex & /*other*/) { return *this;};
/**
* No Equality operators.
* @internal
*/
virtual UBool operator==(const AlphabeticIndex& other) const;
/**
* Inequality operator.
* @internal
*/
virtual UBool operator!=(const AlphabeticIndex& other) const;
// Common initialization, for use from all constructors.
void init(UErrorCode &status);
// Initialize & destruct static constants used by this class.
static void staticInit(UErrorCode &status);
// Pinyin stuff. If the input name is Chinese, add the Pinyin prefix to the dest string.
void hackName(UnicodeString &dest, const UnicodeString &name, const Collator *coll);
void initPinyinBounds(const Collator *coll, UErrorCode &status);
public:
/**
* Delete all shared (static) data associated with an AlphabeticIndex.
* Internal function, not intended for direct use.
* @internal.
*/
static void staticCleanup();
private:
// Add index characters from the specified locale to the dest set.
// Does not remove any previous contents from dest.
static void getIndexExemplars(UnicodeSet &dest, const Locale &locale, UErrorCode &status);
UVector *firstStringsInScript(UErrorCode &status);
static UnicodeString separated(const UnicodeString &item);
static UnicodeSet *getScriptSet(UnicodeSet &dest, const UnicodeString &codePoint, UErrorCode &status);
void buildIndex(UErrorCode &status);
void buildBucketList(UErrorCode &status);
void bucketRecords(UErrorCode &status);
public:
// The following internal items are declared public only to allow access from
// implementation code written in plain C. They are not intended for
// public use.
/**
* A record, or item, in the index.
* @internal
*/
struct Record: public UMemory {
AlphabeticIndex *alphaIndex_;
const UnicodeString name_;
UnicodeString sortingName_; // Usually the same as name_; different for Pinyin.
const void *data_;
int32_t serialNumber_; // Defines sorting order for names that compare equal.
Record(AlphabeticIndex *alphaIndex, const UnicodeString &name, const void *data);
~Record();
};
/**
* Holds all user records before they are distributed into buckets.
* Type of contents is (Record *)
* @internal
*/
UVector *inputRecords_;
/**
* A Bucket holds an index label and references to everything belonging to that label.
* For implementation use only. Declared public because pure C implementation code needs access.
* @internal
*/
struct Bucket: public UMemory {
UnicodeString label_;
UnicodeString lowerBoundary_;
UAlphabeticIndexLabelType labelType_;
UVector *records_; // Records are owned by inputRecords_ vector.
Bucket(const UnicodeString &label, // Parameter strings are copied.
const UnicodeString &lowerBoundary,
UAlphabeticIndexLabelType type, UErrorCode &status);
~Bucket();
};
public:
/**
* Language Types. For internal ICU use only.
* @internal
*/
enum ELangType {
/** @internal */
kNormal,
/** @internal */
kSimplified,
/** @internal */
kTraditional
};
/**
* Get the Language Type for this Index. Based on the locale.
* @internal
*/
static ELangType langTypeFromLocale(const Locale &loc);
private:
// Holds the contents of this index, buckets of user items.
// UVector elements are of type (Bucket *)
UVector *bucketList_;
int32_t labelsIterIndex_; // Index of next item to return.
int32_t itemsIterIndex_;
Bucket *currentBucket_; // While an iteration of the index in underway,
// point to the bucket for the current label.
// NULL when no iteration underway.
UBool indexBuildRequired_; // Caller has made changes to the index that
// require rebuilding & bucketing before the
// contents can be iterated.
int32_t maxLabelCount_; // Limit on # of labels permitted in the index.
UHashtable *alreadyIn_; // Key=UnicodeString, value=UnicodeSet
UnicodeSet *initialLabels_; // Initial (unprocessed) set of Labels. Union
// of those explicitly set by the user plus
// those from locales. Raw values, before
// crunching into bucket labels.
UVector *labels_; // List of Labels, after processing, sorting.
// Contents are (UnicodeString *)
UnicodeSet *noDistinctSorting_; // As the set of labels is built, strings may
// be discarded from the exemplars. This contains
// some of the discards, and is
// intended for debugging.
UnicodeSet *notAlphabetic_; // As the set of labels is built, strings may
// be discarded from the exemplars. This contains
// some of the discards, and is
// intended for debugging.
UVector *firstScriptCharacters_; // The first character from each script,
// in collation order.
Locale locale_;
Collator *collator_;
Collator *collatorPrimaryOnly_;
UnicodeString inflowLabel_;
UnicodeString overflowLabel_;
UnicodeString underflowLabel_;
UnicodeString overflowComparisonString_;
ELangType langType_; // The language type, simplified Chinese, Traditional Chinese,
// or not Chinese (Normal). Part of the Pinyin support
typedef const UChar PinyinLookup[24][3];
static PinyinLookup HACK_PINYIN_LOOKUP_SHORT;
static PinyinLookup HACK_PINYIN_LOOKUP_LONG;
// These will be lazily set to the short or long tables based on which
// Chinese collation has been configured into the ICU library.
static PinyinLookup *HACK_PINYIN_LOOKUP;
static const UChar *PINYIN_LOWER_BOUNDS;
int32_t recordCounter_; // Counts Records created. For minting record serial numbers.
// Constants. Lazily initialized the first time an AlphabeticIndex object is created.
static UnicodeSet *ALPHABETIC;
static UnicodeSet *CORE_LATIN;
static UnicodeSet *ETHIOPIC;
static UnicodeSet *HANGUL;
static UnicodeSet *IGNORE_SCRIPTS;
static UnicodeSet *TO_TRY;
static UnicodeSet *UNIHAN;
static const UnicodeString *EMPTY_STRING;
};
U_NAMESPACE_END
#endif

9
icu4c/source/i18n/unicode/ulocdata.h
View file

@ -1,7 +1,7 @@
/*
******************************************************************************
* *
* Copyright (C) 2003-2010, International Business Machines *
* Copyright (C) 2003-2011, International Business Machines *
* Corporation and others. All Rights Reserved. *
* *
******************************************************************************
@ -41,7 +41,8 @@ typedef struct ULocaleData ULocaleData;
typedef enum ULocaleDataExemplarSetType {
ULOCDATA_ES_STANDARD=0, /* Basic set */
ULOCDATA_ES_AUXILIARY=1, /* Auxiliary set */
ULOCDATA_ES_COUNT=2
ULOCDATA_ES_INDEX=2, /* Index Character set */
ULOCDATA_ES_COUNT=3
} ULocaleDataExemplarSetType;
/** The possible types of delimiters.
@ -142,9 +143,11 @@ ulocdata_getNoSubstitute(ULocaleData *uld);
* always set, regardless of the value of 'options'.
* @param extype Specifies the type of exemplar set to be retrieved.
* @param status Pointer to an input-output error code value;
* must not be NULL.
* must not be NULL. Will be set to U_MISSING_RESOURCE_ERROR
* if the requested data is not available.
* @return USet* Either fillIn, or if fillIn is NULL, a pointer to
* a newly-allocated USet that the user must close.
* In case of error, NULL is returned.
* @stable ICU 3.4
*/
U_STABLE USet* U_EXPORT2

39
icu4c/source/test/cintltst/cldrtest.c
View file

@ -1234,6 +1234,43 @@ static void TestCoverage(void){
ulocdata_close(uld);
}
static void TestIndexChars(void) {
/* Very basic test of ULOCDATA_ES_INDEX.
* No comprehensive test of data, just basic check that the code path is alive.
*/
UErrorCode status = U_ZERO_ERROR;
ULocaleData *uld;
USet *exemplarChars;
USet *indexChars;
uld = ulocdata_open("en", &status);
exemplarChars = uset_openEmpty();
indexChars = uset_openEmpty();
ulocdata_getExemplarSet(uld, exemplarChars, 0, ULOCDATA_ES_STANDARD, &status);
ulocdata_getExemplarSet(uld, indexChars, 0, ULOCDATA_ES_INDEX, &status);
if (U_FAILURE(status)) {
log_err("File %s, line %d, Failure opening exemplar chars: %s", __FILE__, __LINE__, u_errorName(status));
goto close_sets;
}
/* en data, standard exemplars are [a-z], lower case. */
/* en data, index characters are [A-Z], upper case. */
if ((uset_contains(exemplarChars, (UChar32)0x41) || uset_contains(indexChars, (UChar32)0x61))) {
log_err("File %s, line %d, Exemplar characters incorrect.", __FILE__, __LINE__ );
goto close_sets;
}
if (!(uset_contains(exemplarChars, (UChar32)0x61) && uset_contains(indexChars, (UChar32)0x41) )) {
log_err("File %s, line %d, Exemplar characters incorrect.", __FILE__, __LINE__ );
goto close_sets;
}
close_sets:
uset_close(exemplarChars);
uset_close(indexChars);
ulocdata_close(uld);
}
static void TestCurrencyList(void){
#if !UCONFIG_NO_FORMATTING
UErrorCode errorCode = U_ZERO_ERROR;
@ -1284,4 +1321,6 @@ void addCLDRTest(TestNode** root)
TESTCASE(TestExemplarSet);
TESTCASE(TestLocaleDisplayPattern);
TESTCASE(TestCoverage);
TESTCASE(TestIndexChars);
}

2
icu4c/source/test/intltest/Makefile.in
View file

@ -58,7 +58,7 @@ itrbnf.o itrbnfrt.o itrbnfp.o ucaconf.o icusvtst.o \
uobjtest.o idnaref.o idnaconf.o nptrans.o punyref.o testidn.o testidna.o uts46test.o \
incaltst.o calcasts.o v32test.o uvectest.o textfile.o tokiter.o utxttest.o \
windttst.o winnmtst.o winutil.o csdetest.o tzrulets.o tzoffloc.o tzfmttst.o ssearch.o dtifmtts.o \
tufmtts.o itspoof.o simplethread.o bidiconf.o locnmtst.o dcfmtest.o
tufmtts.o itspoof.o simplethread.o bidiconf.o locnmtst.o dcfmtest.o alphaindextst.o
DEPS = $(OBJECTS:.o=.d)

417
icu4c/source/test/intltest/alphaindextst.cpp Normal file
View file

@ -0,0 +1,417 @@
/********************************************************************
* COPYRIGHT:
* Copyright (c) 2011, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
//
// file: alphaindex.cpp
// Alphabetic Index Tests.
//
#include "intltest.h"
#include "alphaindextst.h"
#include "unicode/alphaindex.h"
#include "unicode/coll.h"
#include "unicode/tblcoll.h"
#include "unicode/uniset.h"
// #include <string>
// #include <iostream>
AlphabeticIndexTest::AlphabeticIndexTest() {
}
AlphabeticIndexTest::~AlphabeticIndexTest() {
}
void AlphabeticIndexTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
{
if (exec) logln("TestSuite AlphabeticIndex: ");
switch (index) {
case 0: name = "APITest";
if (exec) APITest();
break;
case 1: name = "ManyLocales";
if (exec) ManyLocalesTest();
break;
case 2: name = "HackPinyinTest";
if (exec) HackPinyinTest();
break;
default: name = "";
break; //needed to end loop
}
}
#define TEST_CHECK_STATUS {if (U_FAILURE(status)) {dataerrln("%s:%d: Test failure. status=%s", \
__FILE__, __LINE__, u_errorName(status)); return;}}
#define TEST_ASSERT(expr) {if ((expr)==FALSE) {errln("%s:%d: Test failure \n", __FILE__, __LINE__);};}
//
// APITest. Invoke every function at least once, and check that it does something.
// Does not attempt to check complete functionality.
//
void AlphabeticIndexTest::APITest() {
//
// Simple constructor and destructor, getBucketCount()
//
UErrorCode status = U_ZERO_ERROR;
int32_t lc = 0;
int32_t i = 0;
AlphabeticIndex *index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
lc = index->getBucketCount(status);
TEST_CHECK_STATUS;
TEST_ASSERT(28 == lc); // 26 letters plus two under/overflow labels.
//printf("getBucketCount() == %d\n", lc);
delete index;
// addLabels()
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
UnicodeSet additions;
additions.add((UChar32)0x410).add((UChar32)0x415); // A couple of Cyrillic letters
index->addLabels(additions, status);
TEST_CHECK_STATUS;
lc = index->getBucketCount(status);
TEST_CHECK_STATUS;
// TODO: should get 31. Java also gives 30. Needs fixing
TEST_ASSERT(30 == lc); // 26 Latin letters plus
// TEST_ASSERT(31 == lc); // 26 Latin letters plus
// 2 Cyrillic letters plus
// 1 inflow label plus
// two under/overflow labels.
// std::cout << lc << std::endl;
delete index;
// addLabels(Locale)
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
AlphabeticIndex &aip = index->addLabels(Locale::getJapanese(), status);
TEST_ASSERT(&aip == index);
TEST_CHECK_STATUS;
lc = index->getBucketCount(status);
TEST_CHECK_STATUS;
TEST_ASSERT(35 < lc); // Japanese should add a bunch. Don't rely on the exact value.
delete index;
// GetCollator(), Get under/in/over flow labels
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getGerman(), status);
TEST_CHECK_STATUS;
Collator *germanCol = Collator::createInstance(Locale::getGerman(), status);
TEST_CHECK_STATUS;
const RuleBasedCollator &indexCol = index->getCollator();
TEST_ASSERT(*germanCol == indexCol);
delete germanCol;
UnicodeString ELLIPSIS; ELLIPSIS.append((UChar32)0x2026);
UnicodeString s = index->getUnderflowLabel();
TEST_ASSERT(ELLIPSIS == s);
s = index->getOverflowLabel();
TEST_ASSERT(ELLIPSIS == s);
s = index->getInflowLabel();
TEST_ASSERT(ELLIPSIS == s);
index->setOverflowLabel(UNICODE_STRING_SIMPLE("O"), status);
index->setUnderflowLabel(UNICODE_STRING_SIMPLE("U"), status).setInflowLabel(UNICODE_STRING_SIMPLE("I"), status);
s = index->getUnderflowLabel();
TEST_ASSERT(UNICODE_STRING_SIMPLE("U") == s);
s = index->getOverflowLabel();
TEST_ASSERT(UNICODE_STRING_SIMPLE("O") == s);
s = index->getInflowLabel();
TEST_ASSERT(UNICODE_STRING_SIMPLE("I") == s);
delete index;
const UnicodeString adam = UNICODE_STRING_SIMPLE("Adam");
const UnicodeString baker = UNICODE_STRING_SIMPLE("Baker");
const UnicodeString charlie = UNICODE_STRING_SIMPLE("Charlie");
const UnicodeString chad = UNICODE_STRING_SIMPLE("Chad");
const UnicodeString zed = UNICODE_STRING_SIMPLE("Zed");
const UnicodeString Cyrillic = UNICODE_STRING_SIMPLE("\\u0410\\u0443\\u0435").unescape();
// addRecord(), verify that it comes back out.
//
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
index->addRecord(UnicodeString("Adam"), this, status);
UBool b;
TEST_CHECK_STATUS;
index->resetBucketIterator(status);
TEST_CHECK_STATUS;
index->nextBucket(status); // Move to underflow label
index->nextBucket(status); // Move to "A"
TEST_CHECK_STATUS;
const UnicodeString &label2 = index->getBucketLabel();
UnicodeString A_STR = UNICODE_STRING_SIMPLE("A");
TEST_ASSERT(A_STR == label2);
b = index->nextRecord(status);
TEST_CHECK_STATUS;
TEST_ASSERT(b);
const UnicodeString &itemName = index->getRecordName();
TEST_ASSERT(adam == itemName);
const void *itemContext = index->getRecordData();
TEST_ASSERT(itemContext == this);
delete index;
// clearRecords, addRecord(), Iteration
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
while (index->nextBucket(status)) {
TEST_CHECK_STATUS;
while (index->nextRecord(status)) {
TEST_CHECK_STATUS;
TEST_ASSERT(FALSE); // No items have been added.
}
TEST_CHECK_STATUS;
}
index->addRecord(adam, NULL, status);
index->addRecord(baker, NULL, status);
index->addRecord(charlie, NULL, status);
index->addRecord(chad, NULL, status);
TEST_CHECK_STATUS;
int itemCount = 0;
index->resetBucketIterator(status);
while (index->nextBucket(status)) {
TEST_CHECK_STATUS;
while (index->nextRecord(status)) {
TEST_CHECK_STATUS;
++itemCount;
}
}
TEST_CHECK_STATUS;
TEST_ASSERT(itemCount == 4);
TEST_ASSERT(index->nextBucket(status) == FALSE);
index->resetBucketIterator(status);
TEST_CHECK_STATUS;
TEST_ASSERT(index->nextBucket(status) == TRUE);
index->clearRecords(status);
TEST_CHECK_STATUS;
index->resetBucketIterator(status);
while (index->nextBucket(status)) {
TEST_CHECK_STATUS;
while (index->nextRecord(status)) {
TEST_ASSERT(FALSE); // No items have been added.
}
}
TEST_CHECK_STATUS;
delete index;
// getBucketLabel(), getBucketType()
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
index->setUnderflowLabel(adam, status).setOverflowLabel(charlie, status);
TEST_CHECK_STATUS;
for (i=0; index->nextBucket(status); i++) {
TEST_CHECK_STATUS;
UnicodeString label = index->getBucketLabel();
UAlphabeticIndexLabelType type = index->getBucketLabelType();
if (i == 0) {
TEST_ASSERT(type == U_ALPHAINDEX_UNDERFLOW);
TEST_ASSERT(label == adam);
} else if (i <= 26) {
// Labels A - Z for English locale
TEST_ASSERT(type == U_ALPHAINDEX_NORMAL);
UnicodeString expectedLabel((UChar)(0x40 + i));
TEST_ASSERT(expectedLabel == label);
} else if (i == 27) {
TEST_ASSERT(type == U_ALPHAINDEX_OVERFLOW);
TEST_ASSERT(label == charlie);
} else {
TEST_ASSERT(FALSE);
}
}
TEST_ASSERT(i==28);
delete index;
// getBucketIndex()
status = U_ZERO_ERROR;
index = new AlphabeticIndex(Locale::getEnglish(), status);
TEST_CHECK_STATUS;
int32_t n = index->getBucketIndex(adam, status);
TEST_CHECK_STATUS;
TEST_ASSERT(n == 1); /* Label #0 is underflow, 1 is A, etc. */
n = index->getBucketIndex(baker, status);
TEST_ASSERT(n == 2);
n = index->getBucketIndex(Cyrillic, status);
TEST_ASSERT(n == 27); // Overflow label
n = index->getBucketIndex(zed, status);
TEST_ASSERT(n == 26);
for (i=0; index->nextBucket(status); i++) {
n = index->getBucketIndex();
TEST_ASSERT(n == i);
UnicodeString label = index->getBucketLabel();
TEST_ASSERT(n == i);
}
TEST_ASSERT(i == 28);
delete index;
index = new AlphabeticIndex(Locale::createFromName("ru"), status);
//Locale loc = Locale::createFromName(localeName);
TEST_CHECK_STATUS;
n = index->getBucketIndex(adam, status);
TEST_CHECK_STATUS;
TEST_ASSERT(n == 0); // Label #0 is underflow
n = index->getBucketIndex(baker, status);
TEST_ASSERT(n == 0);
n = index->getBucketIndex(Cyrillic, status);
TEST_ASSERT(n == 1); // Overflow label
n = index->getBucketIndex(zed, status);
TEST_ASSERT(n == 0);
delete index;
}
static const char * KEY_LOCALES[] = {
"en", "es", "de", "fr", "ja", "it", "tr", "pt", "zh", "nl",
"pl", "ar", "ru", "zh_Hant", "ko", "th", "sv", "fi", "da",
"he", "nb", "el", "hr", "bg", "sk", "lt", "vi", "lv", "sr",
"pt_PT", "ro", "hu", "cs", "id", "sl", "fil", "fa", "uk",
"ca", "hi", "et", "eu", "is", "sw", "ms", "bn", "am", "ta",
"te", "mr", "ur", "ml", "kn", "gu", "or", ""};
void AlphabeticIndexTest::ManyLocalesTest() {
UErrorCode status = U_ZERO_ERROR;
int32_t lc = 0;
AlphabeticIndex *index = NULL;
for (int i=0; ; ++i) {
status = U_ZERO_ERROR;
const char *localeName = KEY_LOCALES[i];
if (localeName[0] == 0) {
break;
}
// std::cout << localeName << " ";
Locale loc = Locale::createFromName(localeName);
index = new AlphabeticIndex(loc, status);
TEST_CHECK_STATUS;
lc = index->getBucketCount(status);
TEST_CHECK_STATUS;
// std::cout << "getBucketCount() == " << lc << std::endl;
while (index->nextBucket(status)) {
TEST_CHECK_STATUS;
const UnicodeString &label = index->getBucketLabel();
TEST_ASSERT(label.length()>0);
// std::string ss;
// std::cout << ":" << label.toUTF8String(ss);
}
// std::cout << ":" << std::endl;
delete index;
}
}
// Test data for Pinyin based indexes.
// The Chinese characters should be distributed under latin labels in
// an index.
static const char *pinyinTestData[] = {
"\\u0101", "\\u5416", "\\u58ba", //
"b", "\\u516b", "\\u62d4", "\\u8500", //
"c", "\\u5693", "\\u7938", "\\u9e7e", //
"d", "\\u5491", "\\u8fcf", "\\u964a", //
"\\u0113","\\u59b8", "\\u92e8", "\\u834b", //
"f", "\\u53d1", "\\u9197", "\\u99a5", //
"g", "\\u7324", "\\u91d3", "\\u8142", //
"h", "\\u598e", "\\u927f", "\\u593b", //
"j", "\\u4e0c", "\\u6785", "\\u9d58", //
"k", "\\u5494", "\\u958b", "\\u7a52", //
"l", "\\u5783", "\\u62c9", "\\u9ba5", //
"m", "\\u5638", "\\u9ebb", "\\u65c0", //
"n", "\\u62ff", "\\u80ad", "\\u685b", //
"\\u014D", "\\u5662", "\\u6bee", "\\u8bb4", //
"p", "\\u5991", "\\u8019", "\\u8c31", //
"q", "\\u4e03", "\\u6053", "\\u7f56", //
"r", "\\u5465", "\\u72aa", "\\u6e03", //
"s", "\\u4ee8", "\\u9491", "\\u93c1", //
"t", "\\u4ed6", "\\u9248", "\\u67dd", //
"w", "\\u5c72", "\\u5558", "\\u5a7a", //
"x", "\\u5915", "\\u5438", "\\u6bbe", //
"y", "\\u4e2b", "\\u82bd", "\\u8574", //
"z", "\\u5e00", "\\u707d", "\\u5c0a",
NULL
};
void AlphabeticIndexTest::HackPinyinTest() {
UErrorCode status = U_ZERO_ERROR;
AlphabeticIndex aindex(Locale::createFromName("zh"), status);
TEST_CHECK_STATUS;
UnicodeString names[sizeof(pinyinTestData) / sizeof(pinyinTestData[0])];
int32_t nameCount;
for (nameCount=0; pinyinTestData[nameCount] != NULL; nameCount++) {
names[nameCount] = UnicodeString(pinyinTestData[nameCount], -1, UnicodeString::kInvariant).unescape();
aindex.addRecord(names[nameCount], &names[nameCount], status);
TEST_CHECK_STATUS;
if (U_FAILURE(status)) {
return;
}
}
TEST_ASSERT(nameCount == aindex.getRecordCount(status));
// Weak checking: make sure that none of the Chinese names landed in the overflow bucket
// of the index, and that the names are distributed among several buckets.
// (Exact expected data would be subject to change with evolution of the collation rules.)
int32_t bucketCount = 0;
int32_t filledBucketCount = 0;
while (aindex.nextBucket(status)) {
bucketCount++;
UnicodeString label = aindex.getBucketLabel();
// std::string s;
// std::cout << label.toUTF8String(s) << ": ";
UBool bucketHasContents = FALSE;
while (aindex.nextRecord(status)) {
bucketHasContents = TRUE;
UnicodeString name = aindex.getRecordName();
if (aindex.getBucketLabelType() != U_ALPHAINDEX_NORMAL) {
errln("File %s, Line %d, Name \"\\u%x\" is in an under or overflow bucket.",
__FILE__, __LINE__, name.char32At(0));
}
// s.clear();
// std::cout << aindex.getRecordName().toUTF8String(s) << " ";
}
if (bucketHasContents) {
filledBucketCount++;
}
// std::cout << std::endl;
}
TEST_ASSERT(bucketCount > 25);
TEST_ASSERT(filledBucketCount > 15);
}

28
icu4c/source/test/intltest/alphaindextst.h Normal file
View file

@ -0,0 +1,28 @@
/********************************************************************
* COPYRIGHT:
* Copyright (c) 2011, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
//
// file: alphaindex.h
// Alphabetic Index Tests.
//
#ifndef ALPHAINDEXTST_H
#define ALPHAINDEXTST_H
#include "intltest.h"
class AlphabeticIndexTest: public IntlTest {
public:
AlphabeticIndexTest();
virtual ~AlphabeticIndexTest();
virtual void runIndexedTest(int32_t index, UBool exec, const char* &name, char* par = NULL );
virtual void APITest();
virtual void ManyLocalesTest();
virtual void HackPinyinTest();
};
#endif

4
icu4c/source/test/intltest/tscoll.cpp
View file

@ -1,6 +1,6 @@
/********************************************************************
* COPYRIGHT:
* Copyright (c) 1997-2009, International Business Machines Corporation and
* Copyright (c) 1997-2011, International Business Machines Corporation and
* others. All Rights Reserved.
********************************************************************/
@ -49,6 +49,7 @@
#include "ucaconf.h"
#include "svccoll.h"
#include "cmemory.h"
#include "alphaindextst.h"
//#include "rndmcoll.h"
// Set to 1 to test offsets in backAndForth()
@ -97,6 +98,7 @@ void IntlTestCollator::runIndexedTest( int32_t index, UBool exec, const char* &n
TESTCLASS(20, CollationFinnishTest); // removed by weiv - we have changed Finnish collation
//TESTCLASS(21, RandomCollatorTest); // See ticket 5747 about reenabling this test.
TESTCLASS(21, SSearchTest);
TESTCLASS(22, AlphabeticIndexTest);
default: name = ""; break;
}