diff --git a/icu4c/source/i18n/plurrule.cpp b/icu4c/source/i18n/plurrule.cpp index 7d1037f8bdd..edc3d883027 100644 --- a/icu4c/source/i18n/plurrule.cpp +++ b/icu4c/source/i18n/plurrule.cpp @@ -26,6 +26,7 @@ #include "hash.h" #include "locutil.h" #include "mutex.h" +#include "number_decnum.h" #include "patternprops.h" #include "plurrule_impl.h" #include "putilimp.h" @@ -45,7 +46,9 @@ U_NAMESPACE_BEGIN using namespace icu::pluralimpl; +using icu::number::impl::DecNum; using icu::number::impl::DecimalQuantity; +using icu::number::impl::RoundingMode; static const UChar PLURAL_KEYWORD_OTHER[]={LOW_O,LOW_T,LOW_H,LOW_E,LOW_R,0}; static const UChar PLURAL_DEFAULT_RULE[]={LOW_O,LOW_T,LOW_H,LOW_E,LOW_R,COLON,SPACE,LOW_N,0}; @@ -369,36 +372,18 @@ PluralRules::getAllKeywordValues(const UnicodeString & /* keyword */, double * / return 0; } - -static double scaleForInt(double d) { - double scale = 1.0; - while (d != floor(d)) { - d = d * 10.0; - scale = scale * 10.0; - } - return scale; -} - -static const double powers10[7] = {1.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0}; // powers of 10 for 0..6 -static double applyExponent(double source, int32_t exponent) { - if (exponent >= 0 && exponent <= 6) { - return source * powers10[exponent]; - } - return source * pow(10.0, exponent); -} - /** - * Helper method for the overrides of getSamples() for double and FixedDecimal - * return value types. Provide only one of an allocated array of doubles or - * FixedDecimals, and a nullptr for the other. + * Helper method for the overrides of getSamples() for double and DecimalQuantity + * return value types. Provide only one of an allocated array of double or + * DecimalQuantity, and a nullptr for the other. */ static int32_t getSamplesFromString(const UnicodeString &samples, double *destDbl, - FixedDecimal* destFd, int32_t destCapacity, + DecimalQuantity* destDq, int32_t destCapacity, UErrorCode& status) { - if ((destDbl == nullptr && destFd == nullptr) - || (destDbl != nullptr && destFd != nullptr)) { + if ((destDbl == nullptr && destDq == nullptr) + || (destDbl != nullptr && destDq != nullptr)) { status = U_INTERNAL_PROGRAM_ERROR; return 0; } @@ -420,58 +405,75 @@ getSamplesFromString(const UnicodeString &samples, double *destDbl, // std::cout << "PluralRules::getSamples(), samplesRange = \"" << sampleRange.toUTF8String(ss) << "\"\n"; int32_t tildeIndex = sampleRange.indexOf(TILDE); if (tildeIndex < 0) { - FixedDecimal fixed(sampleRange, status); + DecimalQuantity dq = DecimalQuantity::fromExponentString(sampleRange, status); if (isDouble) { - double sampleValue = fixed.source; - if (fixed.visibleDecimalDigitCount == 0 || sampleValue != floor(sampleValue)) { - destDbl[sampleCount++] = applyExponent(sampleValue, fixed.exponent); + // See warning note below about lack of precision for floating point samples for numbers with + // trailing zeroes in the decimal fraction representation. + double dblValue = dq.toDouble(); + if (!(dblValue == floor(dblValue) && dq.fractionCount() > 0)) { + destDbl[sampleCount++] = dblValue; } } else { - destFd[sampleCount++] = fixed; + destDq[sampleCount++] = dq; } } else { - FixedDecimal fixedLo(sampleRange.tempSubStringBetween(0, tildeIndex), status); - FixedDecimal fixedHi(sampleRange.tempSubStringBetween(tildeIndex+1), status); - double rangeLo = fixedLo.source; - double rangeHi = fixedHi.source; + DecimalQuantity rangeLo = + DecimalQuantity::fromExponentString(sampleRange.tempSubStringBetween(0, tildeIndex), status); + DecimalQuantity rangeHi = DecimalQuantity::fromExponentString(sampleRange.tempSubStringBetween(tildeIndex+1), status); if (U_FAILURE(status)) { break; } - if (rangeHi < rangeLo) { + if (rangeHi.toDouble() < rangeLo.toDouble()) { status = U_INVALID_FORMAT_ERROR; break; } - // For ranges of samples with fraction decimal digits, scale the number up so that we - // are adding one in the units place. Avoids roundoffs from repetitive adds of tenths. + DecimalQuantity incrementDq; + incrementDq.setToInt(1); + int32_t lowerDispMag = rangeLo.getLowerDisplayMagnitude(); + int32_t exponent = rangeLo.getExponent(); + int32_t incrementScale = lowerDispMag + exponent; + incrementDq.adjustMagnitude(incrementScale); + double incrementVal = incrementDq.toDouble(); // 10 ^ incrementScale + - double scale = scaleForInt(rangeLo); - double t = scaleForInt(rangeHi); - if (t > scale) { - scale = t; - } - rangeLo *= scale; - rangeHi *= scale; - for (double n=rangeLo; n<=rangeHi; n+=1) { - double sampleValue = n/scale; + DecimalQuantity dq(rangeLo); + double dblValue = dq.toDouble(); + double end = rangeHi.toDouble(); + + while (dblValue <= end) { if (isDouble) { // Hack Alert: don't return any decimal samples with integer values that // originated from a format with trailing decimals. // This API is returning doubles, which can't distinguish having displayed // zeros to the right of the decimal. // This results in test failures with values mapping back to a different keyword. - if (!(sampleValue == floor(sampleValue) && fixedLo.visibleDecimalDigitCount > 0)) { - destDbl[sampleCount++] = sampleValue; + if (!(dblValue == floor(dblValue) && dq.fractionCount() > 0)) { + destDbl[sampleCount++] = dblValue; } } else { - int32_t v = (int32_t) fixedLo.getPluralOperand(PluralOperand::PLURAL_OPERAND_V); - int32_t e = (int32_t) fixedLo.getPluralOperand(PluralOperand::PLURAL_OPERAND_E); - FixedDecimal newSample = FixedDecimal::createWithExponent(sampleValue, v, e); - destFd[sampleCount++] = newSample; + destDq[sampleCount++] = dq; } if (sampleCount >= destCapacity) { break; } + + // Increment dq for next iteration + + // Because DecNum and DecimalQuantity do not support + // add operations, we need to convert to/from double, + // despite precision lossiness for decimal fractions like 0.1. + dblValue += incrementVal; + DecNum newDqDecNum; + newDqDecNum.setTo(dblValue, status); + DecimalQuantity newDq; + newDq.setToDecNum(newDqDecNum, status); + newDq.setMinFraction(-lowerDispMag); + newDq.roundToMagnitude(lowerDispMag, RoundingMode::UNUM_ROUND_HALFEVEN, status); + newDq.adjustMagnitude(-exponent); + newDq.adjustExponent(exponent); + dblValue = newDq.toDouble(); + dq = newDq; } } sampleStartIdx = sampleEndIdx + 1; @@ -505,7 +507,7 @@ PluralRules::getSamples(const UnicodeString &keyword, double *dest, } int32_t -PluralRules::getSamples(const UnicodeString &keyword, FixedDecimal *dest, +PluralRules::getSamples(const UnicodeString &keyword, DecimalQuantity *dest, int32_t destCapacity, UErrorCode& status) { if (U_FAILURE(status)) { return 0; diff --git a/icu4c/source/i18n/plurrule_impl.h b/icu4c/source/i18n/plurrule_impl.h index 7274da58f06..c27b655fcde 100644 --- a/icu4c/source/i18n/plurrule_impl.h +++ b/icu4c/source/i18n/plurrule_impl.h @@ -34,7 +34,7 @@ * A FixedDecimal version of UPLRULES_NO_UNIQUE_VALUE used in PluralRulesTest * for parsing of samples. */ -#define UPLRULES_NO_UNIQUE_VALUE_DECIMAL (FixedDecimal((double)-0.00123456777)) +#define UPLRULES_NO_UNIQUE_VALUE_DECIMAL(ERROR_CODE) (DecimalQuantity::fromExponentString(u"-0.00123456777", ERROR_CODE)) class PluralRulesTest; diff --git a/icu4c/source/i18n/unicode/plurrule.h b/icu4c/source/i18n/unicode/plurrule.h index e3822dd829e..b4298de63f9 100644 --- a/icu4c/source/i18n/unicode/plurrule.h +++ b/icu4c/source/i18n/unicode/plurrule.h @@ -59,9 +59,15 @@ class FormattedNumber; class FormattedNumberRange; namespace impl { class UFormattedNumberRangeData; +class DecimalQuantity; +class DecNum; } } +#ifndef U_HIDE_INTERNAL_API +using icu::number::impl::DecimalQuantity; +#endif /* U_HIDE_INTERNAL_API */ + /** * Defines rules for mapping non-negative numeric values onto a small set of * keywords. Rules are constructed from a text description, consisting @@ -468,7 +474,7 @@ public: #ifndef U_HIDE_INTERNAL_API /** - * Internal-only function that returns FixedDecimals instead of doubles. + * Internal-only function that returns DecimalQuantitys instead of doubles. * * Returns sample values for which select() would return the keyword. If * the keyword is unknown, returns no values, but this is not an error. @@ -488,7 +494,7 @@ public: * @internal */ int32_t getSamples(const UnicodeString &keyword, - FixedDecimal *dest, int32_t destCapacity, + DecimalQuantity *dest, int32_t destCapacity, UErrorCode& status); #endif /* U_HIDE_INTERNAL_API */ diff --git a/icu4c/source/test/intltest/plurults.cpp b/icu4c/source/test/intltest/plurults.cpp index 54cc77a0c9e..8de9745ade9 100644 --- a/icu4c/source/test/intltest/plurults.cpp +++ b/icu4c/source/test/intltest/plurults.cpp @@ -50,7 +50,9 @@ void PluralRulesTest::runIndexedTest( int32_t index, UBool exec, const char* &na TESTCASE_AUTO(testAPI); // TESTCASE_AUTO(testGetUniqueKeywordValue); TESTCASE_AUTO(testGetSamples); - TESTCASE_AUTO(testGetFixedDecimalSamples); + TESTCASE_AUTO(testGetDecimalQuantitySamples); + TESTCASE_AUTO(testGetOrAddSamplesFromString); + TESTCASE_AUTO(testGetOrAddSamplesFromStringCompactNotation); TESTCASE_AUTO(testSamplesWithExponent); TESTCASE_AUTO(testSamplesWithCompactNotation); TESTCASE_AUTO(testWithin); @@ -396,9 +398,16 @@ void PluralRulesTest::testGetUniqueKeywordValue() { assertRuleKeyValue("a: n is 1", "other", UPLRULES_NO_UNIQUE_VALUE); // key matches default rule } +/** + * Using the double API for getting plural samples, assert all samples match the keyword + * they are listed under, for all locales. + * + * Specifically, iterate over all locales, get plural rules for the locale, iterate over every rule, + * then iterate over every sample in the rule, parse sample to a number (double), use that number + * as an input to .select() for the rules object, and assert the actual return plural keyword matches + * what we expect based on the plural rule string. + */ void PluralRulesTest::testGetSamples() { - // TODO: fix samples, re-enable this test. - // no get functional equivalent API in ICU4C, so just // test every locale... UErrorCode status = U_ZERO_ERROR; @@ -457,21 +466,24 @@ void PluralRulesTest::testGetSamples() { } } -void PluralRulesTest::testGetFixedDecimalSamples() { - // TODO: fix samples, re-enable this test. - +/** + * Using the DecimalQuantity API for getting plural samples, assert all samples match the keyword + * they are listed under, for all locales. + * + * Specifically, iterate over all locales, get plural rules for the locale, iterate over every rule, + * then iterate over every sample in the rule, parse sample to a number (DecimalQuantity), use that number + * as an input to .select() for the rules object, and assert the actual return plural keyword matches + * what we expect based on the plural rule string. + */ +void PluralRulesTest::testGetDecimalQuantitySamples() { // no get functional equivalent API in ICU4C, so just // test every locale... UErrorCode status = U_ZERO_ERROR; int32_t numLocales; const Locale* locales = Locale::getAvailableLocales(numLocales); - FixedDecimal values[1000]; + DecimalQuantity values[1000]; for (int32_t i = 0; U_SUCCESS(status) && i < numLocales; ++i) { - //if (uprv_strcmp(locales[i].getLanguage(), "fr") == 0 && - // logKnownIssue("21322", "PluralRules::getSamples cannot distinguish 1e5 from 100000")) { - // continue; - //} LocalPointer rules(PluralRules::forLocale(locales[i], status)); if (U_FAILURE(status)) { break; @@ -501,21 +513,24 @@ void PluralRulesTest::testGetFixedDecimalSamples() { count = UPRV_LENGTHOF(values); } for (int32_t j = 0; j < count; ++j) { - if (values[j] == UPLRULES_NO_UNIQUE_VALUE_DECIMAL) { + if (values[j] == UPLRULES_NO_UNIQUE_VALUE_DECIMAL(status)) { errln("got 'no unique value' among values"); } else { + if (U_FAILURE(status)){ + errln(UnicodeString(u"getSamples() failed for sample ") + + values[j].toExponentString() + + UnicodeString(u", keyword ") + *keyword); + continue; + } UnicodeString resultKeyword = rules->select(values[j]); // if (strcmp(locales[i].getName(), "uk") == 0) { // Debug only. // std::cout << " uk " << US(resultKeyword).cstr() << " " << values[j] << std::endl; // } if (*keyword != resultKeyword) { - if (values[j].exponent == 0 || !logKnownIssue("21714", "PluralRules::select treats 1c6 as 1")) { - UnicodeString valueString(values[j].toString()); - char valueBuf[16]; - valueString.extract(0, valueString.length(), valueBuf, sizeof(valueBuf)); - errln("file %s, line %d, Locale %s, sample for keyword \"%s\": %s, select(%s) returns keyword \"%s\"", - __FILE__, __LINE__, locales[i].getName(), US(*keyword).cstr(), valueBuf, valueBuf, US(resultKeyword).cstr()); - } + errln("file %s, line %d, Locale %s, sample for keyword \"%s\": %s, select(%s) returns keyword \"%s\"", + __FILE__, __LINE__, locales[i].getName(), US(*keyword).cstr(), + US(values[j].toExponentString()).cstr(), US(values[j].toExponentString()).cstr(), + US(resultKeyword).cstr()); } } } @@ -523,6 +538,102 @@ void PluralRulesTest::testGetFixedDecimalSamples() { } } +/** + * Test addSamples (Java) / getSamplesFromString (C++) to ensure the expansion of plural rule sample range + * expands to a sequence of sample numbers that is incremented as the right scale. + * + * Do this for numbers with fractional digits but no exponent. + */ +void PluralRulesTest::testGetOrAddSamplesFromString() { + UErrorCode status = U_ZERO_ERROR; + UnicodeString description(u"testkeyword: e != 0 @decimal 2.0c6~4.0c6, …"); + LocalPointer rules(PluralRules::createRules(description, status)); + if (U_FAILURE(status)) { + errln("Couldn't create plural rules from a string, with error = %s", u_errorName(status)); + return; + } + + LocalPointer keywords(rules->getKeywords(status)); + if (U_FAILURE(status)) { + errln("Couldn't get keywords from a parsed rules object, with error = %s", u_errorName(status)); + return; + } + + DecimalQuantity values[1000]; + const UnicodeString keyword(u"testkeyword"); + int32_t count = rules->getSamples(keyword, values, UPRV_LENGTHOF(values), status); + if (U_FAILURE(status)) { + errln(UnicodeString(u"getSamples() failed for plural rule keyword ") + keyword); + return; + } + + UnicodeString expDqStrs[] = { + u"2.0c6", u"2.1c6", u"2.2c6", u"2.3c6", u"2.4c6", u"2.5c6", u"2.6c6", u"2.7c6", u"2.8c6", u"2.9c6", + u"3.0c6", u"3.1c6", u"3.2c6", u"3.3c6", u"3.4c6", u"3.5c6", u"3.6c6", u"3.7c6", u"3.8c6", u"3.9c6", + u"4.0c6" + }; + assertEquals(u"Number of parsed samples from test string incorrect", 21, count); + for (int i = 0; i < count; i++) { + UnicodeString expDqStr = expDqStrs[i]; + DecimalQuantity sample = values[i]; + UnicodeString sampleStr = sample.toExponentString(); + + assertEquals(u"Expansion of sample range to sequence of sample values should increment at the right scale", + expDqStr, sampleStr); + } +} + +/** + * Test addSamples (Java) / getSamplesFromString (C++) to ensure the expansion of plural rule sample range + * expands to a sequence of sample numbers that is incremented as the right scale. + * + * Do this for numbers written in a notation that has an exponent, for which the number is an + * integer (also as defined in the UTS 35 spec for the plural operands) but whose representation + * has fractional digits in the significand written before the exponent. + */ +void PluralRulesTest::testGetOrAddSamplesFromStringCompactNotation() { + UErrorCode status = U_ZERO_ERROR; + UnicodeString description(u"testkeyword: e != 0 @decimal 2.0~4.0, …"); + LocalPointer rules(PluralRules::createRules(description, status)); + if (U_FAILURE(status)) { + errln("Couldn't create plural rules from a string, with error = %s", u_errorName(status)); + return; + } + + LocalPointer keywords(rules->getKeywords(status)); + if (U_FAILURE(status)) { + errln("Couldn't get keywords from a parsed rules object, with error = %s", u_errorName(status)); + return; + } + + DecimalQuantity values[1000]; + const UnicodeString keyword(u"testkeyword"); + int32_t count = rules->getSamples(keyword, values, UPRV_LENGTHOF(values), status); + if (U_FAILURE(status)) { + errln(UnicodeString(u"getSamples() failed for plural rule keyword ") + keyword); + return; + } + + UnicodeString expDqStrs[] = { + u"2.0", u"2.1", u"2.2", u"2.3", u"2.4", u"2.5", u"2.6", u"2.7", u"2.8", u"2.9", + u"3.0", u"3.1", u"3.2", u"3.3", u"3.4", u"3.5", u"3.6", u"3.7", u"3.8", u"3.9", + u"4.0" + }; + assertEquals(u"Number of parsed samples from test string incorrect", 21, count); + for (int i = 0; i < count; i++) { + UnicodeString expDqStr = expDqStrs[i]; + DecimalQuantity sample = values[i]; + UnicodeString sampleStr = sample.toExponentString(); + + assertEquals(u"Expansion of sample range to sequence of sample values should increment at the right scale", + expDqStr, sampleStr); + } +} + +/** + * This test is for the support of X.YeZ scientific notation of numbers in + * the plural sample string. + */ void PluralRulesTest::testSamplesWithExponent() { // integer samples UErrorCode status = U_ZERO_ERROR; @@ -538,9 +649,9 @@ void PluralRulesTest::testSamplesWithExponent() { errln("Couldn't create plural rules from a string using exponent notation, with error = %s", u_errorName(status)); return; } - checkNewSamples(description, test, u"one", u"@integer 0, 1, 1e5", FixedDecimal(0)); - checkNewSamples(description, test, u"many", u"@integer 1000000, 2e6, 3e6, 4e6, 5e6, 6e6, 7e6, …", FixedDecimal(1000000)); - checkNewSamples(description, test, u"other", u"@integer 2~17, 100, 1000, 10000, 100000, 2e5, 3e5, 4e5, 5e5, 6e5, 7e5, …", FixedDecimal(2)); + checkNewSamples(description, test, u"one", u"@integer 0, 1, 1e5", DecimalQuantity::fromExponentString(u"0", status)); + checkNewSamples(description, test, u"many", u"@integer 1000000, 2e6, 3e6, 4e6, 5e6, 6e6, 7e6, …", DecimalQuantity::fromExponentString(u"1000000", status)); + checkNewSamples(description, test, u"other", u"@integer 2~17, 100, 1000, 10000, 100000, 2e5, 3e5, 4e5, 5e5, 6e5, 7e5, …", DecimalQuantity::fromExponentString(u"2", status)); // decimal samples status = U_ZERO_ERROR; @@ -555,12 +666,15 @@ void PluralRulesTest::testSamplesWithExponent() { errln("Couldn't create plural rules from a string using exponent notation, with error = %s", u_errorName(status)); return; } - checkNewSamples(description2, test2, u"one", u"@decimal 0.0~1.5, 1.1e5", FixedDecimal(0, 1)); - checkNewSamples(description2, test2, u"many", u"@decimal 2.1e6, 3.1e6, 4.1e6, 5.1e6, 6.1e6, 7.1e6, …", FixedDecimal::createWithExponent(2.1, 1, 6)); - checkNewSamples(description2, test2, u"other", u"@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1e5, 3.1e5, 4.1e5, 5.1e5, 6.1e5, 7.1e5, …", FixedDecimal(2.0, 1)); + checkNewSamples(description2, test2, u"one", u"@decimal 0.0~1.5, 1.1e5", DecimalQuantity::fromExponentString(u"0.0", status)); + checkNewSamples(description2, test2, u"many", u"@decimal 2.1e6, 3.1e6, 4.1e6, 5.1e6, 6.1e6, 7.1e6, …", DecimalQuantity::fromExponentString(u"2.1c6", status)); + checkNewSamples(description2, test2, u"other", u"@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1e5, 3.1e5, 4.1e5, 5.1e5, 6.1e5, 7.1e5, …", DecimalQuantity::fromExponentString(u"2.0", status)); } - +/** + * This test is for the support of X.YcZ compact notation of numbers in + * the plural sample string. + */ void PluralRulesTest::testSamplesWithCompactNotation() { // integer samples UErrorCode status = U_ZERO_ERROR; @@ -576,9 +690,9 @@ void PluralRulesTest::testSamplesWithCompactNotation() { errln("Couldn't create plural rules from a string using exponent notation, with error = %s", u_errorName(status)); return; } - checkNewSamples(description, test, u"one", u"@integer 0, 1, 1c5", FixedDecimal(0)); - checkNewSamples(description, test, u"many", u"@integer 1000000, 2c6, 3c6, 4c6, 5c6, 6c6, 7c6, …", FixedDecimal(1000000)); - checkNewSamples(description, test, u"other", u"@integer 2~17, 100, 1000, 10000, 100000, 2c5, 3c5, 4c5, 5c5, 6c5, 7c5, …", FixedDecimal(2)); + checkNewSamples(description, test, u"one", u"@integer 0, 1, 1c5", DecimalQuantity::fromExponentString(u"0", status)); + checkNewSamples(description, test, u"many", u"@integer 1000000, 2c6, 3c6, 4c6, 5c6, 6c6, 7c6, …", DecimalQuantity::fromExponentString(u"1000000", status)); + checkNewSamples(description, test, u"other", u"@integer 2~17, 100, 1000, 10000, 100000, 2c5, 3c5, 4c5, 5c5, 6c5, 7c5, …", DecimalQuantity::fromExponentString(u"2", status)); // decimal samples status = U_ZERO_ERROR; @@ -593,9 +707,9 @@ void PluralRulesTest::testSamplesWithCompactNotation() { errln("Couldn't create plural rules from a string using exponent notation, with error = %s", u_errorName(status)); return; } - checkNewSamples(description2, test2, u"one", u"@decimal 0.0~1.5, 1.1c5", FixedDecimal(0, 1)); - checkNewSamples(description2, test2, u"many", u"@decimal 2.1c6, 3.1c6, 4.1c6, 5.1c6, 6.1c6, 7.1c6, …", FixedDecimal::createWithExponent(2.1, 1, 6)); - checkNewSamples(description2, test2, u"other", u"@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1c5, 3.1c5, 4.1c5, 5.1c5, 6.1c5, 7.1c5, …", FixedDecimal(2.0, 1)); + checkNewSamples(description2, test2, u"one", u"@decimal 0.0~1.5, 1.1c5", DecimalQuantity::fromExponentString(u"0.0", status)); + checkNewSamples(description2, test2, u"many", u"@decimal 2.1c6, 3.1c6, 4.1c6, 5.1c6, 6.1c6, 7.1c6, …", DecimalQuantity::fromExponentString(u"2.1c6", status)); + checkNewSamples(description2, test2, u"other", u"@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1c5, 3.1c5, 4.1c5, 5.1c5, 6.1c5, 7.1c5, …", DecimalQuantity::fromExponentString(u"2.0", status)); } void PluralRulesTest::checkNewSamples( @@ -603,17 +717,17 @@ void PluralRulesTest::checkNewSamples( const LocalPointer &test, UnicodeString keyword, UnicodeString samplesString, - FixedDecimal firstInRange) { + DecimalQuantity firstInRange) { UErrorCode status = U_ZERO_ERROR; - FixedDecimal samples[1000]; + DecimalQuantity samples[1000]; test->getSamples(keyword, samples, UPRV_LENGTHOF(samples), status); if (U_FAILURE(status)) { errln("Couldn't retrieve plural samples, with error = %s", u_errorName(status)); return; } - FixedDecimal actualFirstSample = samples[0]; + DecimalQuantity actualFirstSample = samples[0]; if (!(firstInRange == actualFirstSample)) { CStr descCstr(description); @@ -776,6 +890,11 @@ PluralRulesTest::testGetAllKeywordValues() { // For the time being, the compact notation exponent operand `c` is an alias // for the scientific exponent operand `e` and compact notation. +/** + * Test the proper plural rule keyword selection given an input number that is + * already formatted into scientific notation. This exercises the `e` plural operand + * for the formatted number. + */ void PluralRulesTest::testScientificPluralKeyword() { IcuTestErrorCode errorCode(*this, "testScientificPluralKeyword"); @@ -838,6 +957,11 @@ PluralRulesTest::testScientificPluralKeyword() { } } +/** + * Test the proper plural rule keyword selection given an input number that is + * already formatted into compact notation. This exercises the `c` plural operand + * for the formatted number. + */ void PluralRulesTest::testCompactDecimalPluralKeyword() { IcuTestErrorCode errorCode(*this, "testCompactDecimalPluralKeyword"); diff --git a/icu4c/source/test/intltest/plurults.h b/icu4c/source/test/intltest/plurults.h index 76ecb6bf191..f02a484ccd3 100644 --- a/icu4c/source/test/intltest/plurults.h +++ b/icu4c/source/test/intltest/plurults.h @@ -14,6 +14,7 @@ #if !UCONFIG_NO_FORMATTING #include "intltest.h" +#include "number_decimalquantity.h" #include "unicode/localpointer.h" #include "unicode/plurrule.h" @@ -30,7 +31,9 @@ private: void testAPI(); void testGetUniqueKeywordValue(); void testGetSamples(); - void testGetFixedDecimalSamples(); + void testGetDecimalQuantitySamples(); + void testGetOrAddSamplesFromString(); + void testGetOrAddSamplesFromStringCompactNotation(); void testSamplesWithExponent(); void testSamplesWithCompactNotation(); void testWithin(); @@ -55,7 +58,7 @@ private: const LocalPointer &test, UnicodeString keyword, UnicodeString samplesString, - FixedDecimal firstInRange); + ::icu::number::impl::DecimalQuantity firstInRange); UnicodeString getPluralKeyword(const LocalPointer &rules, Locale locale, double number, const char16_t* skeleton); void checkSelect(const LocalPointer &rules, UErrorCode &status, diff --git a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java b/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java index bf0e411d621..9cc79233939 100644 --- a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java +++ b/icu4j/main/classes/core/src/com/ibm/icu/text/PluralRules.java @@ -15,21 +15,22 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.ObjectStreamException; import java.io.Serializable; +import java.math.BigDecimal; import java.text.ParseException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Set; -import java.util.TreeSet; import java.util.regex.Pattern; import com.ibm.icu.impl.PluralRulesLoader; import com.ibm.icu.impl.StandardPlural; +import com.ibm.icu.impl.number.DecimalQuantity; +import com.ibm.icu.impl.number.DecimalQuantity_DualStorageBCD; import com.ibm.icu.impl.number.range.StandardPluralRanges; import com.ibm.icu.number.FormattedNumber; import com.ibm.icu.number.FormattedNumberRange; @@ -329,6 +330,16 @@ public class PluralRules implements Serializable { */ public static final double NO_UNIQUE_VALUE = -0.00123456777; + /** + * Value returned by {@link #getUniqueKeywordDecimalQuantityValue} when there is no + * unique value to return. + * @internal CLDR + * @deprecated This API is ICU internal only. + */ + @Deprecated + public static final DecimalQuantity NO_UNIQUE_VALUE_DECIMAL_QUANTITY = + new DecimalQuantity_DualStorageBCD(-0.00123456777); + /** * Type of plurals and PluralRules. * @stable ICU 50 @@ -867,50 +878,6 @@ public class PluralRules implements Serializable { this.baseFactor = other.baseFactor; } - /** - * @internal CLDR - * @deprecated This API is ICU internal only. - */ - @Deprecated - public FixedDecimal (String n) { - // Ugly, but for samples we don't care. - this(parseDecimalSampleRangeNumString(n)); - } - - /** - * @internal CLDR - * @deprecated This API is ICU internal only - */ - @Deprecated - private static FixedDecimal parseDecimalSampleRangeNumString(String num) { - if (num.contains("e") || num.contains("c")) { - int ePos = num.lastIndexOf('e'); - if (ePos < 0) { - ePos = num.lastIndexOf('c'); - } - int expNumPos = ePos + 1; - String exponentStr = num.substring(expNumPos); - int exponent = Integer.parseInt(exponentStr); - String fractionStr = num.substring(0, ePos); - return FixedDecimal.createWithExponent( - Double.parseDouble(fractionStr), - getVisibleFractionCount(fractionStr), - exponent); - } else { - return new FixedDecimal(Double.parseDouble(num), getVisibleFractionCount(num)); - } - } - - private static int getVisibleFractionCount(String value) { - value = value.trim(); - int decimalPos = value.indexOf('.') + 1; - if (decimalPos == 0) { - return 0; - } else { - return value.length() - decimalPos; - } - } - /** * {@inheritDoc} * @@ -1070,19 +1037,6 @@ public class PluralRules implements Serializable { return (isNegative ? -source : source) * Math.pow(10, exponent); } - /** - * @internal CLDR - * @deprecated This API is ICU internal only. - */ - @Deprecated - public long getShiftedValue() { - if (exponent != 0 && visibleDecimalDigitCount == 0 && decimalDigits == 0) { - // Need to take exponent into account if we have it - return (long)(source * Math.pow(10, exponent)); - } - return integerValue * baseFactor + decimalDigits; - } - private void writeObject( ObjectOutputStream out) throws IOException { @@ -1141,31 +1095,32 @@ public class PluralRules implements Serializable { } /** - * A range of NumberInfo that includes all values with the same visibleFractionDigitCount. + * A range of DecimalQuantity representing PluralRules samples that includes + * all values with the same visibleFractionDigitCount. * @internal CLDR * @deprecated This API is ICU internal only. */ @Deprecated - public static class FixedDecimalRange { + public static class DecimalQuantitySamplesRange { /** * @internal CLDR * @deprecated This API is ICU internal only. */ @Deprecated - public final FixedDecimal start; + public final DecimalQuantity start; /** * @internal CLDR * @deprecated This API is ICU internal only. */ @Deprecated - public final FixedDecimal end; + public final DecimalQuantity end; /** * @internal CLDR * @deprecated This API is ICU internal only. */ @Deprecated - public FixedDecimalRange(FixedDecimal start, FixedDecimal end) { - if (start.visibleDecimalDigitCount != end.visibleDecimalDigitCount) { + public DecimalQuantitySamplesRange(DecimalQuantity start, DecimalQuantity end) { + if (start.getPluralOperand(Operand.v)!= end.getPluralOperand(Operand.v)) { throw new IllegalArgumentException("Ranges must have the same number of visible decimals: " + start + "~" + end); } this.start = start; @@ -1178,17 +1133,18 @@ public class PluralRules implements Serializable { @Deprecated @Override public String toString() { - return start + (end == start ? "" : "~" + end); + return start.toExponentString() + (end == start ? "" : "~" + end.toExponentString()); } } /** - * A list of NumberInfo that includes all values with the same visibleFractionDigitCount. + * A list of DecimalQuantity representing PluralRules that includes all + * values with the same visibleFractionDigitCount. * @internal CLDR * @deprecated This API is ICU internal only. */ @Deprecated - public static class FixedDecimalSamples { + public static class DecimalQuantitySamples { /** * @internal CLDR * @deprecated This API is ICU internal only. @@ -1200,7 +1156,7 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public final Set samples; + public final Set samples; /** * @internal CLDR * @deprecated This API is ICU internal only. @@ -1212,7 +1168,7 @@ public class PluralRules implements Serializable { * @param sampleType * @param samples */ - private FixedDecimalSamples(SampleType sampleType, Set samples, boolean bounded) { + private DecimalQuantitySamples(SampleType sampleType, Set samples, boolean bounded) { super(); this.sampleType = sampleType; this.samples = samples; @@ -1221,11 +1177,11 @@ public class PluralRules implements Serializable { /* * Parse a list of the form described in CLDR. The source must be trimmed. */ - static FixedDecimalSamples parse(String source) { + static DecimalQuantitySamples parse(String source) { SampleType sampleType2; boolean bounded2 = true; boolean haveBound = false; - Set samples2 = new LinkedHashSet<>(); + Set samples2 = new LinkedHashSet<>(); if (source.startsWith("integer")) { sampleType2 = SampleType.INTEGER; @@ -1248,25 +1204,33 @@ public class PluralRules implements Serializable { String[] rangeParts = TILDE_SEPARATED.split(range, 0); switch (rangeParts.length) { case 1: - FixedDecimal sample = new FixedDecimal(rangeParts[0]); + DecimalQuantity sample = + DecimalQuantity_DualStorageBCD.fromExponentString(rangeParts[0]); checkDecimal(sampleType2, sample); - samples2.add(new FixedDecimalRange(sample, sample)); + samples2.add(new DecimalQuantitySamplesRange(sample, sample)); break; case 2: - FixedDecimal start = new FixedDecimal(rangeParts[0]); - FixedDecimal end = new FixedDecimal(rangeParts[1]); + DecimalQuantity start = + DecimalQuantity_DualStorageBCD.fromExponentString(rangeParts[0]); + DecimalQuantity end = + DecimalQuantity_DualStorageBCD.fromExponentString(rangeParts[1]); checkDecimal(sampleType2, start); checkDecimal(sampleType2, end); - samples2.add(new FixedDecimalRange(start, end)); + samples2.add(new DecimalQuantitySamplesRange(start, end)); break; default: throw new IllegalArgumentException("Ill-formed number range: " + range); } } - return new FixedDecimalSamples(sampleType2, Collections.unmodifiableSet(samples2), bounded2); + return new DecimalQuantitySamples(sampleType2, Collections.unmodifiableSet(samples2), bounded2); } - private static void checkDecimal(SampleType sampleType2, FixedDecimal sample) { - if ((sampleType2 == SampleType.INTEGER) != (sample.getVisibleDecimalDigitCount() == 0)) { + private static void checkDecimal(SampleType sampleType2, DecimalQuantity sample) { + // TODO(CLDR-15452): Remove the need for the fallback check for exponent notation integers classified + // as "@decimal" type samples, if/when changes are made to + // resolve https://unicode-org.atlassian.net/browse/CLDR-15452 + if ((sampleType2 == SampleType.INTEGER && sample.getPluralOperand(Operand.v) != 0) + || (sampleType2 == SampleType.DECIMAL && sample.getPluralOperand(Operand.v) == 0 + && sample.getPluralOperand(Operand.e) == 0)) { throw new IllegalArgumentException("Ill-formed number range: " + sample); } } @@ -1276,17 +1240,64 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public Set addSamples(Set result) { - for (FixedDecimalRange item : samples) { - // we have to convert to longs so we don't get strange double issues - long startDouble = item.start.getShiftedValue(); - long endDouble = item.end.getShiftedValue(); + public Collection addSamples(Collection result) { + addSamples(result, null); + return result; + } - for (long d = startDouble; d <= endDouble; d += 1) { - result.add(d/(double)item.start.baseFactor); + /** + * @internal CLDR + * @deprecated This API is ICU internal only. + */ + @Deprecated + public Collection addDecimalQuantitySamples(Collection result) { + addSamples(null, result); + return result; + } + + /** + * @internal CLDR + * @deprecated This API is ICU internal only. + */ + @Deprecated + public void addSamples(Collection doubleResult, Collection dqResult) { + if ((doubleResult == null && dqResult == null) + || (doubleResult != null && dqResult != null)) { + return; + } + boolean isDouble = doubleResult != null; + for (DecimalQuantitySamplesRange range : samples) { + DecimalQuantity start = range.start; + DecimalQuantity end = range.end; + int lowerDispMag = start.getLowerDisplayMagnitude(); + int exponent = start.getExponent(); + int incrementScale = lowerDispMag + exponent; + BigDecimal incrementBd = BigDecimal.ONE.movePointRight(incrementScale); + + for (DecimalQuantity dq = start.createCopy(); dq.toDouble() <= end.toDouble(); ) { + if (isDouble) { + double dblValue = dq.toDouble(); + // Hack Alert: don't return any decimal samples with integer values that + // originated from a format with trailing decimals. + // This API is returning doubles, which can't distinguish having displayed + // zeros to the right of the decimal. + // This results in test failures with values mapping back to a different keyword. + if (!(dblValue == Math.floor(dblValue)) && dq.getPluralOperand(Operand.v) > 0) { + doubleResult.add(dblValue); + } + } else { + dqResult.add(dq); + } + + // Increment dq for next iteration + java.math.BigDecimal dqBd = dq.toBigDecimal(); + java.math.BigDecimal newDqBd = dqBd.add(incrementBd); + dq = new DecimalQuantity_DualStorageBCD(newDqBd); + dq.setMinFraction(-lowerDispMag); + dq.adjustMagnitude(-exponent); + dq.adjustExponent(exponent); } } - return result; } /** @@ -1298,7 +1309,7 @@ public class PluralRules implements Serializable { public String toString() { StringBuilder b = new StringBuilder("@").append(sampleType.toString().toLowerCase(Locale.ENGLISH)); boolean first = true; - for (FixedDecimalRange item : samples) { + for (DecimalQuantitySamplesRange item : samples) { if (first) { first = false; } else { @@ -1317,7 +1328,7 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public Set getSamples() { + public Set getSamples() { return samples; } @@ -1326,10 +1337,10 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public void getStartEndSamples(Set target) { - for (FixedDecimalRange item : samples) { - target.add(item.start); - target.add(item.end); + public void getStartEndSamples(Set target) { + for (DecimalQuantitySamplesRange range : samples) { + target.add(range.start); + target.add(range.end); } } } @@ -1610,19 +1621,19 @@ public class PluralRules implements Serializable { description = description.substring(x+1).trim(); String[] constraintOrSamples = AT_SEPARATED.split(description, 0); boolean sampleFailure = false; - FixedDecimalSamples integerSamples = null, decimalSamples = null; + DecimalQuantitySamples integerSamples = null, decimalSamples = null; switch (constraintOrSamples.length) { case 1: break; case 2: - integerSamples = FixedDecimalSamples.parse(constraintOrSamples[1]); + integerSamples = DecimalQuantitySamples.parse(constraintOrSamples[1]); if (integerSamples.sampleType == SampleType.DECIMAL) { decimalSamples = integerSamples; integerSamples = null; } break; case 3: - integerSamples = FixedDecimalSamples.parse(constraintOrSamples[1]); - decimalSamples = FixedDecimalSamples.parse(constraintOrSamples[2]); + integerSamples = DecimalQuantitySamples.parse(constraintOrSamples[1]); + decimalSamples = DecimalQuantitySamples.parse(constraintOrSamples[2]); if (integerSamples.sampleType != SampleType.INTEGER || decimalSamples.sampleType != SampleType.DECIMAL) { throw new IllegalArgumentException("Must have @integer then @decimal in " + description); } @@ -1857,10 +1868,10 @@ public class PluralRules implements Serializable { private static final long serialVersionUID = 1; private final String keyword; private final Constraint constraint; - private final FixedDecimalSamples integerSamples; - private final FixedDecimalSamples decimalSamples; + private final DecimalQuantitySamples integerSamples; + private final DecimalQuantitySamples decimalSamples; - public Rule(String keyword, Constraint constraint, FixedDecimalSamples integerSamples, FixedDecimalSamples decimalSamples) { + public Rule(String keyword, Constraint constraint, DecimalQuantitySamples integerSamples, DecimalQuantitySamples decimalSamples) { this.keyword = keyword; this.constraint = constraint; this.integerSamples = integerSamples; @@ -1972,7 +1983,7 @@ public class PluralRules implements Serializable { public boolean isLimited(String keyword, SampleType sampleType) { if (hasExplicitBoundingInfo) { - FixedDecimalSamples mySamples = getDecimalSamples(keyword, sampleType); + DecimalQuantitySamples mySamples = getDecimalSamples(keyword, sampleType); return mySamples == null ? true : mySamples.bounded; } @@ -2024,7 +2035,7 @@ public class PluralRules implements Serializable { return false; } - public FixedDecimalSamples getDecimalSamples(String keyword, SampleType sampleType) { + public DecimalQuantitySamples getDecimalSamples(String keyword, SampleType sampleType) { for (Rule rule : rules) { if (rule.getKeyword().equals(keyword)) { return sampleType == SampleType.INTEGER ? rule.integerSamples : rule.decimalSamples; @@ -2285,11 +2296,29 @@ public class PluralRules implements Serializable { * @stable ICU 4.8 */ public double getUniqueKeywordValue(String keyword) { - Collection values = getAllKeywordValues(keyword); + DecimalQuantity uniqValDq = getUniqueKeywordDecimalQuantityValue(keyword); + if (uniqValDq.equals(NO_UNIQUE_VALUE_DECIMAL_QUANTITY)) { + return NO_UNIQUE_VALUE; + } else { + return uniqValDq.toDouble(); + } + } + + /** + * Returns the unique value that this keyword matches, or {@link #NO_UNIQUE_VALUE} + * if the keyword matches multiple values or is not defined for this PluralRules. + * + * @param keyword the keyword to check for a unique value + * @internal Visible For Testing + * @deprecated This API is ICU internal only. + */ + @Deprecated + public DecimalQuantity getUniqueKeywordDecimalQuantityValue(String keyword) { + Collection values = getAllKeywordDecimalQuantityValues(keyword); if (values != null && values.size() == 1) { return values.iterator().next(); } - return NO_UNIQUE_VALUE; + return NO_UNIQUE_VALUE_DECIMAL_QUANTITY; } /** @@ -2302,6 +2331,31 @@ public class PluralRules implements Serializable { * @stable ICU 4.8 */ public Collection getAllKeywordValues(String keyword) { + Collection samples = getAllKeywordDecimalQuantityValues(keyword); + if (samples == null) { + return null; + } else { + Collection result = new LinkedHashSet<>(); + for (DecimalQuantity dq : samples) { + result.add(dq.toDouble()); + } + return result; + } + } + + /** + * Returns all the values that trigger this keyword, or null if the number of such + * values is unlimited. + * + * @param keyword the keyword + * @return the values that trigger this keyword, or null. The returned collection + * is immutable. It will be empty if the keyword is not defined. + * + * @internal Visible For Testing + * @deprecated This API is ICU internal only. + */ + @Deprecated + public Collection getAllKeywordDecimalQuantityValues(String keyword) { return getAllKeywordValues(keyword, SampleType.INTEGER); } @@ -2318,12 +2372,11 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public Collection getAllKeywordValues(String keyword, SampleType type) { + public Collection getAllKeywordValues(String keyword, SampleType type) { if (!isLimited(keyword, type)) { return null; } - Collection samples = getSamples(keyword, type); - return samples == null ? null : Collections.unmodifiableCollection(samples); + return getDecimalQuantitySamples(keyword, type); } /** @@ -2340,6 +2393,22 @@ public class PluralRules implements Serializable { return getSamples(keyword, SampleType.INTEGER); } + /** + * Returns a list of integer values for which select() would return that keyword, + * or null if the keyword is not defined. The returned collection is unmodifiable. + * The returned list is not complete, and there might be additional values that + * would return the keyword. + * + * @param keyword the keyword to test + * @return a list of values matching the keyword. + * @internal CLDR + * @deprecated ICU internal only + */ + @Deprecated + public Collection getDecimalQuantitySamples(String keyword) { + return getDecimalQuantitySamples(keyword, SampleType.INTEGER); + } + /** * Returns a list of values for which select() would return that keyword, * or null if the keyword is not defined. @@ -2356,15 +2425,43 @@ public class PluralRules implements Serializable { */ @Deprecated public Collection getSamples(String keyword, SampleType sampleType) { + Collection samples = getDecimalQuantitySamples(keyword, sampleType); + if (samples == null) { + return null; + } else { + Collection result = new LinkedHashSet<>(); + for (DecimalQuantity dq: samples) { + result.add(dq.toDouble()); + } + return result; + } + } + + /** + * Returns a list of values for which select() would return that keyword, + * or null if the keyword is not defined. + * The returned collection is unmodifiable. + * The returned list is not complete, and there might be additional values that + * would return the keyword. The keyword might be defined, and yet have an empty set of samples, + * IF there are samples for the other sampleType. + * + * @param keyword the keyword to test + * @param sampleType the type of samples requested, INTEGER or DECIMAL + * @return a list of values matching the keyword. + * @internal CLDR + * @deprecated ICU internal only + */ + @Deprecated + public Collection getDecimalQuantitySamples(String keyword, SampleType sampleType) { if (!keywords.contains(keyword)) { return null; } - Set result = new TreeSet<>(); + Set result = new LinkedHashSet<>(); if (rules.hasExplicitBoundingInfo) { - FixedDecimalSamples samples = rules.getDecimalSamples(keyword, sampleType); + DecimalQuantitySamples samples = rules.getDecimalSamples(keyword, sampleType); return samples == null ? Collections.unmodifiableSet(result) - : Collections.unmodifiableSet(samples.addSamples(result)); + : Collections.unmodifiableCollection(samples.addDecimalQuantitySamples(result)); } // hack in case the rule is created without explicit samples @@ -2373,28 +2470,31 @@ public class PluralRules implements Serializable { switch (sampleType) { case INTEGER: for (int i = 0; i < 200; ++i) { - if (!addSample(keyword, i, maxCount, result)) { + if (!addSample(keyword, new DecimalQuantity_DualStorageBCD(i), maxCount, result)) { break; } } - addSample(keyword, 1000000, maxCount, result); // hack for Welsh + addSample(keyword, new DecimalQuantity_DualStorageBCD(1000000), maxCount, result); // hack for Welsh break; case DECIMAL: for (int i = 0; i < 2000; ++i) { - if (!addSample(keyword, new FixedDecimal(i/10d, 1), maxCount, result)) { + DecimalQuantity_DualStorageBCD nextSample = new DecimalQuantity_DualStorageBCD(i); + nextSample.adjustMagnitude(-1); + if (!addSample(keyword, nextSample, maxCount, result)) { break; } } - addSample(keyword, new FixedDecimal(1000000d, 1), maxCount, result); // hack for Welsh + addSample(keyword, DecimalQuantity_DualStorageBCD.fromExponentString("1000000.0"), maxCount, result); // hack for Welsh break; } + return result.size() == 0 ? null : Collections.unmodifiableSet(result); } - private boolean addSample(String keyword, Number sample, int maxCount, Set result) { - String selectedKeyword = sample instanceof FixedDecimal ? select((FixedDecimal)sample) : select(sample.doubleValue()); + private boolean addSample(String keyword, DecimalQuantity sample, int maxCount, Set result) { + String selectedKeyword = select(sample); if (selectedKeyword.equals(keyword)) { - result.add(sample.doubleValue()); + result.add(sample); if (--maxCount < 0) { return false; } @@ -2416,7 +2516,7 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public FixedDecimalSamples getDecimalSamples(String keyword, SampleType sampleType) { + public DecimalQuantitySamples getDecimalSamples(String keyword, SampleType sampleType) { return rules.getDecimalSamples(keyword, sampleType); } @@ -2525,14 +2625,14 @@ public class PluralRules implements Serializable { * the offset used, or 0.0d if not. Internally, the offset is subtracted from each explicit value before * checking against the keyword values. * @param explicits - * a set of Doubles that are used explicitly (eg [=0], "[=1]"). May be empty or null. + * a set of {@code DecimalQuantity}s that are used explicitly (eg [=0], "[=1]"). May be empty or null. * @param uniqueValue * If non null, set to the unique value. * @return the KeywordStatus * @draft ICU 50 */ - public KeywordStatus getKeywordStatus(String keyword, int offset, Set explicits, - Output uniqueValue) { + public KeywordStatus getKeywordStatus(String keyword, int offset, Set explicits, + Output uniqueValue) { return getKeywordStatus(keyword, offset, explicits, uniqueValue, SampleType.INTEGER); } /** @@ -2544,7 +2644,7 @@ public class PluralRules implements Serializable { * the offset used, or 0.0d if not. Internally, the offset is subtracted from each explicit value before * checking against the keyword values. * @param explicits - * a set of Doubles that are used explicitly (eg [=0], "[=1]"). May be empty or null. + * a set of {@code DecimalQuantity}s that are used explicitly (eg [=0], "[=1]"). May be empty or null. * @param sampleType * request KeywordStatus relative to INTEGER or DECIMAL values * @param uniqueValue @@ -2554,8 +2654,8 @@ public class PluralRules implements Serializable { * @deprecated This API is ICU internal only. */ @Deprecated - public KeywordStatus getKeywordStatus(String keyword, int offset, Set explicits, - Output uniqueValue, SampleType sampleType) { + public KeywordStatus getKeywordStatus(String keyword, int offset, + Set explicits, Output uniqueValue, SampleType sampleType) { if (uniqueValue != null) { uniqueValue.value = null; } @@ -2568,7 +2668,7 @@ public class PluralRules implements Serializable { return KeywordStatus.UNBOUNDED; } - Collection values = getSamples(keyword, sampleType); + Collection values = getDecimalQuantitySamples(keyword, sampleType); int originalSize = values.size(); @@ -2590,9 +2690,12 @@ public class PluralRules implements Serializable { // Compute if the quick test is insufficient. - HashSet subtractedSet = new HashSet<>(values); - for (Double explicit : explicits) { - subtractedSet.remove(explicit - offset); + ArrayList subtractedSet = new ArrayList<>(values); + for (DecimalQuantity explicit : explicits) { + BigDecimal explicitBd = explicit.toBigDecimal(); + BigDecimal valToRemoveBd = explicitBd.subtract(new BigDecimal(offset)); + DecimalQuantity_DualStorageBCD valToRemove = new DecimalQuantity_DualStorageBCD(valToRemoveBd); + subtractedSet.remove(valToRemove); } if (subtractedSet.size() == 0) { return KeywordStatus.SUPPRESSED; diff --git a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralSamples.java b/icu4j/main/classes/core/src/com/ibm/icu/text/PluralSamples.java deleted file mode 100644 index 6b260c62af2..00000000000 --- a/icu4j/main/classes/core/src/com/ibm/icu/text/PluralSamples.java +++ /dev/null @@ -1,332 +0,0 @@ -// © 2016 and later: Unicode, Inc. and others. -// License & terms of use: http://www.unicode.org/copyright.html -/* - ******************************************************************************* - * Copyright (C) 2013-2015, International Business Machines Corporation and - * others. All Rights Reserved. - ******************************************************************************* - */ -package com.ibm.icu.text; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeSet; - -import com.ibm.icu.text.PluralRules.FixedDecimal; -import com.ibm.icu.text.PluralRules.KeywordStatus; -import com.ibm.icu.util.Output; - -/** - * @author markdavis - * Refactor samples as first step to moving into CLDR - * - * @internal - * @deprecated This API is ICU internal only. - */ -@Deprecated -public class PluralSamples { - - private PluralRules pluralRules; - private final Map> _keySamplesMap; - - /** - * @internal - * @deprecated This API is ICU internal only. - */ - @Deprecated - public final Map _keyLimitedMap; - private final Map> _keyFractionSamplesMap; - private final Set _fractionSamples; - - /** - * @internal - * @deprecated This API is ICU internal only. - */ - @Deprecated - public PluralSamples(PluralRules pluralRules) { - this.pluralRules = pluralRules; - Set keywords = pluralRules.getKeywords(); - // ensure both _keySamplesMap and _keyLimitedMap are initialized. - // If this were allowed to vary on a per-call basis, we'd have to recheck and - // possibly rebuild the samples cache. Doesn't seem worth it. - // This 'max samples' value only applies to keywords that are unlimited, for - // other keywords all the matching values are returned. This might be a lot. - final int MAX_SAMPLES = 3; - - Map temp = new HashMap(); - for (String k : keywords) { - temp.put(k, pluralRules.isLimited(k)); - } - _keyLimitedMap = temp; - - Map> sampleMap = new HashMap>(); - int keywordsRemaining = keywords.size(); - - int limit = 128; // Math.max(5, getRepeatLimit() * MAX_SAMPLES) * 2; - - for (int i = 0; keywordsRemaining > 0 && i < limit; ++i) { - keywordsRemaining = addSimpleSamples(pluralRules, MAX_SAMPLES, sampleMap, keywordsRemaining, i / 2.0); - } - // Hack for Celtic - keywordsRemaining = addSimpleSamples(pluralRules, MAX_SAMPLES, sampleMap, keywordsRemaining, 1000000); - - - // collect explicit samples - Map> sampleFractionMap = new HashMap>(); - Set mentioned = new TreeSet(); - // make sure that there is at least one 'other' value - Map> foundKeywords = new HashMap>(); - for (FixedDecimal s : mentioned) { - String keyword = pluralRules.select(s); - addRelation(foundKeywords, keyword, s); - } - main: - if (foundKeywords.size() != keywords.size()) { - for (int i = 1; i < 1000; ++i) { - boolean done = addIfNotPresent(i, mentioned, foundKeywords); - if (done) break main; - } - // if we are not done, try tenths - for (int i = 10; i < 1000; ++i) { - boolean done = addIfNotPresent(i/10d, mentioned, foundKeywords); - if (done) break main; - } - System.out.println("Failed to find sample for each keyword: " + foundKeywords + "\n\t" + pluralRules + "\n\t" + mentioned); - } - mentioned.add(new FixedDecimal(0)); // always there - mentioned.add(new FixedDecimal(1)); // always there - mentioned.add(new FixedDecimal(2)); // always there - mentioned.add(new FixedDecimal(0.1,1)); // always there - mentioned.add(new FixedDecimal(1.99,2)); // always there - mentioned.addAll(fractions(mentioned)); - for (FixedDecimal s : mentioned) { - String keyword = pluralRules.select(s); - Set list = sampleFractionMap.get(keyword); - if (list == null) { - list = new LinkedHashSet(); // will be sorted because the iteration is - sampleFractionMap.put(keyword, list); - } - list.add(s); - } - - if (keywordsRemaining > 0) { - for (String k : keywords) { - if (!sampleMap.containsKey(k)) { - sampleMap.put(k, Collections.emptyList()); - } - if (!sampleFractionMap.containsKey(k)) { - sampleFractionMap.put(k, Collections.emptySet()); - } - } - } - - // Make lists immutable so we can return them directly - for (Entry> entry : sampleMap.entrySet()) { - sampleMap.put(entry.getKey(), Collections.unmodifiableList(entry.getValue())); - } - for (Entry> entry : sampleFractionMap.entrySet()) { - sampleFractionMap.put(entry.getKey(), Collections.unmodifiableSet(entry.getValue())); - } - _keySamplesMap = sampleMap; - _keyFractionSamplesMap = sampleFractionMap; - _fractionSamples = Collections.unmodifiableSet(mentioned); - } - - private int addSimpleSamples(PluralRules pluralRules, final int MAX_SAMPLES, Map> sampleMap, - int keywordsRemaining, double val) { - String keyword = pluralRules.select(val); - boolean keyIsLimited = _keyLimitedMap.get(keyword); - - List list = sampleMap.get(keyword); - if (list == null) { - list = new ArrayList(MAX_SAMPLES); - sampleMap.put(keyword, list); - } else if (!keyIsLimited && list.size() == MAX_SAMPLES) { - return keywordsRemaining; - } - list.add(Double.valueOf(val)); - - if (!keyIsLimited && list.size() == MAX_SAMPLES) { - --keywordsRemaining; - } - return keywordsRemaining; - } - - private void addRelation(Map> foundKeywords, String keyword, FixedDecimal s) { - Set set = foundKeywords.get(keyword); - if (set == null) { - foundKeywords.put(keyword, set = new HashSet()); - } - set.add(s); - } - - private boolean addIfNotPresent(double d, Set mentioned, Map> foundKeywords) { - FixedDecimal numberInfo = new FixedDecimal(d); - String keyword = pluralRules.select(numberInfo); - if (!foundKeywords.containsKey(keyword) || keyword.equals("other")) { - addRelation(foundKeywords, keyword, numberInfo); - mentioned.add(numberInfo); - if (keyword.equals("other")) { - if (foundKeywords.get("other").size() > 1) { - return true; - } - } - } - return false; - } - - private static final int[] TENS = {1, 10, 100, 1000, 10000, 100000, 1000000}; - - private static final int LIMIT_FRACTION_SAMPLES = 3; - - - private Set fractions(Set original) { - Set toAddTo = new HashSet(); - - Set result = new HashSet(); - for (FixedDecimal base1 : original) { - result.add((int)base1.integerValue); - } - List ints = new ArrayList(result); - Set keywords = new HashSet(); - - for (int j = 0; j < ints.size(); ++j) { - Integer base = ints.get(j); - String keyword = pluralRules.select(base); - if (keywords.contains(keyword)) { - continue; - } - keywords.add(keyword); - toAddTo.add(new FixedDecimal(base,1)); // add .0 - toAddTo.add(new FixedDecimal(base,2)); // add .00 - Integer fract = getDifferentCategory(ints, keyword); - if (fract >= TENS[LIMIT_FRACTION_SAMPLES-1]) { // make sure that we always get the value - toAddTo.add(new FixedDecimal(base + "." + fract)); - } else { - for (int visibleFractions = 1; visibleFractions < LIMIT_FRACTION_SAMPLES; ++visibleFractions) { - for (int i = 1; i <= visibleFractions; ++i) { - // with visible fractions = 3, and fract = 1, then we should get x.10, 0.01 - // with visible fractions = 3, and fract = 15, then we should get x.15, x.15 - if (fract >= TENS[i]) { - continue; - } - toAddTo.add(new FixedDecimal(base + fract/(double)TENS[i], visibleFractions)); - } - } - } - } - return toAddTo; - } - - private Integer getDifferentCategory(List ints, String keyword) { - for (int i = ints.size() - 1; i >= 0; --i) { - Integer other = ints.get(i); - String keywordOther = pluralRules.select(other); - if (!keywordOther.equals(keyword)) { - return other; - } - } - return 37; - } - - /** - * @internal - * @deprecated This API is ICU internal only. - */ - @Deprecated - public KeywordStatus getStatus(String keyword, int offset, Set explicits, Output uniqueValue) { - if (uniqueValue != null) { - uniqueValue.value = null; - } - - if (!pluralRules.getKeywords().contains(keyword)) { - return KeywordStatus.INVALID; - } - Collection values = pluralRules.getAllKeywordValues(keyword); - if (values == null) { - return KeywordStatus.UNBOUNDED; - } - int originalSize = values.size(); - - if (explicits == null) { - explicits = Collections.emptySet(); - } - - // Quick check on whether there are multiple elements - - if (originalSize > explicits.size()) { - if (originalSize == 1) { - if (uniqueValue != null) { - uniqueValue.value = values.iterator().next(); - } - return KeywordStatus.UNIQUE; - } - return KeywordStatus.BOUNDED; - } - - // Compute if the quick test is insufficient. - - HashSet subtractedSet = new HashSet(values); - for (Double explicit : explicits) { - subtractedSet.remove(explicit - offset); - } - if (subtractedSet.size() == 0) { - return KeywordStatus.SUPPRESSED; - } - - if (uniqueValue != null && subtractedSet.size() == 1) { - uniqueValue.value = subtractedSet.iterator().next(); - } - - return originalSize == 1 ? KeywordStatus.UNIQUE : KeywordStatus.BOUNDED; - } - - Map> getKeySamplesMap() { - return _keySamplesMap; - } - - Map> getKeyFractionSamplesMap() { - return _keyFractionSamplesMap; - } - - Set getFractionSamples() { - return _fractionSamples; - } - - /** - * Returns all the values that trigger this keyword, or null if the number of such - * values is unlimited. - * - * @param keyword the keyword - * @return the values that trigger this keyword, or null. The returned collection - * is immutable. It will be empty if the keyword is not defined. - * @stable ICU 4.8 - */ - - Collection getAllKeywordValues(String keyword) { - // HACK for now - if (!pluralRules.getKeywords().contains(keyword)) { - return Collections.emptyList(); - } - Collection result = getKeySamplesMap().get(keyword); - - // We depend on MAX_SAMPLES here. It's possible for a conjunction - // of unlimited rules that 'looks' unlimited to return a limited - // number of values. There's no bounds to this limited number, in - // general, because you can construct arbitrarily complex rules. Since - // we always generate 3 samples if a rule is really unlimited, that's - // where we put the cutoff. - if (result.size() > 2 && !_keyLimitedMap.get(keyword)) { - return null; - } - return result; - } -} diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralFormatUnitTest.java b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralFormatUnitTest.java index 0b22c8ea2b6..1e2c0d94649 100644 --- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralFormatUnitTest.java +++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralFormatUnitTest.java @@ -23,6 +23,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import com.ibm.icu.dev.test.TestFmwk; +import com.ibm.icu.impl.number.DecimalQuantity; import com.ibm.icu.text.DecimalFormat; import com.ibm.icu.text.DecimalFormatSymbols; import com.ibm.icu.text.MessageFormat; @@ -228,10 +229,10 @@ public class PluralFormatUnitTest extends TestFmwk { logln(localeName + "\ttoString\t" + rules.toString()); Set keywords = rules.getKeywords(); for (String keyword : keywords) { - Collection list = rules.getSamples(keyword); + Collection list = rules.getDecimalQuantitySamples(keyword); if (list.size() == 0) { // if there aren't any integer samples, get the decimal ones. - list = rules.getSamples(keyword, SampleType.DECIMAL); + list = rules.getDecimalQuantitySamples(keyword, SampleType.DECIMAL); } if (list == null || list.size() == 0) { diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java index d07f41e3a75..f0bed534147 100644 --- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java +++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/PluralRulesTest.java @@ -23,6 +23,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -31,6 +32,8 @@ import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,6 +44,8 @@ import com.ibm.icu.dev.test.serializable.SerializableTestUtility; import com.ibm.icu.dev.util.CollectionUtilities; import com.ibm.icu.impl.Relation; import com.ibm.icu.impl.Utility; +import com.ibm.icu.impl.number.DecimalQuantity; +import com.ibm.icu.impl.number.DecimalQuantity_DualStorageBCD; import com.ibm.icu.number.FormattedNumber; import com.ibm.icu.number.FormattedNumberRange; import com.ibm.icu.number.LocalizedNumberFormatter; @@ -50,9 +55,9 @@ import com.ibm.icu.number.Precision; import com.ibm.icu.number.UnlocalizedNumberFormatter; import com.ibm.icu.text.NumberFormat; import com.ibm.icu.text.PluralRules; +import com.ibm.icu.text.PluralRules.DecimalQuantitySamples; +import com.ibm.icu.text.PluralRules.DecimalQuantitySamplesRange; import com.ibm.icu.text.PluralRules.FixedDecimal; -import com.ibm.icu.text.PluralRules.FixedDecimalRange; -import com.ibm.icu.text.PluralRules.FixedDecimalSamples; import com.ibm.icu.text.PluralRules.KeywordStatus; import com.ibm.icu.text.PluralRules.PluralType; import com.ibm.icu.text.PluralRules.SampleType; @@ -177,17 +182,27 @@ public class PluralRulesTest extends TestFmwk { PluralRules test = PluralRules.createRules(description); checkNewSamples(description, test, "one", PluralRules.SampleType.INTEGER, "@integer 3, 19", true, - new FixedDecimal(3)); + DecimalQuantity_DualStorageBCD.fromExponentString("3")); checkNewSamples(description, test, "one", PluralRules.SampleType.DECIMAL, "@decimal 3.50~3.53, …", false, - new FixedDecimal(3.5, 2)); - checkOldSamples(description, test, "one", SampleType.INTEGER, 3d, 19d); - checkOldSamples(description, test, "one", SampleType.DECIMAL, 3.5d, 3.51d, 3.52d, 3.53d); + DecimalQuantity_DualStorageBCD.fromExponentString("3.50")); + checkOldSamples(description, test, "one", SampleType.INTEGER, + DecimalQuantity_DualStorageBCD.fromExponentString("3"), + DecimalQuantity_DualStorageBCD.fromExponentString("19")); + checkOldSamples(description, test, "one", SampleType.DECIMAL, + DecimalQuantity_DualStorageBCD.fromExponentString("3.50"), + DecimalQuantity_DualStorageBCD.fromExponentString("3.51"), + DecimalQuantity_DualStorageBCD.fromExponentString("3.52"), + DecimalQuantity_DualStorageBCD.fromExponentString("3.53")); checkNewSamples(description, test, "other", PluralRules.SampleType.INTEGER, "", true, null); checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 99.0~99.2, 999.0, …", - false, new FixedDecimal(99d, 1)); + false, DecimalQuantity_DualStorageBCD.fromExponentString("99.0")); checkOldSamples(description, test, "other", SampleType.INTEGER); - checkOldSamples(description, test, "other", SampleType.DECIMAL, 99d, 99.1, 99.2d, 999d); + checkOldSamples(description, test, "other", SampleType.DECIMAL, + DecimalQuantity_DualStorageBCD.fromExponentString("99.0"), + DecimalQuantity_DualStorageBCD.fromExponentString("99.1"), + DecimalQuantity_DualStorageBCD.fromExponentString("99.2"), + DecimalQuantity_DualStorageBCD.fromExponentString("999.0")); } /** @@ -205,22 +220,26 @@ public class PluralRulesTest extends TestFmwk { // Creating the PluralRules object means being able to parse numbers // like 1e5 and 1.1e5 PluralRules test = PluralRules.createRules(description); + + // Currently, 'c' is the canonical representation of numbers with suppressed exponent, and 'e' + // is an alias. The test helpers here skip 'e' for round-trip sample string parsing and formatting. + checkNewSamples(description, test, "one", PluralRules.SampleType.INTEGER, "@integer 0, 1, 1e5", true, - new FixedDecimal(0)); + DecimalQuantity_DualStorageBCD.fromExponentString("0")); checkNewSamples(description, test, "one", PluralRules.SampleType.DECIMAL, "@decimal 0.0~1.5, 1.1e5", true, - new FixedDecimal(0, 1)); + DecimalQuantity_DualStorageBCD.fromExponentString("0.0")); checkNewSamples(description, test, "many", PluralRules.SampleType.INTEGER, "@integer 1000000, 2e6, 3e6, 4e6, 5e6, 6e6, 7e6, …", false, - new FixedDecimal(1000000)); + DecimalQuantity_DualStorageBCD.fromExponentString("1000000")); checkNewSamples(description, test, "many", PluralRules.SampleType.DECIMAL, "@decimal 2.1e6, 3.1e6, 4.1e6, 5.1e6, 6.1e6, 7.1e6, …", false, - FixedDecimal.createWithExponent(2.1, 1, 6)); + DecimalQuantity_DualStorageBCD.fromExponentString("2.1c6")); checkNewSamples(description, test, "other", PluralRules.SampleType.INTEGER, "@integer 2~17, 100, 1000, 10000, 100000, 2e5, 3e5, 4e5, 5e5, 6e5, 7e5, …", false, - new FixedDecimal(2)); + DecimalQuantity_DualStorageBCD.fromExponentString("2")); checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1e5, 3.1e5, 4.1e5, 5.1e5, 6.1e5, 7.1e5, …", false, - new FixedDecimal(2.0, 1)); + DecimalQuantity_DualStorageBCD.fromExponentString("2.0")); } /** - * This test is for the support of X.YcZ compactnotation of numbers in + * This test is for the support of X.YcZ compact notation of numbers in * the plural sample string. */ @Test @@ -236,34 +255,53 @@ public class PluralRulesTest extends TestFmwk { // Note: Since `c` is currently an alias to `e`, the toString() of // FixedDecimal will return "1e5" even when input is "1c5". PluralRules test = PluralRules.createRules(description); - checkNewSamples(description, test, "one", PluralRules.SampleType.INTEGER, "@integer 0, 1, 1e5", true, - new FixedDecimal(0)); - checkNewSamples(description, test, "one", PluralRules.SampleType.DECIMAL, "@decimal 0.0~1.5, 1.1e5", true, - new FixedDecimal(0, 1)); - checkNewSamples(description, test, "many", PluralRules.SampleType.INTEGER, "@integer 1000000, 2e6, 3e6, 4e6, 5e6, 6e6, 7e6, …", false, - new FixedDecimal(1000000)); - checkNewSamples(description, test, "many", PluralRules.SampleType.DECIMAL, "@decimal 2.1e6, 3.1e6, 4.1e6, 5.1e6, 6.1e6, 7.1e6, …", false, - FixedDecimal.createWithExponent(2.1, 1, 6)); - checkNewSamples(description, test, "other", PluralRules.SampleType.INTEGER, "@integer 2~17, 100, 1000, 10000, 100000, 2e5, 3e5, 4e5, 5e5, 6e5, 7e5, …", false, - new FixedDecimal(2)); - checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1e5, 3.1e5, 4.1e5, 5.1e5, 6.1e5, 7.1e5, …", false, - new FixedDecimal(2.0, 1)); + + checkNewSamples(description, test, "one", PluralRules.SampleType.INTEGER, "@integer 0, 1, 1c5", true, + DecimalQuantity_DualStorageBCD.fromExponentString("0")); + checkNewSamples(description, test, "one", PluralRules.SampleType.DECIMAL, "@decimal 0.0~1.5, 1.1c5", true, + DecimalQuantity_DualStorageBCD.fromExponentString("0.0")); + checkNewSamples(description, test, "many", PluralRules.SampleType.INTEGER, "@integer 1000000, 2c6, 3c6, 4c6, 5c6, 6c6, 7c6, …", false, + DecimalQuantity_DualStorageBCD.fromExponentString("1000000")); + checkNewSamples(description, test, "many", PluralRules.SampleType.DECIMAL, "@decimal 2.1c6, 3.1c6, 4.1c6, 5.1c6, 6.1c6, 7.1c6, …", false, + DecimalQuantity_DualStorageBCD.fromExponentString("2.1c6")); + checkNewSamples(description, test, "other", PluralRules.SampleType.INTEGER, "@integer 2~17, 100, 1000, 10000, 100000, 2c5, 3c5, 4c5, 5c5, 6c5, 7c5, …", false, + DecimalQuantity_DualStorageBCD.fromExponentString("2")); + checkNewSamples(description, test, "other", PluralRules.SampleType.DECIMAL, "@decimal 2.0~3.5, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 2.1c5, 3.1c5, 4.1c5, 5.1c5, 6.1c5, 7.1c5, …", false, + DecimalQuantity_DualStorageBCD.fromExponentString("2.0")); } public void checkOldSamples(String description, PluralRules rules, String keyword, SampleType sampleType, - Double... expected) { - Collection oldSamples = rules.getSamples(keyword, sampleType); - if (!assertEquals("getOldSamples; " + keyword + "; " + description, new HashSet(Arrays.asList(expected)), - oldSamples)) { + DecimalQuantity... expected) { + Collection oldSamples = rules.getDecimalQuantitySamples(keyword, sampleType); + + // Collect actual (oldSamples) and expected (expectedSamplesList) into the + // same concrete collection for comparison purposes. + ArrayList oldSamplesList = new ArrayList(oldSamples); + ArrayList expectedSamplesList = new ArrayList(Arrays.asList(expected)); + + if (!assertEquals("getOldSamples; " + keyword + "; " + description, expectedSamplesList, + oldSamplesList)) { rules.getSamples(keyword, sampleType); } } public void checkNewSamples(String description, PluralRules test, String keyword, SampleType sampleType, - String samplesString, boolean isBounded, FixedDecimal firstInRange) { + String samplesString, boolean isBounded, DecimalQuantity firstInRange) { String title = description + ", " + sampleType; - FixedDecimalSamples samples = test.getDecimalSamples(keyword, sampleType); + DecimalQuantitySamples samples = test.getDecimalSamples(keyword, sampleType); if (samples != null) { + + // For now, skip round-trip formatting test when samples string uses + // 'e' instead of 'c' for compact notation. + + // We are skipping tests for 'e' by replacing 'e' with 'c' in exponent + // notation. + Pattern p = Pattern.compile("(\\d+)(e)([-]?\\d+)"); + Matcher m = p.matcher(samplesString); + if (m.find()) { + samplesString = m.replaceAll("$1c$3"); + } + assertEquals("samples; " + title, samplesString, samples.toString()); assertEquals("bounded; " + title, isBounded, samples.bounded); assertEquals("first; " + title, firstInRange, samples.samples.iterator().next().start); @@ -391,11 +429,11 @@ public class PluralRulesTest extends TestFmwk { main: for (ULocale locale : factory.getAvailableULocales()) { PluralRules rules = factory.forLocale(locale); Map keywordToRule = new HashMap<>(); - Collection samples = new LinkedHashSet<>(); + Collection samples = new LinkedHashSet<>(); for (String keyword : rules.getKeywords()) { for (SampleType sampleType : SampleType.values()) { - FixedDecimalSamples samples2 = rules.getDecimalSamples(keyword, sampleType); + DecimalQuantitySamples samples2 = rules.getDecimalSamples(keyword, sampleType); if (samples2 != null) { samples.add(samples2); } @@ -412,15 +450,16 @@ public class PluralRulesTest extends TestFmwk { } keywordToRule.put(keyword, singleRule); } - Map collisionTest = new TreeMap(); - for (FixedDecimalSamples sample3 : samples) { - Set samples2 = sample3.getSamples(); + + Map collisionTest = new LinkedHashMap(); + for (DecimalQuantitySamples sample3 : samples) { + Set samples2 = sample3.getSamples(); if (samples2 == null) { continue; } - for (FixedDecimalRange sample : samples2) { + for (DecimalQuantitySamplesRange sample : samples2) { for (int i = 0; i < 1; ++i) { - FixedDecimal item = i == 0 ? sample.start : sample.end; + DecimalQuantity item = i == 0 ? sample.start : sample.end; collisionTest.clear(); for (Entry entry : keywordToRule.entrySet()) { PluralRules rule = entry.getValue(); @@ -460,9 +499,9 @@ public class PluralRulesTest extends TestFmwk { } public void checkValue(String title1, PluralRules rules, String expected, String value) { - FixedDecimal fdNum = new FixedDecimal(value); + DecimalQuantity dqNum = DecimalQuantity_DualStorageBCD.fromExponentString(value); - String result = rules.select(fdNum); + String result = rules.select(dqNum); ULocale locale = null; assertEquals(getAssertMessage(title1, locale, rules, expected) + "; value: " + value, expected, result); } @@ -733,13 +772,20 @@ public class PluralRulesTest extends TestFmwk { } } - private void assertRuleValue(String rule, double value) { + private void assertRuleValue(String rule, DecimalQuantity value) { assertRuleKeyValue("a:" + rule, "a", value); } - private void assertRuleKeyValue(String rule, String key, double value) { + private void assertRuleKeyValue(String rule, String key, DecimalQuantity value) { PluralRules pr = PluralRules.createRules(rule); - assertEquals(rule, value, pr.getUniqueKeywordValue(key)); + + // as a DecimalQuantity + assertEquals(rule, value, pr.getUniqueKeywordDecimalQuantityValue(key)); + + // as a double + double expDouble = value.equals(PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY) ? + PluralRules.NO_UNIQUE_VALUE : value.toDouble(); + assertEquals(rule, expDouble, pr.getUniqueKeywordValue(key)); } /* @@ -747,26 +793,36 @@ public class PluralRulesTest extends TestFmwk { */ @Test public void TestGetUniqueKeywordValue() { - assertRuleKeyValue("a: n is 1", "not_defined", PluralRules.NO_UNIQUE_VALUE); // key not defined - assertRuleValue("n within 2..2", 2); - assertRuleValue("n is 1", 1); - assertRuleValue("n in 2..2", 2); - assertRuleValue("n in 3..4", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n within 3..4", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n is 2 or n is 2", 2); - assertRuleValue("n is 2 and n is 2", 2); - assertRuleValue("n is 2 or n is 3", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n is 2 and n is 3", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n is 2 or n in 2..3", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n is 2 and n in 2..3", 2); - assertRuleKeyValue("a: n is 1", "other", PluralRules.NO_UNIQUE_VALUE); // key matches default rule - assertRuleValue("n in 2,3", PluralRules.NO_UNIQUE_VALUE); - assertRuleValue("n in 2,3..6 and n not in 2..3,5..6", 4); + LocalizedNumberFormatter fmtr = NumberFormatter.withLocale(ULocale.ROOT); + + assertRuleKeyValue("a: n is 1", "not_defined", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); // key not defined + assertRuleValue("n within 2..2", new DecimalQuantity_DualStorageBCD(2)); + assertRuleValue("n is 1", new DecimalQuantity_DualStorageBCD(1)); + assertRuleValue("n in 2..2", new DecimalQuantity_DualStorageBCD(2)); + assertRuleValue("n in 3..4", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n within 3..4", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n is 2 or n is 2", new DecimalQuantity_DualStorageBCD(2)); + assertRuleValue("n is 2 and n is 2", new DecimalQuantity_DualStorageBCD(2)); + assertRuleValue("n is 2 or n is 3", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n is 2 and n is 3", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n is 2 or n in 2..3", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n is 2 and n in 2..3", new DecimalQuantity_DualStorageBCD(2)); + assertRuleKeyValue("a: n is 1", "other", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); // key matches default rule + assertRuleValue("n in 2,3", PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); + assertRuleValue("n in 2,3..6 and n not in 2..3,5..6", new DecimalQuantity_DualStorageBCD(4)); } /** * The version in PluralFormatUnitTest is not really a test, and it's in the wrong place anyway, so I'm putting a * variant of it here. + * + * Using the double API for getting plural samples, assert all samples match the keyword + * they are listed under, for all locales. + * + * Specifically, iterate over all locales, get plural rules for the locale, iterate over every rule, + * then iterate over every sample in the rule, parse sample to a number (double), use that number + * as an input to .select() for the rules object, and assert the actual return plural keyword matches + * what we expect based on the plural rule string. */ @Test public void TestGetSamples() { @@ -775,10 +831,6 @@ public class PluralRulesTest extends TestFmwk { uniqueRuleSet.add(PluralRules.getFunctionalEquivalent(locale, null)); } for (ULocale locale : uniqueRuleSet) { - //if (locale.getLanguage().equals("fr") && - // logKnownIssue("21322", "PluralRules::getSamples cannot distinguish 1e5 from 100000")) { - // continue; - //} PluralRules rules = factory.forLocale(locale); logln("\nlocale: " + (locale == ULocale.ROOT ? "root" : locale.toString()) + ", rules: " + rules); Set keywords = rules.getKeywords(); @@ -791,8 +843,8 @@ public class PluralRulesTest extends TestFmwk { if (list.size() == 0) { // when the samples (meaning integer samples) are null, then then integerSamples must be, and the // decimalSamples must not be - FixedDecimalSamples integerSamples = rules.getDecimalSamples(keyword, SampleType.INTEGER); - FixedDecimalSamples decimalSamples = rules.getDecimalSamples(keyword, SampleType.DECIMAL); + DecimalQuantitySamples integerSamples = rules.getDecimalSamples(keyword, SampleType.INTEGER); + DecimalQuantitySamples decimalSamples = rules.getDecimalSamples(keyword, SampleType.DECIMAL); assertTrue(getAssertMessage("List is not null", locale, rules, keyword), integerSamples == null && decimalSamples != null && decimalSamples.samples.size() != 0); } else { @@ -816,6 +868,131 @@ public class PluralRulesTest extends TestFmwk { } } + /** + * This replicates the setup of TestGetSamples(), but parses samples as DecimalQuantity instead of double. + * + * Using the DecimalQuantity API for getting plural samples, assert all samples match the keyword + * they are listed under, for all locales. + * + * Specifically, iterate over all locales, get plural rules for the locale, iterate over every rule, + * then iterate over every sample in the rule, parse sample to a number (DecimalQuantity), use that number + * as an input to .select() for the rules object, and assert the actual return plural keyword matches + * what we expect based on the plural rule string. + */ + @Test + public void TestGetDecimalQuantitySamples() { + Set uniqueRuleSet = new HashSet<>(); + for (ULocale locale : factory.getAvailableULocales()) { + uniqueRuleSet.add(PluralRules.getFunctionalEquivalent(locale, null)); + } + for (ULocale locale : uniqueRuleSet) { + PluralRules rules = factory.forLocale(locale); + logln("\nlocale: " + (locale == ULocale.ROOT ? "root" : locale.toString()) + ", rules: " + rules); + Set keywords = rules.getKeywords(); + for (String keyword : keywords) { + Collection list = rules.getDecimalQuantitySamples(keyword); + logln("keyword: " + keyword + ", samples: " + list); + // with fractions, the samples can be empty and thus the list null. In that case, however, there will be + // FixedDecimal values. + // So patch the test for that. + if (list.size() == 0) { + // when the samples (meaning integer samples) are null, then then integerSamples must be, and the + // decimalSamples must not be + DecimalQuantitySamples integerSamples = rules.getDecimalSamples(keyword, SampleType.INTEGER); + DecimalQuantitySamples decimalSamples = rules.getDecimalSamples(keyword, SampleType.DECIMAL); + assertTrue(getAssertMessage("List is not null", locale, rules, keyword), integerSamples == null + && decimalSamples != null && decimalSamples.samples.size() != 0); + } else { + if (!assertTrue(getAssertMessage("Test getSamples.isEmpty", locale, rules, keyword), + !list.isEmpty())) { + rules.getDecimalQuantitySamples(keyword); + } + if (rules.toString().contains(": j")) { + // hack until we remove j + } else { + for (DecimalQuantity value : list) { + assertEquals(getAssertMessage("Match keyword", locale, rules, keyword) + "; value '" + + value + "'", keyword, rules.select(value)); + } + } + } + } + + assertNull(locale + ", list is null", rules.getDecimalQuantitySamples("@#$%^&*")); + assertNull(locale + ", list is null", rules.getDecimalQuantitySamples("@#$%^&*", SampleType.DECIMAL)); + } + } + + /** + * Test addSamples (Java) / getSamplesFromString (C++) to ensure the expansion of plural rule sample range + * expands to a sequence of sample numbers that is incremented as the right scale. + * + * Do this for numbers with fractional digits but no exponent. + */ + @Test + public void testGetOrAddSamplesFromString() { + PluralRules rules = PluralRules.createRules("testkeyword: e != 0 @decimal 2.0~4.0, …"); + + Set keywords = rules.getKeywords(); + assertTrue("At least parse the test keyword in the test rule string", 0 < keywords.size()); + + String expKeyword = "testkeyword"; + Collection list = rules.getDecimalQuantitySamples(expKeyword, SampleType.DECIMAL); + + String[] expDqStrs = { + "2.0", "2.1", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8", "2.9", + "3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", + "4.0" + }; + assertEquals("Number of parsed samples from test string incorrect", expDqStrs.length, list.size()); + ArrayList actSamples = new ArrayList<>(list); + for (int i = 0; i < list.size(); i++) { + String expDqStr = expDqStrs[i]; + DecimalQuantity sample = actSamples.get(i); + String sampleStr = sample.toExponentString(); + + assertEquals("Expansion of sample range to sequence of sample values should increment at the right scale", + expDqStr, sampleStr); + + } + } + + /** + * Test addSamples (Java) / getSamplesFromString (C++) to ensure the expansion of plural rule sample range + * expands to a sequence of sample numbers that is incremented as the right scale. + * + * Do this for numbers written in a notation that has an exponent, for which the number is an + * integer (also as defined in the UTS 35 spec for the plural operands) but whose representation + * has fractional digits in the significand written before the exponent. + */ + @Test + public void testGetOrAddSamplesFromStringCompactNotation() { + PluralRules rules = PluralRules.createRules("testkeyword: e != 0 @decimal 2.0c6~4.0c6, …"); + + Set keywords = rules.getKeywords(); + assertTrue("At least parse the test keyword in the test rule string", 0 < keywords.size()); + + String expKeyword = "testkeyword"; + Collection list = rules.getDecimalQuantitySamples(expKeyword, SampleType.DECIMAL); + + String[] expDqStrs = { + "2.0c6", "2.1c6", "2.2c6", "2.3c6", "2.4c6", "2.5c6", "2.6c6", "2.7c6", "2.8c6", "2.9c6", + "3.0c6", "3.1c6", "3.2c6", "3.3c6", "3.4c6", "3.5c6", "3.6c6", "3.7c6", "3.8c6", "3.9c6", + "4.0c6" + }; + assertEquals("Number of parsed samples from test string incorrect", expDqStrs.length, list.size()); + ArrayList actSamples = new ArrayList<>(list); + for (int i = 0; i < list.size(); i++) { + String expDqStr = expDqStrs[i]; + DecimalQuantity sample = actSamples.get(i); + String sampleStr = sample.toExponentString(); + + assertEquals("Expansion of sample range to sequence of sample values should increment at the right scale", + expDqStr, sampleStr); + + } + } + public String getAssertMessage(String message, ULocale locale, PluralRules rules, String keyword) { String ruleString = ""; if (keyword != null) { @@ -882,24 +1059,42 @@ public class PluralRulesTest extends TestFmwk { if (valueList != null) { valueList = valueList.trim(); } - Collection values; + Collection values; if (valueList == null || valueList.length() == 0) { values = Collections.EMPTY_SET; } else if ("null".equals(valueList)) { values = null; } else { - values = new TreeSet<>(); + values = new LinkedHashSet<>(); for (String value : valueList.split(",")) { - values.add(Double.parseDouble(value)); + values.add(DecimalQuantity_DualStorageBCD.fromExponentString(value)); } } - Collection results = p.getAllKeywordValues(keyword); - assertEquals(keyword + " in " + ruleDescription, values, results == null ? null : new HashSet(results)); + Collection results = p.getAllKeywordDecimalQuantityValues(keyword); + + // Convert DecimalQuantity using a 1:1 conversion to String for comparison purposes + Set valuesForComparison = new HashSet<>(); + if (values != null) { + for (DecimalQuantity dq : values) { + valuesForComparison.add(dq.toExponentString()); + } + } + Set resultsForComparison = new HashSet<>(); + if (results != null) { + for (DecimalQuantity dq : results) { + resultsForComparison.add(dq.toExponentString()); + } + } + + assertEquals(keyword + " in " + ruleDescription, + values == null ? null : valuesForComparison, + results == null ? null : resultsForComparison + ); if (results != null) { try { - results.add(PluralRules.NO_UNIQUE_VALUE); + results.add(PluralRules.NO_UNIQUE_VALUE_DECIMAL_QUANTITY); fail("returned set is modifiable"); } catch (UnsupportedOperationException e) { // pass @@ -967,13 +1162,9 @@ public class PluralRulesTest extends TestFmwk { for (String keyword : rules.getKeywords()) { boolean isLimited = rules.isLimited(keyword, sampleType); boolean computeLimited = rules.computeLimited(keyword, sampleType); - if (!keyword.equals("other") && !(locale.getLanguage().equals("fr") && logKnownIssue("ICU-21322", "fr plurals many case computeLimited == isLimited"))) { - assertEquals(getAssertMessage("computeLimited == isLimited", locale, rules, keyword), - computeLimited, isLimited); - } - Collection samples = rules.getSamples(keyword, sampleType); + Collection samples = rules.getDecimalQuantitySamples(keyword, sampleType); assertNotNull(getAssertMessage("Samples must not be null", locale, rules, keyword), samples); - /* FixedDecimalSamples decimalSamples = */rules.getDecimalSamples(keyword, sampleType); + rules.getDecimalSamples(keyword, sampleType); // assertNotNull(getAssertMessage("Decimal samples must be null if unlimited", locale, rules, // keyword), decimalSamples); } @@ -985,29 +1176,30 @@ public class PluralRulesTest extends TestFmwk { @Test public void TestKeywords() { Set possibleKeywords = new LinkedHashSet(Arrays.asList("zero", "one", "two", "few", "many", "other")); + DecimalQuantity ONE_INTEGER = DecimalQuantity_DualStorageBCD.fromExponentString("1"); Object[][][] tests = { // format is locale, explicits, then triples of keyword, status, unique value. - { { "en", null }, { "one", KeywordStatus.UNIQUE, 1.0d }, { "other", KeywordStatus.UNBOUNDED, null } }, - { { "pl", null }, { "one", KeywordStatus.UNIQUE, 1.0d }, { "few", KeywordStatus.UNBOUNDED, null }, + { { "en", null }, { "one", KeywordStatus.UNIQUE, ONE_INTEGER }, { "other", KeywordStatus.UNBOUNDED, null } }, + { { "pl", null }, { "one", KeywordStatus.UNIQUE, ONE_INTEGER }, { "few", KeywordStatus.UNBOUNDED, null }, { "many", KeywordStatus.UNBOUNDED, null }, { "other", KeywordStatus.SUPPRESSED, null, KeywordStatus.UNBOUNDED, null } // note that it is // suppressed in // INTEGER but not // DECIMAL - }, { { "en", new HashSet<>(Arrays.asList(1.0d)) }, // check that 1 is suppressed + }, { { "en", new HashSet<>(Arrays.asList(ONE_INTEGER)) }, // check that 1 is suppressed { "one", KeywordStatus.SUPPRESSED, null }, { "other", KeywordStatus.UNBOUNDED, null } }, }; - Output uniqueValue = new Output<>(); + Output uniqueValue = new Output<>(); for (Object[][] test : tests) { ULocale locale = new ULocale((String) test[0][0]); // NumberType numberType = (NumberType) test[1]; - Set explicits = (Set) test[0][1]; + Set explicits = (Set) test[0][1]; PluralRules pluralRules = factory.forLocale(locale); LinkedHashSet remaining = new LinkedHashSet(possibleKeywords); for (int i = 1; i < test.length; ++i) { Object[] row = test[i]; String keyword = (String) row[0]; KeywordStatus statusExpected = (KeywordStatus) row[1]; - Double uniqueExpected = (Double) row[2]; + DecimalQuantity uniqueExpected = (DecimalQuantity) row[2]; remaining.remove(keyword); KeywordStatus status = pluralRules.getKeywordStatus(keyword, 0, explicits, uniqueValue); assertEquals(getAssertMessage("Unique Value", locale, pluralRules, keyword), uniqueExpected, @@ -1015,7 +1207,7 @@ public class PluralRulesTest extends TestFmwk { assertEquals(getAssertMessage("Keyword Status", locale, pluralRules, keyword), statusExpected, status); if (row.length > 3) { statusExpected = (KeywordStatus) row[3]; - uniqueExpected = (Double) row[4]; + uniqueExpected = (DecimalQuantity) row[4]; status = pluralRules.getKeywordStatus(keyword, 0, explicits, uniqueValue, SampleType.DECIMAL); assertEquals(getAssertMessage("Unique Value - decimal", locale, pluralRules, keyword), uniqueExpected, uniqueValue.value); @@ -1033,6 +1225,11 @@ public class PluralRulesTest extends TestFmwk { // For the time being, the compact notation exponent operand `c` is an alias // for the scientific exponent operand `e` and compact notation. + /** + * Test the proper plural rule keyword selection given an input number that is + * already formatted into scientific notation. This exercises the `e` plural operand + * for the formatted number. + */ @Test public void testScientificPluralKeyword() { PluralRules rules = PluralRules.createRules("one: i = 0,1 @integer 0, 1 @decimal 0.0~1.5; many: e = 0 and i % 1000000 = 0 and v = 0 or " + @@ -1082,6 +1279,11 @@ public class PluralRulesTest extends TestFmwk { } } + /** + * Test the proper plural rule keyword selection given an input number that is + * already formatted into compact notation. This exercises the `c` plural operand + * for the formatted number. + */ @Test public void testCompactDecimalPluralKeyword() { PluralRules rules = PluralRules.createRules("one: i = 0,1 @integer 0, 1 @decimal 0.0~1.5; many: c = 0 and i % 1000000 = 0 and v = 0 or " + @@ -1261,8 +1463,13 @@ public class PluralRulesTest extends TestFmwk { @Test public void TestLocales() { + // This test will fail when the locale snapshot gets out of sync with the real CLDR data. + // In that case, temporarily use "if (true)", + // copy & paste the output into the initializer above, + // and revert to "if (false)" for normal testing. if (false) { generateLOCALE_SNAPSHOT(); + return; } for (String test : LOCALE_SNAPSHOT) { test = test.trim(); @@ -1308,7 +1515,7 @@ public class PluralRulesTest extends TestFmwk { System.out.print(" \"" + CollectionUtilities.join(locales, ",")); for (StandardPluralCategories spc : set) { String keyword = spc.toString(); - FixedDecimalSamples samples = rule.getDecimalSamples(keyword, SampleType.INTEGER); + DecimalQuantitySamples samples = rule.getDecimalSamples(keyword, SampleType.INTEGER); System.out.print("; " + spc + ": " + samples); } System.out.println("\",");