mirror of
https://github.com/unicode-org/icu.git
synced 2025-04-06 14:05:32 +00:00
allenwtsu
Markus Scherer
parent
23081486ff
commit
d0290c03db
16 changed files with 563 additions and 17 deletions
|
|
@ -79,6 +79,7 @@ UnhandledEngine::findBreaks( UText *text,
|
|||
int32_t /* startPos */,
|
||||
int32_t endPos,
|
||||
UVector32 &/*foundBreaks*/,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode &status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
UChar32 c = utext_current32(text);
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ class LanguageBreakEngine : public UMemory {
|
|||
int32_t startPos,
|
||||
int32_t endPos,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode &status) const = 0;
|
||||
|
||||
};
|
||||
|
|
@ -194,6 +195,7 @@ class UnhandledEngine : public LanguageBreakEngine {
|
|||
int32_t startPos,
|
||||
int32_t endPos,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode &status) const override;
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@
|
|||
#include "unicode/ures.h"
|
||||
#include "unicode/ustring.h"
|
||||
#include "unicode/filteredbrk.h"
|
||||
#include "bytesinkutil.h"
|
||||
#include "ucln_cmn.h"
|
||||
#include "cstring.h"
|
||||
#include "umutex.h"
|
||||
|
|
@ -115,7 +116,7 @@ BreakIterator::buildInstance(const Locale& loc, const char *type, UErrorCode &st
|
|||
}
|
||||
|
||||
// Create a RuleBasedBreakIterator
|
||||
result = new RuleBasedBreakIterator(file, status);
|
||||
result = new RuleBasedBreakIterator(file, uprv_strcmp(type, "line_phrase") == 0, status);
|
||||
|
||||
// If there is a result, set the valid locale and actual locale, and the kind
|
||||
if (U_SUCCESS(status) && result != NULL) {
|
||||
|
|
@ -430,16 +431,24 @@ BreakIterator::makeInstance(const Locale& loc, int32_t kind, UErrorCode& status)
|
|||
{
|
||||
UTRACE_ENTRY(UTRACE_UBRK_CREATE_LINE);
|
||||
uprv_strcpy(lbType, "line");
|
||||
char lbKeyValue[kKeyValueLenMax] = {0};
|
||||
UErrorCode kvStatus = U_ZERO_ERROR;
|
||||
int32_t kLen = loc.getKeywordValue("lb", lbKeyValue, kKeyValueLenMax, kvStatus);
|
||||
if (U_SUCCESS(kvStatus) && kLen > 0 && (uprv_strcmp(lbKeyValue,"strict")==0 || uprv_strcmp(lbKeyValue,"normal")==0 || uprv_strcmp(lbKeyValue,"loose")==0)) {
|
||||
CharString value;
|
||||
CharStringByteSink valueSink(&value);
|
||||
loc.getKeywordValue("lb", valueSink, kvStatus);
|
||||
if (U_SUCCESS(kvStatus) && (value == "strict" || value == "normal" || value == "loose")) {
|
||||
uprv_strcat(lbType, "_");
|
||||
uprv_strcat(lbType, lbKeyValue);
|
||||
uprv_strcat(lbType, value.data());
|
||||
} else {
|
||||
value.clear();
|
||||
loc.getKeywordValue("lw", valueSink, kvStatus);
|
||||
if (U_SUCCESS(kvStatus) && value == "phrase") {
|
||||
uprv_strcat(lbType, "_");
|
||||
uprv_strcat(lbType, value.data());
|
||||
}
|
||||
}
|
||||
result = BreakIterator::buildInstance(loc, lbType, status);
|
||||
|
||||
UTRACE_DATA1(UTRACE_INFO, "lb=%s", lbKeyValue);
|
||||
UTRACE_DATA1(UTRACE_INFO, "lb=%s", value.data());
|
||||
UTRACE_EXIT_STATUS(status);
|
||||
}
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,10 @@
|
|||
#include "dictbe.h"
|
||||
#include "unicode/uniset.h"
|
||||
#include "unicode/chariter.h"
|
||||
#include "unicode/resbund.h"
|
||||
#include "unicode/ubrk.h"
|
||||
#include "unicode/usetiter.h"
|
||||
#include "ubrkimpl.h"
|
||||
#include "utracimp.h"
|
||||
#include "uvectr32.h"
|
||||
#include "uvector.h"
|
||||
|
|
@ -48,6 +51,7 @@ DictionaryBreakEngine::findBreaks( UText *text,
|
|||
int32_t startPos,
|
||||
int32_t endPos,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
(void)startPos; // TODO: remove this param?
|
||||
|
|
@ -68,7 +72,7 @@ DictionaryBreakEngine::findBreaks( UText *text,
|
|||
}
|
||||
rangeStart = start;
|
||||
rangeEnd = current;
|
||||
result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks, status);
|
||||
result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks, isPhraseBreaking, status);
|
||||
utext_setNativeIndex(text, current);
|
||||
|
||||
return result;
|
||||
|
|
@ -230,6 +234,7 @@ ThaiBreakEngine::divideUpDictionaryRange( UText *text,
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode& status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
utext_setNativeIndex(text, rangeStart);
|
||||
|
|
@ -469,6 +474,7 @@ LaoBreakEngine::divideUpDictionaryRange( UText *text,
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode& status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
if ((rangeEnd - rangeStart) < LAO_MIN_WORD_SPAN) {
|
||||
|
|
@ -661,6 +667,7 @@ BurmeseBreakEngine::divideUpDictionaryRange( UText *text,
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode& status ) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
if ((rangeEnd - rangeStart) < BURMESE_MIN_WORD_SPAN) {
|
||||
|
|
@ -866,6 +873,7 @@ KhmerBreakEngine::divideUpDictionaryRange( UText *text,
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode& status ) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
if ((rangeEnd - rangeStart) < KHMER_MIN_WORD_SPAN) {
|
||||
|
|
@ -1053,6 +1061,10 @@ CjkBreakEngine::CjkBreakEngine(DictionaryMatcher *adoptDictionary, LanguageType
|
|||
// Korean dictionary only includes Hangul syllables
|
||||
fHangulWordSet.applyPattern(UnicodeString(u"[\\uac00-\\ud7a3]"), status);
|
||||
fHangulWordSet.compact();
|
||||
fNumberOrOpenPunctuationSet.applyPattern(UnicodeString(u"[[:Nd:][:Pi:][:Ps:]]"), status);
|
||||
fNumberOrOpenPunctuationSet.compact();
|
||||
fClosePunctuationSet.applyPattern(UnicodeString(u"[[:Pc:][:Pd:][:Pe:][:Pf:][:Po:]]"), status);
|
||||
fClosePunctuationSet.compact();
|
||||
|
||||
// handle Korean and Japanese/Chinese using different dictionaries
|
||||
if (type == kKorean) {
|
||||
|
|
@ -1063,6 +1075,7 @@ CjkBreakEngine::CjkBreakEngine(DictionaryMatcher *adoptDictionary, LanguageType
|
|||
UnicodeSet cjSet(UnicodeString(u"[[:Han:][:Hiragana:][:Katakana:]\\u30fc\\uff70\\uff9e\\uff9f]"), status);
|
||||
if (U_SUCCESS(status)) {
|
||||
setCharacters(cjSet);
|
||||
initJapanesePhraseParameter(status);
|
||||
}
|
||||
}
|
||||
UTRACE_EXIT_STATUS(status);
|
||||
|
|
@ -1090,14 +1103,12 @@ static inline bool isKatakana(UChar32 value) {
|
|||
(value >= 0xFF66 && value <= 0xFF9f);
|
||||
}
|
||||
|
||||
|
||||
// Function for accessing internal utext flags.
|
||||
// Replicates an internal UText function.
|
||||
|
||||
static inline int32_t utext_i32_flag(int32_t bitIndex) {
|
||||
return (int32_t)1 << bitIndex;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* @param text A UText representing the text
|
||||
|
|
@ -1111,6 +1122,7 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
if (rangeStart >= rangeEnd) {
|
||||
|
|
@ -1341,6 +1353,26 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
|
|||
if ((uint32_t)bestSnlp.elementAti(numCodePts) == kuint32max) {
|
||||
t_boundary.addElement(numCodePts, status);
|
||||
numBreaks++;
|
||||
} else if (isPhraseBreaking) {
|
||||
t_boundary.addElement(numCodePts, status);
|
||||
if(U_SUCCESS(status)) {
|
||||
numBreaks++;
|
||||
int32_t prevIdx = numCodePts;
|
||||
|
||||
int32_t codeUnitIdx = -1;
|
||||
int32_t length = -1;
|
||||
for (int32_t i = prev.elementAti(numCodePts); i > 0; i = prev.elementAti(i)) {
|
||||
codeUnitIdx = inString.moveIndex32(0, i);
|
||||
// Calculate the length by using the code unit.
|
||||
length = inString.moveIndex32(0, prevIdx) - codeUnitIdx;
|
||||
prevIdx = i;
|
||||
// Skip the breakpoint if it belongs to the particle or Hiragana.
|
||||
if (!fSkipSet.containsKey(inString.tempSubString(codeUnitIdx, length))) {
|
||||
t_boundary.addElement(i, status);
|
||||
numBreaks++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int32_t i = numCodePts; i > 0; i = prev.elementAti(i)) {
|
||||
t_boundary.addElement(i, status);
|
||||
|
|
@ -1361,8 +1393,8 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
|
|||
// while reversing t_boundary and pushing values to foundBreaks.
|
||||
int32_t prevCPPos = -1;
|
||||
int32_t prevUTextPos = -1;
|
||||
int correctedNumBreaks = 0;
|
||||
for (int32_t i = numBreaks-1; i >= 0; i--) {
|
||||
int32_t correctedNumBreaks = 0;
|
||||
for (int32_t i = numBreaks - 1; i >= 0; i--) {
|
||||
int32_t cpPos = t_boundary.elementAti(i);
|
||||
U_ASSERT(cpPos > prevCPPos);
|
||||
int32_t utextPos = inputMap.isValid() ? inputMap->elementAti(cpPos) : cpPos + rangeStart;
|
||||
|
|
@ -1370,7 +1402,12 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
|
|||
if (utextPos > prevUTextPos) {
|
||||
// Boundaries are added to foundBreaks output in ascending order.
|
||||
U_ASSERT(foundBreaks.size() == 0 || foundBreaks.peeki() < utextPos);
|
||||
if (utextPos != rangeStart) {
|
||||
// In phrase breaking, there has to be a breakpoint between Cj character and close
|
||||
// punctuation.
|
||||
// E.g.［携帯電話］正しい選択 -> ［携帯▁電話］▁正しい▁選択 -> breakpoint between ］ and 正
|
||||
if (utextPos != rangeStart
|
||||
|| (isPhraseBreaking && utextPos > 0
|
||||
&& fClosePunctuationSet.contains(utext_char32At(inText, utextPos - 1)))) {
|
||||
foundBreaks.push(utextPos, status);
|
||||
correctedNumBreaks++;
|
||||
}
|
||||
|
|
@ -1385,15 +1422,54 @@ CjkBreakEngine::divideUpDictionaryRange( UText *inText,
|
|||
}
|
||||
(void)prevCPPos; // suppress compiler warnings about unused variable
|
||||
|
||||
UChar32 nextChar = utext_char32At(inText, rangeEnd);
|
||||
if (!foundBreaks.isEmpty() && foundBreaks.peeki() == rangeEnd) {
|
||||
foundBreaks.popi();
|
||||
correctedNumBreaks--;
|
||||
// In phrase breaking, there has to be a breakpoint between Cj character and
|
||||
// the number/open punctuation.
|
||||
// E.g. る文字「そうだ、京都」->る▁文字▁「そうだ、▁京都」-> breakpoint between 字 and「
|
||||
// E.g. 乗車率９０％程度だろうか -> 乗車▁率▁９０％▁程度だ▁ろうか -> breakpoint between 率 and ９
|
||||
if (isPhraseBreaking) {
|
||||
if (!fNumberOrOpenPunctuationSet.contains(nextChar)) {
|
||||
foundBreaks.popi();
|
||||
correctedNumBreaks--;
|
||||
}
|
||||
} else {
|
||||
foundBreaks.popi();
|
||||
correctedNumBreaks--;
|
||||
}
|
||||
}
|
||||
|
||||
// inString goes out of scope
|
||||
// inputMap goes out of scope
|
||||
return correctedNumBreaks;
|
||||
}
|
||||
|
||||
void CjkBreakEngine::initJapanesePhraseParameter(UErrorCode& error) {
|
||||
loadJapaneseParticleAndAuxVerbs(error);
|
||||
loadHiragana(error);
|
||||
}
|
||||
|
||||
void CjkBreakEngine::loadJapaneseParticleAndAuxVerbs(UErrorCode& error) {
|
||||
const char* tags[2] = { "particles", "auxVerbs" };
|
||||
ResourceBundle ja(U_ICUDATA_BRKITR, "ja", error);
|
||||
if (U_SUCCESS(error)) {
|
||||
for (int32_t i = 0; i < 2; i++) {
|
||||
ResourceBundle bundle = ja.get(tags[i], error);
|
||||
while (U_SUCCESS(error) && bundle.hasNext()) {
|
||||
fSkipSet.puti(bundle.getNextString(error), 1, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void CjkBreakEngine::loadHiragana(UErrorCode& error) {
|
||||
UnicodeSet hiraganaWordSet(UnicodeString(u"[:Hiragana:]"), error);
|
||||
hiraganaWordSet.compact();
|
||||
UnicodeSetIterator iterator(hiraganaWordSet);
|
||||
while (iterator.next()) {
|
||||
fSkipSet.puti(UnicodeString(iterator.getCodepoint()), 1, error);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
U_NAMESPACE_END
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#include "unicode/utext.h"
|
||||
|
||||
#include "brkeng.h"
|
||||
#include "hash.h"
|
||||
#include "uvectr32.h"
|
||||
|
||||
U_NAMESPACE_BEGIN
|
||||
|
|
@ -80,6 +81,7 @@ class DictionaryBreakEngine : public LanguageBreakEngine {
|
|||
int32_t startPos,
|
||||
int32_t endPos,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status ) const override;
|
||||
|
||||
protected:
|
||||
|
|
@ -105,6 +107,7 @@ class DictionaryBreakEngine : public LanguageBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const = 0;
|
||||
|
||||
};
|
||||
|
|
@ -163,6 +166,7 @@ class ThaiBreakEngine : public DictionaryBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
|
||||
};
|
||||
|
|
@ -220,6 +224,7 @@ class LaoBreakEngine : public DictionaryBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
|
||||
};
|
||||
|
|
@ -277,6 +282,7 @@ class BurmeseBreakEngine : public DictionaryBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
|
||||
};
|
||||
|
|
@ -334,6 +340,7 @@ class KhmerBreakEngine : public DictionaryBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
|
||||
};
|
||||
|
|
@ -362,10 +369,22 @@ class CjkBreakEngine : public DictionaryBreakEngine {
|
|||
* @internal
|
||||
*/
|
||||
UnicodeSet fHangulWordSet;
|
||||
UnicodeSet fNumberOrOpenPunctuationSet;
|
||||
UnicodeSet fClosePunctuationSet;
|
||||
|
||||
DictionaryMatcher *fDictionary;
|
||||
const Normalizer2 *nfkcNorm2;
|
||||
|
||||
private:
|
||||
// Load Japanese particles and auxiliary verbs.
|
||||
void loadJapaneseParticleAndAuxVerbs(UErrorCode& error);
|
||||
// Load Japanese Hiragana.
|
||||
void loadHiragana(UErrorCode& error);
|
||||
// Initialize fSkipSet by loading Japanese Hiragana, particles and auxiliary verbs.
|
||||
void initJapanesePhraseParameter(UErrorCode& error);
|
||||
|
||||
Hashtable fSkipSet;
|
||||
|
||||
public:
|
||||
|
||||
/**
|
||||
|
|
@ -397,6 +416,7 @@ class CjkBreakEngine : public DictionaryBreakEngine {
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
|
||||
};
|
||||
|
|
|
|||
|
|
@ -639,6 +639,7 @@ LSTMBreakEngine::divideUpDictionaryRange( UText *text,
|
|||
int32_t startPos,
|
||||
int32_t endPos,
|
||||
UVector32 &foundBreaks,
|
||||
UBool /* isPhraseBreaking */,
|
||||
UErrorCode& status) const {
|
||||
if (U_FAILURE(status)) return 0;
|
||||
int32_t beginFoundBreakSize = foundBreaks.size();
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ protected:
|
|||
int32_t rangeStart,
|
||||
int32_t rangeEnd,
|
||||
UVector32 &foundBreaks,
|
||||
UBool isPhraseBreaking,
|
||||
UErrorCode& status) const override;
|
||||
private:
|
||||
const LSTMData* fData;
|
||||
|
|
|
|||
|
|
@ -82,6 +82,19 @@ RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode
|
|||
}
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------------------
|
||||
//
|
||||
// Constructor from a UDataMemory handle to precompiled break rules
|
||||
// stored in an ICU data file. This construcotr is private API,
|
||||
// only for internal use.
|
||||
//
|
||||
//-------------------------------------------------------------------------------
|
||||
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UBool isPhraseBreaking,
|
||||
UErrorCode &status) : RuleBasedBreakIterator(udm, status)
|
||||
{
|
||||
fIsPhraseBreaking = isPhraseBreaking;
|
||||
}
|
||||
|
||||
//
|
||||
// Construct from precompiled binary rules (tables). This constructor is public API,
|
||||
// taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules().
|
||||
|
|
@ -322,6 +335,7 @@ void RuleBasedBreakIterator::init(UErrorCode &status) {
|
|||
fBreakCache = nullptr;
|
||||
fDictionaryCache = nullptr;
|
||||
fLookAheadMatches = nullptr;
|
||||
fIsPhraseBreaking = false;
|
||||
|
||||
// Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER.
|
||||
// fText = UTEXT_INITIALIZER;
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ void RuleBasedBreakIterator::DictionaryCache::populateDictionary(int32_t startPo
|
|||
// Ask the language object if there are any breaks. It will add them to the cache and
|
||||
// leave the text pointer on the other side of its range, ready to search for the next one.
|
||||
if (lbe != NULL) {
|
||||
foundBreakCount += lbe->findBreaks(text, rangeStart, rangeEnd, fBreaks, status);
|
||||
foundBreakCount += lbe->findBreaks(text, rangeStart, rangeEnd, fBreaks, fBI->fIsPhraseBreaking, status);
|
||||
}
|
||||
|
||||
// Reload the loop variables for the next go-round
|
||||
|
|
|
|||
|
|
@ -147,6 +147,11 @@ private:
|
|||
*/
|
||||
int32_t *fLookAheadMatches;
|
||||
|
||||
/**
|
||||
* A flag to indicate if phrase based breaking is enabled.
|
||||
*/
|
||||
UBool fIsPhraseBreaking;
|
||||
|
||||
//=======================================================================
|
||||
// constructors
|
||||
//=======================================================================
|
||||
|
|
@ -163,6 +168,21 @@ private:
|
|||
*/
|
||||
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
|
||||
|
||||
/**
|
||||
* This constructor uses the udata interface to create a BreakIterator
|
||||
* whose internal tables live in a memory-mapped file. "image" is an
|
||||
* ICU UDataMemory handle for the pre-compiled break iterator tables.
|
||||
* @param image handle to the memory image for the break iterator data.
|
||||
* Ownership of the UDataMemory handle passes to the Break Iterator,
|
||||
* which will be responsible for closing it when it is no longer needed.
|
||||
* @param status Information on any errors encountered.
|
||||
* @param isPhraseBreaking true if phrase based breaking is required, otherwise false.
|
||||
* @see udata_open
|
||||
* @see #getBinaryRules
|
||||
* @internal (private)
|
||||
*/
|
||||
RuleBasedBreakIterator(UDataMemory* image, UBool isPhraseBreaking, UErrorCode &status);
|
||||
|
||||
/** @internal */
|
||||
friend class RBBIRuleBuilder;
|
||||
/** @internal */
|
||||
|
|
|
|||
|
|
@ -7,5 +7,16 @@ ja{
|
|||
line_loose:process(dependency){"line_loose_cj.brk"}
|
||||
line_normal:process(dependency){"line_normal_cj.brk"}
|
||||
line_strict:process(dependency){"line_cj.brk"}
|
||||
line_phrase:process(dependency){"line_phrase_cj.brk"}
|
||||
}
|
||||
particles{
|
||||
か, かしら, から, が, くらい, けれども, こそ,
|
||||
さ, さえ, しか, だけ, だに, だの, て, で, でも,
|
||||
と, ところが, とも, な, など, なり, に, ね, の,
|
||||
ので, のに, は, ば, ばかり, へ, ほど, まで, も,
|
||||
や, やら, よ, より, わ, を
|
||||
}
|
||||
auxVerbs {
|
||||
です, でしょ, でし, ます, ませ, まし
|
||||
}
|
||||
}
|
||||
|
|
|
|||
376
icu4c/source/data/brkitr/rules/line_phrase_cj.txt
Normal file
376
icu4c/source/data/brkitr/rules/line_phrase_cj.txt
Normal file
|
|
@ -0,0 +1,376 @@
|
|||
# Copyright (C) 2022 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
#
|
||||
# file: line_phrase_cj.txt
|
||||
#
|
||||
# Line Breaking Rules
|
||||
# Implement default line breaking as defined by
|
||||
# Unicode Standard Annex #14 (https://www.unicode.org/reports/tr14/)
|
||||
# for Unicode 14.0, with the following modification:
|
||||
#
|
||||
# Boundaries between hyphens and following letters are suppressed when
|
||||
# there is a boundary preceding the hyphen. See rule 20.9
|
||||
#
|
||||
# This corresponds to CSS line-break-word-handling=phrase (BCP47 -u-lw-phrase).
|
||||
# It sets characters of class CJ to behave like NS.
|
||||
# It allows breaking before 201C and after 201D, for zh_Hans, zh_Hant, and ja.
|
||||
#
|
||||
# The content is the same as line_cj.txt except the following
|
||||
# 1. Add CJK into dictionary.
|
||||
# 2. Add East Asian Width with class F, W and H into $ALPlus.
|
||||
#
|
||||
# Character Classes defined by TR 14.
|
||||
#
|
||||
|
||||
!!chain;
|
||||
!!quoted_literals_only;
|
||||
|
||||
$AI = [:LineBreak = Ambiguous:];
|
||||
$AL = [:LineBreak = Alphabetic:];
|
||||
$BA = [:LineBreak = Break_After:];
|
||||
$HH = [\u2010]; # \u2010 is HYPHEN, default line break is BA.
|
||||
$BB = [:LineBreak = Break_Before:];
|
||||
$BK = [:LineBreak = Mandatory_Break:];
|
||||
$B2 = [:LineBreak = Break_Both:];
|
||||
$CB = [:LineBreak = Contingent_Break:];
|
||||
$CJ = [:LineBreak = Conditional_Japanese_Starter:];
|
||||
$CL = [[:LineBreak = Close_Punctuation:] \u201d];
|
||||
# $CM = [:LineBreak = Combining_Mark:];
|
||||
$CP = [:LineBreak = Close_Parenthesis:];
|
||||
$CR = [:LineBreak = Carriage_Return:];
|
||||
$EB = [:LineBreak = EB:];
|
||||
$EM = [:LineBreak = EM:];
|
||||
$EX = [:LineBreak = Exclamation:];
|
||||
$GL = [:LineBreak = Glue:];
|
||||
$HL = [:LineBreak = Hebrew_Letter:];
|
||||
$HY = [:LineBreak = Hyphen:];
|
||||
$H2 = [:LineBreak = H2:];
|
||||
$H3 = [:LineBreak = H3:];
|
||||
$ID = [:LineBreak = Ideographic:];
|
||||
$IN = [:LineBreak = Inseperable:];
|
||||
$IS = [:LineBreak = Infix_Numeric:];
|
||||
$JL = [:LineBreak = JL:];
|
||||
$JV = [:LineBreak = JV:];
|
||||
$JT = [:LineBreak = JT:];
|
||||
$LF = [:LineBreak = Line_Feed:];
|
||||
$NL = [:LineBreak = Next_Line:];
|
||||
# NS includes CJ for CSS strict line breaking.
|
||||
$NS = [[:LineBreak = Nonstarter:] $CJ];
|
||||
$NU = [:LineBreak = Numeric:];
|
||||
$OP = [[:LineBreak = Open_Punctuation:] \u201c];
|
||||
$PO = [:LineBreak = Postfix_Numeric:];
|
||||
$PR = [:LineBreak = Prefix_Numeric:];
|
||||
$QU = [[:LineBreak = Quotation:] - [\u201c\u201d]];
|
||||
$RI = [:LineBreak = Regional_Indicator:];
|
||||
$SA = [:LineBreak = Complex_Context:];
|
||||
$SG = [:LineBreak = Surrogate:];
|
||||
$SP = [:LineBreak = Space:];
|
||||
$SY = [:LineBreak = Break_Symbols:];
|
||||
$WJ = [:LineBreak = Word_Joiner:];
|
||||
$XX = [:LineBreak = Unknown:];
|
||||
$ZW = [:LineBreak = ZWSpace:];
|
||||
$ZWJ = [:LineBreak = ZWJ:];
|
||||
|
||||
# OP30 and CP30 are variants of OP and CP that appear in-line in rule LB30 from UAX 14,
|
||||
# without a formal name. Because ICU rules require multiple uses of the expressions,
|
||||
# give them a single definition with a name
|
||||
|
||||
$EAFWH = [\p{ea=F}\p{ea=W}\p{ea=H}];
|
||||
$OP30 = [$OP - $EAFWH];
|
||||
$CP30 = [$CP - $EAFWH];
|
||||
|
||||
$ExtPictUnassigned = [\p{Extended_Pictographic} & \p{Cn}];
|
||||
|
||||
# By LB9, a ZWJ also behaves as a CM. Including it in the definition of CM avoids having to explicitly
|
||||
# list it in the numerous rules that use CM.
|
||||
# By LB1, SA characters with general categor of Mn or Mc also resolve to CM.
|
||||
|
||||
$CM = [[:LineBreak = Combining_Mark:] $ZWJ [$SA & [[:Mn:][:Mc:]]]];
|
||||
$CMX = [[$CM] - [$ZWJ]];
|
||||
|
||||
# Dictionary character set, for triggering language-based break engines. Currently
|
||||
# limited to LineBreak=Complex_Context (SA) and $dictionaryCJK.
|
||||
|
||||
# Add CJK dictionary
|
||||
$Han = [:Han:];
|
||||
$Katakana = [:Katakana:];
|
||||
$Hiragana = [:Hiragana:];
|
||||
$HangulSyllable = [\uac00-\ud7a3];
|
||||
$ComplexContext = [:LineBreak = Complex_Context:];
|
||||
$KanaKanji = [$Han $Hiragana $Katakana];
|
||||
$dictionaryCJK = [$KanaKanji $HangulSyllable];
|
||||
$dictionary = [$ComplexContext $dictionaryCJK];
|
||||
|
||||
|
||||
#
|
||||
# Rule LB1. By default, treat AI (characters with ambiguous east Asian width),
|
||||
# SA (Dictionary chars, excluding Mn and Mc)
|
||||
# SG (Unpaired Surrogates)
|
||||
# XX (Unknown, unassigned)
|
||||
# as $AL (Alphabetic)
|
||||
#
|
||||
$ALPlus = [$AL $AI $SG $XX $EAFWH [$dictionary-[[:Mn:][:Mc:]]]];
|
||||
|
||||
|
||||
## -------------------------------------------------
|
||||
|
||||
#
|
||||
# CAN_CM is the set of characters that may combine with CM combining chars.
|
||||
# Note that Linebreak UAX 14's concept of a combining char and the rules
|
||||
# for what they can combine with are _very_ different from the rest of Unicode.
|
||||
#
|
||||
# Note that $CM itself is left out of this set. If CM is needed as a base
|
||||
# it must be listed separately in the rule.
|
||||
#
|
||||
$CAN_CM = [^$SP $BK $CR $LF $NL $ZW $CM]; # Bases that can take CMs
|
||||
$CANT_CM = [ $SP $BK $CR $LF $NL $ZW $CM]; # Bases that can't take CMs
|
||||
|
||||
#
|
||||
# AL_FOLLOW set of chars that can unconditionally follow an AL
|
||||
# Needed in rules where stand-alone $CM s are treated as AL.
|
||||
#
|
||||
$AL_FOLLOW = [$BK $CR $LF $NL $ZW $SP $CL $CP $EX $HL $IS $SY $WJ $GL $OP30 $QU $BA $HY $NS $IN $NU $PR $PO $ALPlus];
|
||||
|
||||
|
||||
#
|
||||
# Rule LB 4, 5 Mandatory (Hard) breaks.
|
||||
#
|
||||
$LB4Breaks = [$BK $CR $LF $NL];
|
||||
$LB4NonBreaks = [^$BK $CR $LF $NL $CM];
|
||||
$CR $LF {100};
|
||||
|
||||
#
|
||||
# LB 6 Do not break before hard line breaks.
|
||||
#
|
||||
$LB4NonBreaks? $LB4Breaks {100}; # LB 5 do not break before hard breaks.
|
||||
$CAN_CM $CM* $LB4Breaks {100};
|
||||
^$CM+ $LB4Breaks {100};
|
||||
|
||||
# LB 7 x SP
|
||||
# x ZW
|
||||
$LB4NonBreaks [$SP $ZW];
|
||||
$CAN_CM $CM* [$SP $ZW];
|
||||
^$CM+ [$SP $ZW];
|
||||
|
||||
#
|
||||
# LB 8 Break after zero width space
|
||||
# ZW SP* ÷
|
||||
#
|
||||
$LB8Breaks = [$LB4Breaks $ZW];
|
||||
$LB8NonBreaks = [[$LB4NonBreaks] - [$ZW]];
|
||||
$ZW $SP* / [^$SP $ZW $LB4Breaks];
|
||||
|
||||
# LB 8a ZWJ x Do not break Emoji ZWJ sequences.
|
||||
#
|
||||
$ZWJ [^$CM];
|
||||
|
||||
# LB 9 Combining marks. X $CM needs to behave like X, where X is not $SP, $BK $CR $LF $NL
|
||||
# $CM not covered by the above needs to behave like $AL
|
||||
# See definition of $CAN_CM.
|
||||
|
||||
$CAN_CM $CM+; # Stick together any combining sequences that don't match other rules.
|
||||
^$CM+;
|
||||
|
||||
#
|
||||
# LB 11 Do not break before or after WORD JOINER & related characters.
|
||||
#
|
||||
$CAN_CM $CM* $WJ;
|
||||
$LB8NonBreaks $WJ;
|
||||
^$CM+ $WJ;
|
||||
|
||||
$WJ $CM* .;
|
||||
|
||||
#
|
||||
# LB 12 Do not break after NBSP and related characters.
|
||||
# GL x
|
||||
#
|
||||
$GL $CM* .;
|
||||
|
||||
#
|
||||
# LB 12a Do not break before NBSP and related characters ...
|
||||
# [^SP BA HY] x GL
|
||||
#
|
||||
[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GL;
|
||||
^$CM+ $GL;
|
||||
|
||||
|
||||
|
||||
|
||||
# LB 13 Don't break before ']' or '!' or '/', even after spaces.
|
||||
#
|
||||
$LB8NonBreaks $CL;
|
||||
$CAN_CM $CM* $CL;
|
||||
^$CM+ $CL; # by rule 10, stand-alone CM behaves as AL
|
||||
|
||||
$LB8NonBreaks $CP;
|
||||
$CAN_CM $CM* $CP;
|
||||
^$CM+ $CP; # by rule 10, stand-alone CM behaves as AL
|
||||
|
||||
$LB8NonBreaks $EX;
|
||||
$CAN_CM $CM* $EX;
|
||||
^$CM+ $EX; # by rule 10, stand-alone CM behaves as AL
|
||||
|
||||
$LB8NonBreaks $SY;
|
||||
$CAN_CM $CM* $SY;
|
||||
^$CM+ $SY; # by rule 10, stand-alone CM behaves as AL
|
||||
|
||||
|
||||
#
|
||||
# LB 14 Do not break after OP, even after spaces
|
||||
# Note subtle interaction with "SP IS /" rules in LB14a.
|
||||
# This rule consumes the SP, chaining happens on the IS, effectivley overriding the SP IS rules,
|
||||
# which is the desired behavior.
|
||||
#
|
||||
$OP $CM* $SP* .;
|
||||
|
||||
$OP $CM* $SP+ $CM+ $AL_FOLLOW?; # by rule 10, stand-alone CM behaves as AL
|
||||
# by rule 8, CM following a SP is stand-alone.
|
||||
|
||||
|
||||
# LB 14a Force a break before start of a number with a leading decimal pt, e.g. " .23"
|
||||
# Note: would be simpler to express as "$SP / $IS $CM* $NU;", but ICU rules have limitations.
|
||||
# See issue ICU-20303
|
||||
|
||||
|
||||
$CanFollowIS = [$BK $CR $LF $NL $SP $ZW $WJ $GL $CL $CP $EX $IS $SY $QU $BA $HY $NS $ALPlus $HL $IN];
|
||||
$SP $IS / [^ $CanFollowIS $NU $CM];
|
||||
$SP $IS $CM* $CMX / [^ $CanFollowIS $NU $CM];
|
||||
|
||||
#
|
||||
# LB 14b Do not break before numeric separators (IS), even after spaces.
|
||||
|
||||
[$LB8NonBreaks - $SP] $IS;
|
||||
$SP $IS $CM* [$CanFollowIS {eof}];
|
||||
$SP $IS $CM* $ZWJ [^$CM $NU];
|
||||
|
||||
$CAN_CM $CM* $IS;
|
||||
^$CM+ $IS; # by rule 10, stand-alone CM behaves as AL
|
||||
|
||||
|
||||
# LB 15
|
||||
$QU $CM* $SP* $OP;
|
||||
|
||||
# LB 16
|
||||
($CL | $CP) $CM* $SP* $NS;
|
||||
|
||||
# LB 17
|
||||
$B2 $CM* $SP* $B2;
|
||||
|
||||
#
|
||||
# LB 18 Break after spaces.
|
||||
#
|
||||
$LB18NonBreaks = [$LB8NonBreaks - [$SP]];
|
||||
$LB18Breaks = [$LB8Breaks $SP];
|
||||
|
||||
|
||||
# LB 19
|
||||
# x QU
|
||||
$LB18NonBreaks $CM* $QU;
|
||||
^$CM+ $QU;
|
||||
|
||||
# QU x
|
||||
$QU $CM* .;
|
||||
|
||||
# LB 20
|
||||
# <break> $CB
|
||||
# $CB <break>
|
||||
#
|
||||
$LB20NonBreaks = [$LB18NonBreaks - $CB];
|
||||
|
||||
# LB 20.09 Don't break between Hyphens and Letters when there is a break preceding the hyphen.
|
||||
# Originally added as a Finnish tailoring, now promoted to default ICU behavior.
|
||||
# Note: this is not default UAX-14 behaviour. See issue ICU-8151.
|
||||
#
|
||||
^($HY | $HH) $CM* $ALPlus;
|
||||
|
||||
# LB 21 x (BA | HY | NS)
|
||||
# BB x
|
||||
#
|
||||
$LB20NonBreaks $CM* ($BA | $HY | $NS);
|
||||
|
||||
|
||||
^$CM+ ($BA | $HY | $NS);
|
||||
|
||||
$BB $CM* [^$CB]; # $BB x
|
||||
$BB $CM* $LB20NonBreaks;
|
||||
|
||||
# LB 21a Don't break after Hebrew + Hyphen
|
||||
# HL (HY | BA) x
|
||||
#
|
||||
$HL $CM* ($HY | $BA) $CM* [^$CB]?;
|
||||
|
||||
# LB 21b (forward) Don't break between SY and HL
|
||||
# (break between HL and SY already disallowed by LB 13 above)
|
||||
$SY $CM* $HL;
|
||||
|
||||
# LB 22 Do not break before ellipses
|
||||
#
|
||||
$LB20NonBreaks $CM* $IN;
|
||||
^$CM+ $IN;
|
||||
|
||||
|
||||
# LB 23
|
||||
#
|
||||
($ALPlus | $HL) $CM* $NU;
|
||||
^$CM+ $NU; # Rule 10, any otherwise unattached CM behaves as AL
|
||||
$NU $CM* ($ALPlus | $HL);
|
||||
|
||||
# LB 23a
|
||||
#
|
||||
$PR $CM* ($ID | $EB | $EM);
|
||||
($ID | $EB | $EM) $CM* $PO;
|
||||
|
||||
|
||||
#
|
||||
# LB 24
|
||||
#
|
||||
($PR | $PO) $CM* ($ALPlus | $HL);
|
||||
($ALPlus | $HL) $CM* ($PR | $PO);
|
||||
^$CM+ ($PR | $PO); # Rule 10, any otherwise unattached CM behaves as AL
|
||||
|
||||
#
|
||||
# LB 25 Numbers.
|
||||
#
|
||||
(($PR | $PO) $CM*)? (($OP | $HY) $CM*)? ($IS $CM*)? $NU ($CM* ($NU | $SY | $IS))*
|
||||
($CM* ($CL | $CP))? ($CM* ($PR | $PO))?;
|
||||
|
||||
# LB 26 Do not break a Korean syllable
|
||||
#
|
||||
$JL $CM* ($JL | $JV | $H2 | $H3);
|
||||
($JV | $H2) $CM* ($JV | $JT);
|
||||
($JT | $H3) $CM* $JT;
|
||||
|
||||
# LB 27 Treat korean Syllable Block the same as ID (don't break it)
|
||||
($JL | $JV | $JT | $H2 | $H3) $CM* $PO;
|
||||
$PR $CM* ($JL | $JV | $JT | $H2 | $H3);
|
||||
|
||||
|
||||
# LB 28 Do not break between alphabetics
|
||||
#
|
||||
($ALPlus | $HL) $CM* ($ALPlus | $HL);
|
||||
^$CM+ ($ALPlus | $HL); # The $CM+ is from rule 10, an unattached CM is treated as AL
|
||||
|
||||
# LB 29
|
||||
$IS $CM* ($ALPlus | $HL);
|
||||
|
||||
# LB 30
|
||||
($ALPlus | $HL | $NU) $CM* $OP30;
|
||||
^$CM+ $OP30; # The $CM+ is from rule 10, an unattached CM is treated as AL.
|
||||
$CP30 $CM* ($ALPlus | $HL | $NU);
|
||||
|
||||
# LB 30a Do not break between regional indicators. Break after pairs of them.
|
||||
# Tricky interaction with LB8a: ZWJ x . together with ZWJ acting like a CM.
|
||||
$RI $CM* $RI / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]];
|
||||
$RI $CM* $RI $CM* [$CM-$ZWJ] / [[^$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $CM]];
|
||||
$RI $CM* $RI $CM* [$BK $CR $LF $NL $SP $ZW $WJ $CL $CP $EX $IS $SY $GL $QU $BA $HY $NS $IN $ZWJ {eof}];
|
||||
# note: the preceding rule includes {eof} rather than having the last [set] term qualified with '?'
|
||||
# because of the chain-out behavior difference. The rule must chain out only from the [set characters],
|
||||
# not from the preceding $RI or $CM, which it would be able to do if the set were optional.
|
||||
|
||||
# LB30b Do not break between an emoji base (or potential emoji) and an emoji modifier.
|
||||
$EB $CM* $EM;
|
||||
$ExtPictUnassigned $CM* $EM;
|
||||
|
||||
# LB 31 Break everywhere else.
|
||||
# Match a single code point if no other rule applies.
|
||||
.;
|
||||
|
|
@ -22,6 +22,7 @@
|
|||
<icu:line alt="loose" icu:dependency="line_loose_cj.brk"/>
|
||||
<icu:line alt="normal" icu:dependency="line_normal_cj.brk"/>
|
||||
<icu:line alt="strict" icu:dependency="line_cj.brk"/>
|
||||
<icu:line alt="phrase" icu:dependency="line_phrase_cj.brk"/>
|
||||
</icu:boundaries>
|
||||
</icu:breakIteratorData>
|
||||
</special>
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ group: breakiterator
|
|||
normlzr # for dictbe.o, should switch to Normalizer2
|
||||
uvector32 # for dictbe.o
|
||||
exp_and_tanhf # for lstmbe.o
|
||||
usetiter # for dictbe.o
|
||||
|
||||
group: unormcmp # unorm_compare()
|
||||
unormcmp.o
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ void LSTMBETest::runTestFromFile(const char* filename) {
|
|||
dataerrln("%s:%d Error %s Could not allocate UVextor32", __FILE__, __LINE__, u_errorName(status));
|
||||
return;
|
||||
}
|
||||
engine->findBreaks(&ut, 0, value.length(), actual, status);
|
||||
engine->findBreaks(&ut, 0, value.length(), actual, false, status);
|
||||
if (U_FAILURE(status)) {
|
||||
dataerrln("%s:%d Error %s findBreaks failed", __FILE__, __LINE__, u_errorName(status));
|
||||
return;
|
||||
|
|
@ -288,7 +288,7 @@ void LSTMBETest::runTestWithLargeMemory( const char* model, UScriptCode script)
|
|||
return;
|
||||
}
|
||||
|
||||
engine->findBreaks(&ut, 0, text.length(), actual, status);
|
||||
engine->findBreaks(&ut, 0, text.length(), actual, false, status);
|
||||
utext_close(&ut);
|
||||
text += text;
|
||||
}
|
||||
|
|
|
|||
13
icu4c/source/test/testdata/rbbitst.txt
vendored
13
icu4c/source/test/testdata/rbbitst.txt
vendored
|
|
@ -1884,6 +1884,19 @@ Bangkok)•</data>
|
|||
# woman astronaut, woman astronaut / fitz4
|
||||
<data>•\U0001F469\u200D\U0001F680•\U0001F469\U0001F3FD\u200D\U0001F680\u0020•</data>
|
||||
|
||||
<locale ja@lw=phrase>
|
||||
<line>
|
||||
#[京都観光］時雨殿に行った。-> [京都•観光］•時雨•殿に•行った。•
|
||||
<data>•\uff3b\u4eac\u90fd•\u89b3\u5149\uff3d•\u6642\u96e8•\u6bbf\u306b•\u884c\u3063\u305f\u3002•</data>
|
||||
#９月に東京から友達が遊びに来た -> ９月に•東京から•友達が•遊びに•来た•
|
||||
<data>•\uff19\u6708\u306b•\u6771\u4eac\u304b\u3089•\u53cb\u9054\u304c•\u904a\u3073\u306b•\u6765\u305f•</data>
|
||||
#る文字「そうだ、京都」-> る•文字•「そうだ、•京都」•
|
||||
<data>•\u308b•\u6587\u5b57•\u300c\u305d\u3046\u3060\u3001•\u4eac\u90fd\u300d•</data>
|
||||
#乗車率９０％程度だろうか -> 乗車•率•９０％•程度だ•ろうか•
|
||||
<data>•\u4e57\u8eca•\u7387•\uff19\uff10\uff05•\u7a0b\u5ea6\u3060•\u308d\u3046\u304b\u3002•</data>
|
||||
#［携帯電話］正しい選択 -> ［携帯•電話］•正しい•選択•
|
||||
<data>•\uff3b\u643a\u5e2f•\u96fb\u8a71\uff3d•\u6b63\u3057\u3044•\u9078\u629e•</data>
|
||||
|
||||
|
||||
####################################################################################
|
||||
#
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue