organicmaps/icu
13
Fork 0
mirror of https://github.com/unicode-org/icu.git synced 2025-04-16 18:25:57 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions 10
icu/icu4c/source/common/rbbi.cpp
Andy Heninger 081cf77330 ICU-21662 Improve UVector error handling. 
- Add updated versions of UVector::addElement() and ensureCapacity() that respect
  incoming errors.
  Follow on to c26aebe, which renamed the original versions.

- Add UVector::adoptElement() as a replacement for addElement() when the vector
  has a deleter function set, meaning that it adopts ownership of its elements.

  The intent is to make the behavior clearer at the call sites when looking
  at unfamiliar code.

- Make all functions with an incoming failure, as indicated by a UErrorCode parameter,
  leave the vector unchanged.

- Change all functions that store object pointers into the vector such that,
  when the store cannot be completed for any reason _and_ the vector has a deleter function,
  then the incoming object is deleted.

  This change can simplify the error handling code around calls to the affected functions
  (addElement() and insertElementAt(), in particular)

- Add index bounds checking on functions where it was possible - that is, on functions
  with both U_ErrorCode and index parameters.

- Changed to more modern C++ idioms in some parts of the UVector implementation.

- Review & update as required all uses of the UVector functions
  setElementAt(), insertElementAt(), setSize(), sortedInsert()
  these being the functions with changed behavior on error conditions
  (aside from addElement()).

This PR will be followed by more, switching call sites in various ICU services
from UVector::addElementX() (old behavior on errors)
to   UVector::addElement()  (new behavior on errors)
2021-09-02 19:15:36 -07:00

1302 lines
43 KiB
C++
Raw Permalink Blame History

// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
***************************************************************************
* Copyright (C) 1999-2016 International Business Machines Corporation
* and others. All rights reserved.
***************************************************************************
*/
//
// file: rbbi.cpp Contains the implementation of the rule based break iterator
// runtime engine and the API implementation for
// class RuleBasedBreakIterator
//
#include "utypeinfo.h" // for 'typeid' to work
#include "unicode/utypes.h"
#if !UCONFIG_NO_BREAK_ITERATION
#include <cinttypes>
#include "unicode/rbbi.h"
#include "unicode/schriter.h"
#include "unicode/uchriter.h"
#include "unicode/uclean.h"
#include "unicode/udata.h"
#include "brkeng.h"
#include "ucln_cmn.h"
#include "cmemory.h"
#include "cstring.h"
#include "localsvc.h"
#include "rbbidata.h"
#include "rbbi_cache.h"
#include "rbbirb.h"
#include "uassert.h"
#include "umutex.h"
#include "uvectr32.h"
#ifdef RBBI_DEBUG
static UBool gTrace = FALSE;
#endif
U_NAMESPACE_BEGIN
// The state number of the starting state
constexpr int32_t START_STATE = 1;
// The state-transition value indicating "stop"
constexpr int32_t STOP_STATE = 0;
UOBJECT_DEFINE_RTTI_IMPLEMENTATION(RuleBasedBreakIterator)
//=======================================================================
// constructors
//=======================================================================
/**
* Constructs a RuleBasedBreakIterator that uses the already-created
* tables object that is passed in as a parameter.
*/
RuleBasedBreakIterator::RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status)
: fSCharIter(UnicodeString())
{
init(status);
fData = new RBBIDataWrapper(data, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
fLookAheadMatches = static_cast<int32_t *>(
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
if (fLookAheadMatches == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
}
//
// Construct from precompiled binary rules (tables). This constructor is public API,
// taking the rules as a (const uint8_t *) to match the type produced by getBinaryRules().
//
RuleBasedBreakIterator::RuleBasedBreakIterator(const uint8_t *compiledRules,
uint32_t ruleLength,
UErrorCode &status)
: fSCharIter(UnicodeString())
{
init(status);
if (U_FAILURE(status)) {
return;
}
if (compiledRules == NULL || ruleLength < sizeof(RBBIDataHeader)) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
const RBBIDataHeader *data = (const RBBIDataHeader *)compiledRules;
if (data->fLength > ruleLength) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return;
}
fData = new RBBIDataWrapper(data, RBBIDataWrapper::kDontAdopt, status);
if (U_FAILURE(status)) {return;}
if(fData == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
fLookAheadMatches = static_cast<int32_t *>(
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
if (fLookAheadMatches == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
}
//-------------------------------------------------------------------------------
//
// Constructor from a UDataMemory handle to precompiled break rules
// stored in an ICU data file.
//
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(UDataMemory* udm, UErrorCode &status)
: fSCharIter(UnicodeString())
{
init(status);
fData = new RBBIDataWrapper(udm, status); // status checked in constructor
if (U_FAILURE(status)) {return;}
if(fData == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
if (fData->fForwardTable->fLookAheadResultsSize > 0) {
fLookAheadMatches = static_cast<int32_t *>(
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
if (fLookAheadMatches == nullptr) {
status = U_MEMORY_ALLOCATION_ERROR;
return;
}
}
}
//-------------------------------------------------------------------------------
//
// Constructor from a set of rules supplied as a string.
//
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator( const UnicodeString &rules,
UParseError &parseError,
UErrorCode &status)
: fSCharIter(UnicodeString())
{
init(status);
if (U_FAILURE(status)) {return;}
RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)
RBBIRuleBuilder::createRuleBasedBreakIterator(rules, &parseError, status);
// Note: This is a bit awkward. The RBBI ruleBuilder has a factory method that
// creates and returns a complete RBBI. From here, in a constructor, we
// can't just return the object created by the builder factory, hence
// the assignment of the factory created object to "this".
if (U_SUCCESS(status)) {
*this = *bi;
delete bi;
}
}
//-------------------------------------------------------------------------------
//
// Default Constructor. Create an empty shell that can be set up later.
// Used when creating a RuleBasedBreakIterator from a set
// of rules.
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator()
: fSCharIter(UnicodeString())
{
UErrorCode status = U_ZERO_ERROR;
init(status);
}
//-------------------------------------------------------------------------------
//
// Copy constructor. Will produce a break iterator with the same behavior,
// and which iterates over the same text, as the one passed in.
//
//-------------------------------------------------------------------------------
RuleBasedBreakIterator::RuleBasedBreakIterator(const RuleBasedBreakIterator& other)
: BreakIterator(other),
fSCharIter(UnicodeString())
{
UErrorCode status = U_ZERO_ERROR;
this->init(status);
*this = other;
}
/**
* Destructor
*/
RuleBasedBreakIterator::~RuleBasedBreakIterator() {
if (fCharIter != &fSCharIter) {
// fCharIter was adopted from the outside.
delete fCharIter;
}
fCharIter = nullptr;
utext_close(&fText);
if (fData != nullptr) {
fData->removeReference();
fData = nullptr;
}
delete fBreakCache;
fBreakCache = nullptr;
delete fDictionaryCache;
fDictionaryCache = nullptr;
delete fLanguageBreakEngines;
fLanguageBreakEngines = nullptr;
delete fUnhandledBreakEngine;
fUnhandledBreakEngine = nullptr;
uprv_free(fLookAheadMatches);
fLookAheadMatches = nullptr;
}
/**
* Assignment operator. Sets this iterator to have the same behavior,
* and iterate over the same text, as the one passed in.
* TODO: needs better handling of memory allocation errors.
*/
RuleBasedBreakIterator&
RuleBasedBreakIterator::operator=(const RuleBasedBreakIterator& that) {
if (this == &that) {
return *this;
}
BreakIterator::operator=(that);
if (fLanguageBreakEngines != NULL) {
delete fLanguageBreakEngines;
fLanguageBreakEngines = NULL; // Just rebuild for now
}
// TODO: clone fLanguageBreakEngines from "that"
UErrorCode status = U_ZERO_ERROR;
utext_clone(&fText, &that.fText, FALSE, TRUE, &status);
if (fCharIter != &fSCharIter) {
delete fCharIter;
}
fCharIter = &fSCharIter;
if (that.fCharIter != NULL && that.fCharIter != &that.fSCharIter) {
// This is a little bit tricky - it will initially appear that
// this->fCharIter is adopted, even if that->fCharIter was
// not adopted. That's ok.
fCharIter = that.fCharIter->clone();
}
fSCharIter = that.fSCharIter;
if (fCharIter == NULL) {
fCharIter = &fSCharIter;
}
if (fData != NULL) {
fData->removeReference();
fData = NULL;
}
if (that.fData != NULL) {
fData = that.fData->addReference();
}
uprv_free(fLookAheadMatches);
fLookAheadMatches = nullptr;
if (fData && fData->fForwardTable->fLookAheadResultsSize > 0) {
fLookAheadMatches = static_cast<int32_t *>(
uprv_malloc(fData->fForwardTable->fLookAheadResultsSize * sizeof(int32_t)));
}
fPosition = that.fPosition;
fRuleStatusIndex = that.fRuleStatusIndex;
fDone = that.fDone;
// TODO: both the dictionary and the main cache need to be copied.
// Current position could be within a dictionary range. Trying to continue
// the iteration without the caches present would go to the rules, with
// the assumption that the current position is on a rule boundary.
fBreakCache->reset(fPosition, fRuleStatusIndex);
fDictionaryCache->reset();
return *this;
}
//-----------------------------------------------------------------------------
//
// init() Shared initialization routine. Used by all the constructors.
// Initializes all fields, leaving the object in a consistent state.
//
//-----------------------------------------------------------------------------
void RuleBasedBreakIterator::init(UErrorCode &status) {
fCharIter = nullptr;
fData = nullptr;
fPosition = 0;
fRuleStatusIndex = 0;
fDone = false;
fDictionaryCharCount = 0;
fLanguageBreakEngines = nullptr;
fUnhandledBreakEngine = nullptr;
fBreakCache = nullptr;
fDictionaryCache = nullptr;
fLookAheadMatches = nullptr;
// Note: IBM xlC is unable to assign or initialize member fText from UTEXT_INITIALIZER.
// fText = UTEXT_INITIALIZER;
static const UText initializedUText = UTEXT_INITIALIZER;
uprv_memcpy(&fText, &initializedUText, sizeof(UText));
if (U_FAILURE(status)) {
return;
}
utext_openUChars(&fText, NULL, 0, &status);
fDictionaryCache = new DictionaryCache(this, status);
fBreakCache = new BreakCache(this, status);
if (U_SUCCESS(status) && (fDictionaryCache == NULL || fBreakCache == NULL)) {
status = U_MEMORY_ALLOCATION_ERROR;
}
#ifdef RBBI_DEBUG
static UBool debugInitDone = FALSE;
if (debugInitDone == FALSE) {
char *debugEnv = getenv("U_RBBIDEBUG");
if (debugEnv && uprv_strstr(debugEnv, "trace")) {
gTrace = TRUE;
}
debugInitDone = TRUE;
}
#endif
}
//-----------------------------------------------------------------------------
//
// clone - Returns a newly-constructed RuleBasedBreakIterator with the same
// behavior, and iterating over the same text, as this one.
// Virtual function: does the right thing with subclasses.
//
//-----------------------------------------------------------------------------
RuleBasedBreakIterator*
RuleBasedBreakIterator::clone() const {
return new RuleBasedBreakIterator(*this);
}
/**
* Equality operator. Returns true if both BreakIterators are of the
* same class, have the same behavior, and iterate over the same text.
*/
bool
RuleBasedBreakIterator::operator==(const BreakIterator& that) const {
if (typeid(*this) != typeid(that)) {
return false;
}
if (this == &that) {
return true;
}
// The base class BreakIterator carries no state that participates in equality,
// and does not implement an equality function that would otherwise be
// checked at this point.
const RuleBasedBreakIterator& that2 = (const RuleBasedBreakIterator&) that;
if (!utext_equals(&fText, &that2.fText)) {
// The two break iterators are operating on different text,
// or have a different iteration position.
// Note that fText's position is always the same as the break iterator's position.
return false;
}
if (!(fPosition == that2.fPosition &&
fRuleStatusIndex == that2.fRuleStatusIndex &&
fDone == that2.fDone)) {
return false;
}
if (that2.fData == fData ||
(fData != NULL && that2.fData != NULL && *that2.fData == *fData)) {
// The two break iterators are using the same rules.
return true;
}
return false;
}
/**
* Compute a hash code for this BreakIterator
* @return A hash code
*/
int32_t
RuleBasedBreakIterator::hashCode(void) const {
int32_t hash = 0;
if (fData != NULL) {
hash = fData->hashCode();
}
return hash;
}
void RuleBasedBreakIterator::setText(UText *ut, UErrorCode &status) {
if (U_FAILURE(status)) {
return;
}
fBreakCache->reset();
fDictionaryCache->reset();
utext_clone(&fText, ut, FALSE, TRUE, &status);
// Set up a dummy CharacterIterator to be returned if anyone
// calls getText(). With input from UText, there is no reasonable
// way to return a characterIterator over the actual input text.
// Return one over an empty string instead - this is the closest
// we can come to signaling a failure.
// (GetText() is obsolete, this failure is sort of OK)
fSCharIter.setText(UnicodeString());
if (fCharIter != &fSCharIter) {
// existing fCharIter was adopted from the outside. Delete it now.
delete fCharIter;
}
fCharIter = &fSCharIter;
this->first();
}
UText *RuleBasedBreakIterator::getUText(UText *fillIn, UErrorCode &status) const {
UText *result = utext_clone(fillIn, &fText, FALSE, TRUE, &status);
return result;
}
//=======================================================================
// BreakIterator overrides
//=======================================================================
/**
* Return a CharacterIterator over the text being analyzed.
*/
CharacterIterator&
RuleBasedBreakIterator::getText() const {
return *fCharIter;
}
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
* @param newText An iterator over the text to analyze.
*/
void
RuleBasedBreakIterator::adoptText(CharacterIterator* newText) {
// If we are holding a CharacterIterator adopted from a
// previous call to this function, delete it now.
if (fCharIter != &fSCharIter) {
delete fCharIter;
}
fCharIter = newText;
UErrorCode status = U_ZERO_ERROR;
fBreakCache->reset();
fDictionaryCache->reset();
if (newText==NULL || newText->startIndex() != 0) {
// startIndex !=0 wants to be an error, but there's no way to report it.
// Make the iterator text be an empty string.
utext_openUChars(&fText, NULL, 0, &status);
} else {
utext_openCharacterIterator(&fText, newText, &status);
}
this->first();
}
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
* @param newText An iterator over the text to analyze.
*/
void
RuleBasedBreakIterator::setText(const UnicodeString& newText) {
UErrorCode status = U_ZERO_ERROR;
fBreakCache->reset();
fDictionaryCache->reset();
utext_openConstUnicodeString(&fText, &newText, &status);
// Set up a character iterator on the string.
// Needed in case someone calls getText().
// Can not, unfortunately, do this lazily on the (probably never)
// call to getText(), because getText is const.
fSCharIter.setText(newText);
if (fCharIter != &fSCharIter) {
// old fCharIter was adopted from the outside. Delete it.
delete fCharIter;
}
fCharIter = &fSCharIter;
this->first();
}
/**
* Provide a new UText for the input text. Must reference text with contents identical
* to the original.
* Intended for use with text data originating in Java (garbage collected) environments
* where the data may be moved in memory at arbitrary times.
*/
RuleBasedBreakIterator &RuleBasedBreakIterator::refreshInputText(UText *input, UErrorCode &status) {
if (U_FAILURE(status)) {
return *this;
}
if (input == NULL) {
status = U_ILLEGAL_ARGUMENT_ERROR;
return *this;
}
int64_t pos = utext_getNativeIndex(&fText);
// Shallow read-only clone of the new UText into the existing input UText
utext_clone(&fText, input, FALSE, TRUE, &status);
if (U_FAILURE(status)) {
return *this;
}
utext_setNativeIndex(&fText, pos);
if (utext_getNativeIndex(&fText) != pos) {
// Sanity check. The new input utext is supposed to have the exact same
// contents as the old. If we can't set to the same position, it doesn't.
// The contents underlying the old utext might be invalid at this point,
// so it's not safe to check directly.
status = U_ILLEGAL_ARGUMENT_ERROR;
}
return *this;
}
/**
* Sets the current iteration position to the beginning of the text, position zero.
* @return The new iterator position, which is zero.
*/
int32_t RuleBasedBreakIterator::first(void) {
UErrorCode status = U_ZERO_ERROR;
if (!fBreakCache->seek(0)) {
fBreakCache->populateNear(0, status);
}
fBreakCache->current();
U_ASSERT(fPosition == 0);
return 0;
}
/**
* Sets the current iteration position to the end of the text.
* @return The text's past-the-end offset.
*/
int32_t RuleBasedBreakIterator::last(void) {
int32_t endPos = (int32_t)utext_nativeLength(&fText);
UBool endShouldBeBoundary = isBoundary(endPos); // Has side effect of setting iterator position.
(void)endShouldBeBoundary;
U_ASSERT(endShouldBeBoundary);
U_ASSERT(fPosition == endPos);
return endPos;
}
/**
* Advances the iterator either forward or backward the specified number of steps.
* Negative values move backward, and positive values move forward. This is
* equivalent to repeatedly calling next() or previous().
* @param n The number of steps to move. The sign indicates the direction
* (negative is backwards, and positive is forwards).
* @return The character offset of the boundary position n boundaries away from
* the current one.
*/
int32_t RuleBasedBreakIterator::next(int32_t n) {
int32_t result = 0;
if (n > 0) {
for (; n > 0 && result != UBRK_DONE; --n) {
result = next();
}
} else if (n < 0) {
for (; n < 0 && result != UBRK_DONE; ++n) {
result = previous();
}
} else {
result = current();
}
return result;
}
/**
* Advances the iterator to the next boundary position.
* @return The position of the first boundary after this one.
*/
int32_t RuleBasedBreakIterator::next(void) {
fBreakCache->next();
return fDone ? UBRK_DONE : fPosition;
}
/**
* Move the iterator backwards, to the boundary preceding the current one.
*
* Starts from the current position within fText.
* Starting position need not be on a boundary.
*
* @return The position of the boundary position immediately preceding the starting position.
*/
int32_t RuleBasedBreakIterator::previous(void) {
UErrorCode status = U_ZERO_ERROR;
fBreakCache->previous(status);
return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
* @param startPos The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
*/
int32_t RuleBasedBreakIterator::following(int32_t startPos) {
// if the supplied position is before the beginning, return the
// text's starting offset
if (startPos < 0) {
return first();
}
// Move requested offset to a code point start. It might be on a trail surrogate,
// or on a trail byte if the input is UTF-8. Or it may be beyond the end of the text.
utext_setNativeIndex(&fText, startPos);
startPos = (int32_t)utext_getNativeIndex(&fText);
UErrorCode status = U_ZERO_ERROR;
fBreakCache->following(startPos, status);
return fDone ? UBRK_DONE : fPosition;
}
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
* @param offset The position to begin searching for a break from.
* @return The position of the last boundary before the starting position.
*/
int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
if (offset > utext_nativeLength(&fText)) {
return last();
}
// Move requested offset to a code point start. It might be on a trail surrogate,
// or on a trail byte if the input is UTF-8.
utext_setNativeIndex(&fText, offset);
int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText));
UErrorCode status = U_ZERO_ERROR;
fBreakCache->preceding(adjustedOffset, status);
return fDone ? UBRK_DONE : fPosition;
}
/**
* Returns true if the specified position is a boundary position. As a side
* effect, leaves the iterator pointing to the first boundary position at
* or after "offset".
*
* @param offset the offset to check.
* @return True if "offset" is a boundary position.
*/
UBool RuleBasedBreakIterator::isBoundary(int32_t offset) {
// out-of-range indexes are never boundary positions
if (offset < 0) {
first(); // For side effects on current position, tag values.
return FALSE;
}
// Adjust offset to be on a code point boundary and not beyond the end of the text.
// Note that isBoundary() is always false for offsets that are not on code point boundaries.
// But we still need the side effect of leaving iteration at the following boundary.
utext_setNativeIndex(&fText, offset);
int32_t adjustedOffset = static_cast<int32_t>(utext_getNativeIndex(&fText));
bool result = false;
UErrorCode status = U_ZERO_ERROR;
if (fBreakCache->seek(adjustedOffset) || fBreakCache->populateNear(adjustedOffset, status)) {
result = (fBreakCache->current() == offset);
}
if (result && adjustedOffset < offset && utext_char32At(&fText, offset) == U_SENTINEL) {
// Original offset is beyond the end of the text. Return FALSE, it's not a boundary,
// but the iteration position remains set to the end of the text, which is a boundary.
return FALSE;
}
if (!result) {
// Not on a boundary. isBoundary() must leave iterator on the following boundary.
// Cache->seek(), above, left us on the preceding boundary, so advance one.
next();
}
return result;
}
/**
* Returns the current iteration position.
* @return The current iteration position.
*/
int32_t RuleBasedBreakIterator::current(void) const {
return fPosition;
}
//=======================================================================
// implementation
//=======================================================================
//
// RBBIRunMode - the state machine runs an extra iteration at the beginning and end
// of user text. A variable with this enum type keeps track of where we
// are. The state machine only fetches user input while in the RUN mode.
//
enum RBBIRunMode {
RBBI_START, // state machine processing is before first char of input
RBBI_RUN, // state machine processing is in the user text
RBBI_END // state machine processing is after end of user text.
};
// Wrapper functions to select the appropriate handleNext() or handleSafePrevious()
// instantiation, based on whether an 8 or 16 bit table is required.
//
// These Trie access functions will be inlined within the handleNext()/Previous() instantions.
static inline uint16_t TrieFunc8(const UCPTrie *trie, UChar32 c) {
return UCPTRIE_FAST_GET(trie, UCPTRIE_8, c);
}
static inline uint16_t TrieFunc16(const UCPTrie *trie, UChar32 c) {
return UCPTRIE_FAST_GET(trie, UCPTRIE_16, c);
}
int32_t RuleBasedBreakIterator::handleNext() {
const RBBIStateTable *statetable = fData->fForwardTable;
bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8;
if (statetable->fFlags & RBBI_8BITS_ROWS) {
if (use8BitsTrie) {
return handleNext<RBBIStateTableRow8, TrieFunc8>();
} else {
return handleNext<RBBIStateTableRow8, TrieFunc16>();
}
} else {
if (use8BitsTrie) {
return handleNext<RBBIStateTableRow16, TrieFunc8>();
} else {
return handleNext<RBBIStateTableRow16, TrieFunc16>();
}
}
}
int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
const RBBIStateTable *statetable = fData->fReverseTable;
bool use8BitsTrie = ucptrie_getValueWidth(fData->fTrie) == UCPTRIE_VALUE_BITS_8;
if (statetable->fFlags & RBBI_8BITS_ROWS) {
if (use8BitsTrie) {
return handleSafePrevious<RBBIStateTableRow8, TrieFunc8>(fromPosition);
} else {
return handleSafePrevious<RBBIStateTableRow8, TrieFunc16>(fromPosition);
}
} else {
if (use8BitsTrie) {
return handleSafePrevious<RBBIStateTableRow16, TrieFunc8>(fromPosition);
} else {
return handleSafePrevious<RBBIStateTableRow16, TrieFunc16>(fromPosition);
}
}
}
//-----------------------------------------------------------------------------------
//
// handleNext()
// Run the state machine to find a boundary
//
//-----------------------------------------------------------------------------------
template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc>
int32_t RuleBasedBreakIterator::handleNext() {
int32_t state;
uint16_t category = 0;
RBBIRunMode mode;
RowType *row;
UChar32 c;
int32_t result = 0;
int32_t initialPosition = 0;
const RBBIStateTable *statetable = fData->fForwardTable;
const char *tableData = statetable->fTableData;
uint32_t tableRowLen = statetable->fRowLen;
uint32_t dictStart = statetable->fDictCategoriesStart;
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPuts("Handle Next pos char state category");
}
#endif
// handleNext always sets the break tag value.
// Set the default for it.
fRuleStatusIndex = 0;
fDictionaryCharCount = 0;
// if we're already at the end of the text, return DONE.
initialPosition = fPosition;
UTEXT_SETNATIVEINDEX(&fText, initialPosition);
result = initialPosition;
c = UTEXT_NEXT32(&fText);
if (c==U_SENTINEL) {
fDone = TRUE;
return UBRK_DONE;
}
// Set the initial state for the state machine
state = START_STATE;
row = (RowType *)
//(statetable->fTableData + (statetable->fRowLen * state));
(tableData + tableRowLen * state);
mode = RBBI_RUN;
if (statetable->fFlags & RBBI_BOF_REQUIRED) {
category = 2;
mode = RBBI_START;
}
// loop until we reach the end of the text or transition to state 0
//
for (;;) {
if (c == U_SENTINEL) {
// Reached end of input string.
if (mode == RBBI_END) {
// We have already run the loop one last time with the
// character set to the psueudo {eof} value. Now it is time
// to unconditionally bail out.
break;
}
// Run the loop one last time with the fake end-of-input character category.
mode = RBBI_END;
category = 1;
}
//
// Get the char category. An incoming category of 1 or 2 means that
// we are preset for doing the beginning or end of input, and
// that we shouldn't get a category from an actual text input character.
//
if (mode == RBBI_RUN) {
// look up the current character's character category, which tells us
// which column in the state table to look at.
category = trieFunc(fData->fTrie, c);
fDictionaryCharCount += (category >= dictStart);
}
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPrintf(" %4" PRId64 " ", utext_getNativeIndex(&fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
} else {
RBBIDebugPrintf("%5x ", c);
}
RBBIDebugPrintf("%3d %3d\n", state, category);
}
#endif
// State Transition - move machine to its next state
//
// fNextState is a variable-length array.
U_ASSERT(category<fData->fHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RowType *)
// (statetable->fTableData + (statetable->fRowLen * state));
(tableData + tableRowLen * state);
uint16_t accepting = row->fAccepting;
if (accepting == ACCEPTING_UNCONDITIONAL) {
// Match found, common case.
if (mode != RBBI_START) {
result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
}
fRuleStatusIndex = row->fTagsIdx; // Remember the break status (tag) values.
} else if (accepting > ACCEPTING_UNCONDITIONAL) {
// Lookahead match is completed.
U_ASSERT(accepting < fData->fForwardTable->fLookAheadResultsSize);
int32_t lookaheadResult = fLookAheadMatches[accepting];
if (lookaheadResult >= 0) {
fRuleStatusIndex = row->fTagsIdx;
fPosition = lookaheadResult;
return lookaheadResult;
}
}
// If we are at the position of the '/' in a look-ahead (hard break) rule;
// record the current position, to be returned later, if the full rule matches.
// TODO: Move this check before the previous check of fAccepting.
// This would enable hard-break rules with no following context.
// But there are line break test failures when trying this. Investigate.
// Issue ICU-20837
uint16_t rule = row->fLookAhead;
U_ASSERT(rule == 0 || rule > ACCEPTING_UNCONDITIONAL);
U_ASSERT(rule == 0 || rule < fData->fForwardTable->fLookAheadResultsSize);
if (rule > ACCEPTING_UNCONDITIONAL) {
int32_t pos = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
fLookAheadMatches[rule] = pos;
}
if (state == STOP_STATE) {
// This is the normal exit from the lookup state machine.
// We have advanced through the string until it is certain that no
// longer match is possible, no matter what characters follow.
break;
}
// Advance to the next character.
// If this is a beginning-of-input loop iteration, don't advance
// the input position. The next iteration will be processing the
// first real input character.
if (mode == RBBI_RUN) {
c = UTEXT_NEXT32(&fText);
} else {
if (mode == RBBI_START) {
mode = RBBI_RUN;
}
}
}
// The state machine is done. Check whether it found a match...
// If the iterator failed to advance in the match engine, force it ahead by one.
// (This really indicates a defect in the break rules. They should always match
// at least one character.)
if (result == initialPosition) {
utext_setNativeIndex(&fText, initialPosition);
utext_next32(&fText);
result = (int32_t)utext_getNativeIndex(&fText);
fRuleStatusIndex = 0;
}
// Leave the iterator at our result position.
fPosition = result;
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
return result;
}
//-----------------------------------------------------------------------------------
//
// handleSafePrevious()
//
// Iterate backwards using the safe reverse rules.
// The logic of this function is similar to handleNext(), but simpler
// because the safe table does not require as many options.
//
//-----------------------------------------------------------------------------------
template <typename RowType, RuleBasedBreakIterator::PTrieFunc trieFunc>
int32_t RuleBasedBreakIterator::handleSafePrevious(int32_t fromPosition) {
int32_t state;
uint16_t category = 0;
RowType *row;
UChar32 c;
int32_t result = 0;
const RBBIStateTable *stateTable = fData->fReverseTable;
UTEXT_SETNATIVEINDEX(&fText, fromPosition);
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPuts("Handle Previous pos char state category");
}
#endif
// if we're already at the start of the text, return DONE.
if (fData == NULL || UTEXT_GETNATIVEINDEX(&fText)==0) {
return BreakIterator::DONE;
}
// Set the initial state for the state machine
c = UTEXT_PREVIOUS32(&fText);
state = START_STATE;
row = (RowType *)
(stateTable->fTableData + (stateTable->fRowLen * state));
// loop until we reach the start of the text or transition to state 0
//
for (; c != U_SENTINEL; c = UTEXT_PREVIOUS32(&fText)) {
// look up the current character's character category, which tells us
// which column in the state table to look at.
//
// Off the dictionary flag bit. For reverse iteration it is not used.
category = trieFunc(fData->fTrie, c);
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPrintf(" %4d ", (int32_t)utext_getNativeIndex(&fText));
if (0x20<=c && c<0x7f) {
RBBIDebugPrintf("\"%c\" ", c);
} else {
RBBIDebugPrintf("%5x ", c);
}
RBBIDebugPrintf("%3d %3d\n", state, category);
}
#endif
// State Transition - move machine to its next state
//
// fNextState is a variable-length array.
U_ASSERT(category<fData->fHeader->fCatCount);
state = row->fNextState[category]; /*Not accessing beyond memory*/
row = (RowType *)
(stateTable->fTableData + (stateTable->fRowLen * state));
if (state == STOP_STATE) {
// This is the normal exit from the lookup state machine.
// Transition to state zero means we have found a safe point.
break;
}
}
// The state machine is done. Check whether it found a match...
result = (int32_t)UTEXT_GETNATIVEINDEX(&fText);
#ifdef RBBI_DEBUG
if (gTrace) {
RBBIDebugPrintf("result = %d\n\n", result);
}
#endif
return result;
}
//-------------------------------------------------------------------------------
//
// getRuleStatus() Return the break rule tag associated with the current
// iterator position. If the iterator arrived at its current
// position by iterating forwards, the value will have been
// cached by the handleNext() function.
//
//-------------------------------------------------------------------------------
int32_t RuleBasedBreakIterator::getRuleStatus() const {
// fLastRuleStatusIndex indexes to the start of the appropriate status record
// (the number of status values.)
// This function returns the last (largest) of the array of status values.
int32_t idx = fRuleStatusIndex + fData->fRuleStatusTable[fRuleStatusIndex];
int32_t tagVal = fData->fRuleStatusTable[idx];
return tagVal;
}
int32_t RuleBasedBreakIterator::getRuleStatusVec(
int32_t *fillInVec, int32_t capacity, UErrorCode &status) {
if (U_FAILURE(status)) {
return 0;
}
int32_t numVals = fData->fRuleStatusTable[fRuleStatusIndex];
int32_t numValsToCopy = numVals;
if (numVals > capacity) {
status = U_BUFFER_OVERFLOW_ERROR;
numValsToCopy = capacity;
}
int i;
for (i=0; i<numValsToCopy; i++) {
fillInVec[i] = fData->fRuleStatusTable[fRuleStatusIndex + i + 1];
}
return numVals;
}
//-------------------------------------------------------------------------------
//
// getBinaryRules Access to the compiled form of the rules,
// for use by build system tools that save the data
// for standard iterator types.
//
//-------------------------------------------------------------------------------
const uint8_t *RuleBasedBreakIterator::getBinaryRules(uint32_t &length) {
const uint8_t *retPtr = NULL;
length = 0;
if (fData != NULL) {
retPtr = (const uint8_t *)fData->fHeader;
length = fData->fHeader->fLength;
}
return retPtr;
}
RuleBasedBreakIterator *RuleBasedBreakIterator::createBufferClone(
void * /*stackBuffer*/, int32_t &bufferSize, UErrorCode &status) {
if (U_FAILURE(status)){
return NULL;
}
if (bufferSize == 0) {
bufferSize = 1; // preflighting for deprecated functionality
return NULL;
}
BreakIterator *clonedBI = clone();
if (clonedBI == NULL) {
status = U_MEMORY_ALLOCATION_ERROR;
} else {
status = U_SAFECLONE_ALLOCATED_WARNING;
}
return (RuleBasedBreakIterator *)clonedBI;
}
U_NAMESPACE_END
static icu::UStack *gLanguageBreakFactories = nullptr;
static const icu::UnicodeString *gEmptyString = nullptr;
static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER;
static icu::UInitOnce gRBBIInitOnce = U_INITONCE_INITIALIZER;
/**
* Release all static memory held by breakiterator.
*/
U_CDECL_BEGIN
UBool U_CALLCONV rbbi_cleanup(void) {
delete gLanguageBreakFactories;
gLanguageBreakFactories = nullptr;
delete gEmptyString;
gEmptyString = nullptr;
gLanguageBreakFactoriesInitOnce.reset();
gRBBIInitOnce.reset();
return TRUE;
}
U_CDECL_END
U_CDECL_BEGIN
static void U_CALLCONV _deleteFactory(void *obj) {
delete (icu::LanguageBreakFactory *) obj;
}
U_CDECL_END
U_NAMESPACE_BEGIN
static void U_CALLCONV rbbiInit() {
gEmptyString = new UnicodeString();
ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
}
static void U_CALLCONV initLanguageFactories() {
UErrorCode status = U_ZERO_ERROR;
U_ASSERT(gLanguageBreakFactories == NULL);
gLanguageBreakFactories = new UStack(_deleteFactory, NULL, status);
if (gLanguageBreakFactories != NULL && U_SUCCESS(status)) {
ICULanguageBreakFactory *builtIn = new ICULanguageBreakFactory(status);
gLanguageBreakFactories->push(builtIn, status);
#ifdef U_LOCAL_SERVICE_HOOK
LanguageBreakFactory *extra = (LanguageBreakFactory *)uprv_svc_hook("languageBreakFactory", &status);
if (extra != NULL) {
gLanguageBreakFactories->push(extra, status);
}
#endif
}
ucln_common_registerCleanup(UCLN_COMMON_RBBI, rbbi_cleanup);
}
static const LanguageBreakEngine*
getLanguageBreakEngineFromFactory(UChar32 c)
{
umtx_initOnce(gLanguageBreakFactoriesInitOnce, &initLanguageFactories);
if (gLanguageBreakFactories == NULL) {
return NULL;
}
int32_t i = gLanguageBreakFactories->size();
const LanguageBreakEngine *lbe = NULL;
while (--i >= 0) {
LanguageBreakFactory *factory = (LanguageBreakFactory *)(gLanguageBreakFactories->elementAt(i));
lbe = factory->getEngineFor(c);
if (lbe != NULL) {
break;
}
}
return lbe;
}
//-------------------------------------------------------------------------------
//
// getLanguageBreakEngine Find an appropriate LanguageBreakEngine for the
// the character c.
//
//-------------------------------------------------------------------------------
const LanguageBreakEngine *
RuleBasedBreakIterator::getLanguageBreakEngine(UChar32 c) {
const LanguageBreakEngine *lbe = NULL;
UErrorCode status = U_ZERO_ERROR;
if (fLanguageBreakEngines == NULL) {
fLanguageBreakEngines = new UStack(status);
if (fLanguageBreakEngines == NULL || U_FAILURE(status)) {
delete fLanguageBreakEngines;
fLanguageBreakEngines = 0;
return NULL;
}
}
int32_t i = fLanguageBreakEngines->size();
while (--i >= 0) {
lbe = (const LanguageBreakEngine *)(fLanguageBreakEngines->elementAt(i));
if (lbe->handles(c)) {
return lbe;
}
}
// No existing dictionary took the character. See if a factory wants to
// give us a new LanguageBreakEngine for this character.
lbe = getLanguageBreakEngineFromFactory(c);
// If we got one, use it and push it on our stack.
if (lbe != NULL) {
fLanguageBreakEngines->push((void *)lbe, status);
// Even if we can't remember it, we can keep looking it up, so
// return it even if the push fails.
return lbe;
}
// No engine is forthcoming for this character. Add it to the
// reject set. Create the reject break engine if needed.
if (fUnhandledBreakEngine == NULL) {
fUnhandledBreakEngine = new UnhandledEngine(status);
if (U_SUCCESS(status) && fUnhandledBreakEngine == NULL) {
status = U_MEMORY_ALLOCATION_ERROR;
return nullptr;
}
// Put it last so that scripts for which we have an engine get tried
// first.
fLanguageBreakEngines->insertElementAt(fUnhandledBreakEngine, 0, status);
// If we can't insert it, or creation failed, get rid of it
U_ASSERT(!fLanguageBreakEngines->hasDeleter());
if (U_FAILURE(status)) {
delete fUnhandledBreakEngine;
fUnhandledBreakEngine = 0;
return NULL;
}
}
// Tell the reject engine about the character; at its discretion, it may
// add more than just the one character.
fUnhandledBreakEngine->handleCharacter(c);
return fUnhandledBreakEngine;
}
void RuleBasedBreakIterator::dumpCache() {
fBreakCache->dumpCache();
}
void RuleBasedBreakIterator::dumpTables() {
fData->printData();
}
/**
* Returns the description used to create this iterator
*/
const UnicodeString&
RuleBasedBreakIterator::getRules() const {
if (fData != NULL) {
return fData->getRuleSourceString();
} else {
umtx_initOnce(gRBBIInitOnce, &rbbiInit);
return *gEmptyString;
}
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
Reference in a new issue View git blame Copy permalink