organicmaps/utfcpp
13
Fork 0
mirror of https://github.com/nemtrif/utfcpp.git synced 2025-04-04 21:15:08 +00:00
Code Issues 1 Projects Releases Wiki Activity Actions

Redefined and renamed types for code units.

Browse source
This commit is contained in:
nemtrif 2022-12-27 13:42:55 -05:00
parent e3e57c811a
commit ab72c87365
7 changed files with 133 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
README.md
View file

@ -199,7 +199,7 @@ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequen
```cpp
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result);
octet_iterator append(utfchar32_t cp, octet_iterator result);
```
`octet_iterator`: an output iterator.
@ -227,7 +227,7 @@ Given the iterator to the beginning of the UTF-8 sequence, it returns the code p
```cpp
template <typename octet_iterator>
uint32_t next(octet_iterator& it, octet_iterator end);
utfchar32_t next(octet_iterator& it, octet_iterator end);
```
`octet_iterator`: an input iterator.
@ -257,7 +257,7 @@ Given the iterator to the beginning of the UTF-8 sequence, it returns the code p
```cpp
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it, octet_iterator end);
utfchar32_t peek_next(octet_iterator it, octet_iterator end);
```
@ -286,7 +286,7 @@ Given a reference to an iterator pointing to an octet in a UTF-8 sequence, it de
```cpp
template <typename octet_iterator>
uint32_t prior(octet_iterator& it, octet_iterator start);
utfchar32_t prior(octet_iterator& it, octet_iterator start);
```
`octet_iterator`: a bidirectional iterator.
@ -887,7 +887,7 @@ Replaces all invalid UTF-8 sequences within a string with a replacement marker.
```cpp
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement);
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement);
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out);
```
@ -1027,7 +1027,7 @@ Thrown by UTF8 CPP functions such as `advance` and `next` if an UTF-8 sequence r
```cpp
class invalid_code_point : public exception {
public:
uint32_t code_point() const;
utfchar32_t code_point() const;
};
```
@ -1042,7 +1042,7 @@ Thrown by UTF8 CPP functions such as `next` and `prior` if an invalid UTF-8 sequ
```cpp
class invalid_utf8 : public exception {
public:
uint8_t utf8_octet() const;
utfchar8_t utf8_octet() const;
};
```
@ -1057,7 +1057,7 @@ Thrown by UTF8 CPP function `utf16to8` if an invalid UTF-16 sequence is detected
```cpp
class invalid_utf16 : public exception {
public:
uint16_t utf16_word() const;
utfchar16_t utf16_word() const;
};
```
@ -1092,7 +1092,7 @@ class iterator;
`octet_iterator base () const;` returns the underlying octet_iterator.
`uint32_t operator * () const;` decodes the utf-8 sequence the underlying octet_iterator is pointing to and returns the code point.
`utfchar32_t operator * () const;` decodes the utf-8 sequence the underlying octet_iterator is pointing to and returns the code point.
`bool operator == (const iterator& rhs) const;` returns `true` if the two underlaying iterators are equal.
@ -1146,7 +1146,7 @@ Encodes a 32 bit code point as a UTF-8 sequence of octets and appends the sequen
```cpp
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result);
octet_iterator append(utfchar32_t cp, octet_iterator result);
```
`cp`: A 32 bit integer representing a code point to append to the sequence.
@ -1171,7 +1171,7 @@ Given the iterator to the beginning of a UTF-8 sequence, it returns the code poi
```cpp
template <typename octet_iterator>
uint32_t next(octet_iterator& it);
utfchar32_t next(octet_iterator& it);
```
`it`: a reference to an iterator pointing to the beginning of an UTF-8 encoded code point. After the function returns, it is incremented to point to the beginning of the next code point.
@ -1197,7 +1197,7 @@ Given the iterator to the beginning of a UTF-8 sequence, it returns the code poi
```cpp
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it);
utfchar32_t peek_next(octet_iterator it);
```
`it`: an iterator pointing to the beginning of an UTF-8 encoded code point.
@ -1223,7 +1223,7 @@ Given a reference to an iterator pointing to an octet in a UTF-8 seqence, it dec
```cpp
template <typename octet_iterator>
uint32_t prior(octet_iterator& it);
utfchar32_t prior(octet_iterator& it);
```
`it`: a reference pointing to an octet within a UTF-8 encoded string. After the function returns, it is decremented to point to the beginning of the previous code point.
@ -1408,7 +1408,7 @@ Replaces all invalid UTF-8 sequences within a string with a replacement marker.
```cpp
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement);
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement);
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out);
```
@ -1458,7 +1458,7 @@ class iterator;
`octet_iterator base () const;` returns the underlying octet_iterator.
`uint32_t operator * () const;` decodes the utf-8 sequence the underlying octet_iterator is pointing to and returns the code point.
`utfchar32_t operator * () const;` decodes the utf-8 sequence the underlying octet_iterator is pointing to and returns the code point.
`bool operator == (const iterator& rhs) const;` returns `true` if the two underlaying iterators are equal.

12
source/utf8.h
View file

@ -28,6 +28,18 @@ DEALINGS IN THE SOFTWARE.
#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
/*
To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro
and set it to one of the values used by the __cplusplus predefined macro.
For instance,
#define UTF_CPP_CPLUSPLUS 199711L
will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard.
Some library features will be disabled.
If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus.
*/
#include "utf8/checked.h"
#include "utf8/unchecked.h"

62
source/utf8/checked.h
View file

@ -39,28 +39,28 @@ namespace utf8
// Exceptions that may be thrown from the library functions.
class invalid_code_point : public exception {
uint32_t cp;
utfchar32_t cp;
public:
invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
invalid_code_point(utfchar32_t codepoint) : cp(codepoint) {}
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; }
uint32_t code_point() const {return cp;}
utfchar32_t code_point() const {return cp;}
};
class invalid_utf8 : public exception {
uint8_t u8;
utfchar8_t u8;
public:
invalid_utf8 (uint8_t u) : u8(u) {}
invalid_utf8 (char c) : u8(static_cast<uint8_t>(c)) {}
invalid_utf8 (utfchar8_t u) : u8(u) {}
invalid_utf8 (char c) : u8(static_cast<utfchar8_t>(c)) {}
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; }
uint8_t utf8_octet() const {return u8;}
utfchar8_t utf8_octet() const {return u8;}
};
class invalid_utf16 : public exception {
uint16_t u16;
utfchar16_t u16;
public:
invalid_utf16 (uint16_t u) : u16(u) {}
invalid_utf16 (utfchar16_t u) : u16(u) {}
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; }
uint16_t utf16_word() const {return u16;}
utfchar16_t utf16_word() const {return u16;}
};
class not_enough_room : public exception {
@ -71,7 +71,7 @@ namespace utf8
/// The library API - functions intended to be called by the users
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
octet_iterator append(utfchar32_t cp, octet_iterator result)
{
if (!utf8::internal::is_code_point_valid(cp))
throw invalid_code_point(cp);
@ -80,7 +80,7 @@ namespace utf8
}
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement)
{
while (start != end) {
octet_iterator sequence_start = start;
@ -115,14 +115,14 @@ namespace utf8
template <typename octet_iterator, typename output_iterator>
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
{
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd);
return utf8::replace_invalid(start, end, out, replacement_marker);
}
template <typename octet_iterator>
uint32_t next(octet_iterator& it, octet_iterator end)
utfchar32_t next(octet_iterator& it, octet_iterator end)
{
uint32_t cp = 0;
utfchar32_t cp = 0;
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
switch (err_code) {
case internal::UTF8_OK :
@ -132,7 +132,7 @@ namespace utf8
case internal::INVALID_LEAD :
case internal::INCOMPLETE_SEQUENCE :
case internal::OVERLONG_SEQUENCE :
throw invalid_utf8(static_cast<uint8_t>(*it));
throw invalid_utf8(static_cast<utfchar8_t>(*it));
case internal::INVALID_CODE_POINT :
throw invalid_code_point(cp);
}
@ -140,13 +140,13 @@ namespace utf8
}
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it, octet_iterator end)
utfchar32_t peek_next(octet_iterator it, octet_iterator end)
{
return utf8::next(it, end);
}
template <typename octet_iterator>
uint32_t prior(octet_iterator& it, octet_iterator start)
utfchar32_t prior(octet_iterator& it, octet_iterator start)
{
// can't do much if it == start
if (it == start)
@ -189,23 +189,23 @@ namespace utf8
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
{
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
utfchar32_t cp = utf8::internal::mask16(*start++);
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp)) {
if (start != end) {
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
const utfchar32_t trail_surrogate = utf8::internal::mask16(*start++);
if (utf8::internal::is_trail_surrogate(trail_surrogate))
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
else
throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
throw invalid_utf16(static_cast<utfchar16_t>(trail_surrogate));
}
else
throw invalid_utf16(static_cast<uint16_t>(cp));
throw invalid_utf16(static_cast<utfchar16_t>(cp));
}
// Lone trail surrogate
else if (utf8::internal::is_trail_surrogate(cp))
throw invalid_utf16(static_cast<uint16_t>(cp));
throw invalid_utf16(static_cast<utfchar16_t>(cp));
result = utf8::append(cp, result);
}
@ -216,13 +216,13 @@ namespace utf8
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
{
while (start < end) {
uint32_t cp = utf8::next(start, end);
const utfchar32_t cp = utf8::next(start, end);
if (cp > 0xffff) { //make a surrogate pair
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
*result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
}
else
*result++ = static_cast<uint16_t>(cp);
*result++ = static_cast<utfchar16_t>(cp);
}
return result;
}
@ -252,9 +252,9 @@ namespace utf8
octet_iterator range_start;
octet_iterator range_end;
public:
typedef uint32_t value_type;
typedef uint32_t* pointer;
typedef uint32_t& reference;
typedef utfchar32_t value_type;
typedef utfchar32_t* pointer;
typedef utfchar32_t& reference;
typedef std::ptrdiff_t difference_type;
typedef std::bidirectional_iterator_tag iterator_category;
iterator () {}
@ -268,7 +268,7 @@ namespace utf8
}
// the default "big three" are OK
octet_iterator base () const { return it; }
uint32_t operator * () const
utfchar32_t operator * () const
{
octet_iterator temp = it;
return utf8::next(temp, range_end);

87
source/utf8/core.h
View file

@ -49,12 +49,20 @@ DEALINGS IN THE SOFTWARE.
namespace utf8
{
// The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
// You may need to change them to match your system.
// These typedefs have the same names as ones from cstdint, or boost/cstdint
typedef unsigned char uint8_t;
typedef unsigned short uint16_t;
typedef unsigned int uint32_t;
// The typedefs for 8-bit, 16-bit and 32-bit code units
#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later
typedef char8_t utfchar8_t;
#else // C++ 11/14/17
typedef unsigned char utfchar8_t;
#endif
typedef char16_t utfchar16_t;
typedef char32_t utfchar32_t;
#else // C++ 98/03
typedef unsigned char utfchar8_t;
typedef unsigned short utfchar16_t;
typedef unsigned int utfchar32_t;
#endif // C++ 11 or later
// Helper code - not intended to be directly called by the library users. May be changed at any time
namespace internal
@ -62,26 +70,27 @@ namespace internal
// Unicode constants
// Leading (high) surrogates: 0xd800 - 0xdbff
// Trailing (low) surrogates: 0xdc00 - 0xdfff
const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
const uint16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10)
const uint32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
const utfchar16_t LEAD_SURROGATE_MIN = 0xd800u;
const utfchar16_t LEAD_SURROGATE_MAX = 0xdbffu;
const utfchar16_t TRAIL_SURROGATE_MIN = 0xdc00u;
const utfchar16_t TRAIL_SURROGATE_MAX = 0xdfffu;
const utfchar16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10)
const utfchar32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
// Maximum valid value for a Unicode code point
const uint32_t CODE_POINT_MAX = 0x0010ffffu;
const utfchar32_t CODE_POINT_MAX = 0x0010ffffu;
template<typename octet_type>
inline uint8_t mask8(octet_type oc)
inline utfchar8_t mask8(octet_type oc)
{
return static_cast<uint8_t>(0xff & oc);
return static_cast<utfchar8_t>(0xff & oc);
}
template<typename u16_type>
inline uint16_t mask16(u16_type oc)
inline utfchar16_t mask16(u16_type oc)
{
return static_cast<uint16_t>(0xffff & oc);
return static_cast<utfchar16_t>(0xffff & oc);
}
template<typename octet_type>
inline bool is_trail(octet_type oc)
{
@ -113,10 +122,9 @@ namespace internal
}
template <typename octet_iterator>
inline typename std::iterator_traits<octet_iterator>::difference_type
sequence_length(octet_iterator lead_it)
int sequence_length(octet_iterator lead_it)
{
uint8_t lead = utf8::internal::mask8(*lead_it);
const utfchar8_t lead = utf8::internal::mask8(*lead_it);
if (lead < 0x80)
return 1;
else if ((lead >> 5) == 0x6)
@ -129,8 +137,7 @@ namespace internal
return 0;
}
template <typename octet_difference_type>
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
inline bool is_overlong_sequence(utfchar32_t cp, int length)
{
if (cp < 0x80) {
if (length != 1)
@ -144,7 +151,6 @@ namespace internal
if (length != 3)
return true;
}
return false;
}
@ -152,7 +158,7 @@ namespace internal
/// Helper for get_sequence_x
template <typename octet_iterator>
utf_error increase_safely(octet_iterator& it, octet_iterator end)
utf_error increase_safely(octet_iterator& it, const octet_iterator end)
{
if (++it == end)
return NOT_ENOUGH_ROOM;
@ -163,11 +169,11 @@ namespace internal
return UTF8_OK;
}
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
/// get_sequence_x functions decode utf-8 sequences of the length x
template <typename octet_iterator>
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@ -178,7 +184,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@ -193,7 +199,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@ -212,7 +218,7 @@ namespace internal
}
template <typename octet_iterator>
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@ -237,7 +243,7 @@ namespace internal
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
template <typename octet_iterator>
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
utf_error validate_next(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
{
if (it == end)
return NOT_ENOUGH_ROOM;
@ -246,10 +252,9 @@ namespace internal
// Of course, it does not make much sense with i.e. stream iterators
octet_iterator original_it = it;
uint32_t cp = 0;
utfchar32_t cp = 0;
// Determine the sequence length based on the lead octet
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
const octet_difference_type length = utf8::internal::sequence_length(it);
const int length = utf8::internal::sequence_length(it);
// Get trail octets and calculate the code point
utf_error err = UTF8_OK;
@ -293,7 +298,7 @@ namespace internal
template <typename octet_iterator>
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
uint32_t ignored;
utfchar32_t ignored;
return utf8::internal::validate_next(it, end, ignored);
}
@ -301,7 +306,7 @@ namespace internal
// This function will be invoked by the overloads below, as they will know
// the octet_type.
template <typename octet_iterator, typename octet_type>
octet_iterator append(uint32_t cp, octet_iterator result) {
octet_iterator append(utfchar32_t cp, octet_iterator result) {
if (cp < 0x80) // one octet
*(result++) = static_cast<octet_type>(cp);
else if (cp < 0x800) { // two octets
@ -325,7 +330,7 @@ namespace internal
// One of the following overloads will be invoked from the API calls
// A simple (but dangerous) case: the caller appends byte(s) to a char array
inline char* append(uint32_t cp, char* result) {
inline char* append(utfchar32_t cp, char* result) {
return append<char*, char>(cp, result);
}
@ -333,17 +338,17 @@ namespace internal
// i.e. append(cp, std::back_inserter(str));
template<typename container_type>
std::back_insert_iterator<container_type> append
(uint32_t cp, std::back_insert_iterator<container_type> result) {
(utfchar32_t cp, std::back_insert_iterator<container_type> result) {
return append<std::back_insert_iterator<container_type>,
typename container_type::value_type>(cp, result);
}
// The caller uses some other kind of output operator - not covered above
// Note that in this case we are not able to determine octet_type
// so we assume it's uint_8; that can cause a conversion warning if we are wrong.
// so we assume it's utfchar_8; that can cause a conversion warning if we are wrong.
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result) {
return append<octet_iterator, uint8_t>(cp, result);
octet_iterator append(utfchar32_t cp, octet_iterator result) {
return append<octet_iterator, utfchar8_t>(cp, result);
}
} // namespace internal
@ -351,7 +356,7 @@ namespace internal
/// The library API - functions intended to be called by the users
// Byte order mark
const uint8_t bom[] = {0xef, 0xbb, 0xbf};
const utfchar8_t bom[] = {0xef, 0xbb, 0xbf};
template <typename octet_iterator>
octet_iterator find_invalid(octet_iterator start, octet_iterator end)

2
source/utf8/cpp11.h
View file

@ -36,7 +36,7 @@ namespace utf8
inline void append(char32_t cp, std::string& s)
{
append(uint32_t(cp), std::back_inserter(s));
append(cp, std::back_inserter(s));
}
inline std::string utf16to8(const std::u16string& s)

2
source/utf8/cpp17.h
View file

@ -36,7 +36,7 @@ namespace utf8
inline void append(char32_t cp, std::string& s)
{
append(uint32_t(cp), std::back_inserter(s));
append(cp, std::back_inserter(s));
}
inline std::string utf16to8(std::u16string_view s)

55
source/utf8/unchecked.h
View file

@ -35,13 +35,13 @@ namespace utf8
namespace unchecked
{
template <typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
octet_iterator append(utfchar32_t cp, octet_iterator result)
{
return internal::append(cp, result);
}
template <typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement)
{
while (start != end) {
octet_iterator sequence_start = start;
@ -52,17 +52,17 @@ namespace utf8
*out++ = *it;
break;
case internal::NOT_ENOUGH_ROOM:
out = utf8::unchecked::append (replacement, out);
out = utf8::unchecked::append(replacement, out);
start = end;
break;
case internal::INVALID_LEAD:
out = utf8::unchecked::append (replacement, out);
out = utf8::unchecked::append(replacement, out);
++start;
break;
case internal::INCOMPLETE_SEQUENCE:
case internal::OVERLONG_SEQUENCE:
case internal::INVALID_CODE_POINT:
out = utf8::unchecked::append (replacement, out);
out = utf8::unchecked::append(replacement, out);
++start;
// just one replacement mark for the sequence
while (start != end && utf8::internal::is_trail(*start))
@ -76,16 +76,15 @@ namespace utf8
template <typename octet_iterator, typename output_iterator>
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
{
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd);
return utf8::unchecked::replace_invalid(start, end, out, replacement_marker);
}
template <typename octet_iterator>
uint32_t next(octet_iterator& it)
utfchar32_t next(octet_iterator& it)
{
uint32_t cp = utf8::internal::mask8(*it);
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
switch (length) {
utfchar32_t cp = utf8::internal::mask8(*it);
switch (utf8::internal::sequence_length(it)) {
case 1:
break;
case 2:
@ -112,13 +111,13 @@ namespace utf8
}
template <typename octet_iterator>
uint32_t peek_next(octet_iterator it)
utfchar32_t peek_next(octet_iterator it)
{
return utf8::unchecked::next(it);
}
template <typename octet_iterator>
uint32_t prior(octet_iterator& it)
utfchar32_t prior(octet_iterator& it)
{
while (utf8::internal::is_trail(*(--it))) ;
octet_iterator temp = it;
@ -126,7 +125,7 @@ namespace utf8
}
template <typename octet_iterator, typename distance_type>
void advance (octet_iterator& it, distance_type n)
void advance(octet_iterator& it, distance_type n)
{
const distance_type zero(0);
if (n < zero) {
@ -142,7 +141,7 @@ namespace utf8
template <typename octet_iterator>
typename std::iterator_traits<octet_iterator>::difference_type
distance (octet_iterator first, octet_iterator last)
distance(octet_iterator first, octet_iterator last)
{
typename std::iterator_traits<octet_iterator>::difference_type dist;
for (dist = 0; first < last; ++dist)
@ -151,13 +150,13 @@ namespace utf8
}
template <typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result)
{
while (start != end) {
uint32_t cp = utf8::internal::mask16(*start++);
utfchar32_t cp = utf8::internal::mask16(*start++);
// Take care of surrogate pairs first
if (utf8::internal::is_lead_surrogate(cp)) {
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
utfchar32_t trail_surrogate = utf8::internal::mask16(*start++);
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
}
result = utf8::unchecked::append(cp, result);
@ -166,22 +165,22 @@ namespace utf8
}
template <typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result)
{
while (start < end) {
uint32_t cp = utf8::unchecked::next(start);
utfchar32_t cp = utf8::unchecked::next(start);
if (cp > 0xffff) { //make a surrogate pair
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
*result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET);
*result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
}
else
*result++ = static_cast<uint16_t>(cp);
*result++ = static_cast<utfchar16_t>(cp);
}
return result;
}
template <typename octet_iterator, typename u32bit_iterator>
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result)
{
while (start != end)
result = utf8::unchecked::append(*(start++), result);
@ -190,7 +189,7 @@ namespace utf8
}
template <typename octet_iterator, typename u32bit_iterator>
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result)
{
while (start < end)
(*result++) = utf8::unchecked::next(start);
@ -203,16 +202,16 @@ namespace utf8
class iterator {
octet_iterator it;
public:
typedef uint32_t value_type;
typedef uint32_t* pointer;
typedef uint32_t& reference;
typedef utfchar32_t value_type;
typedef utfchar32_t* pointer;
typedef utfchar32_t& reference;
typedef std::ptrdiff_t difference_type;
typedef std::bidirectional_iterator_tag iterator_category;
iterator () {}
explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
// the default "big three" are OK
octet_iterator base () const { return it; }
uint32_t operator * () const
utfchar32_t operator * () const
{
octet_iterator temp = it;
return utf8::unchecked::next(temp);