mirror of
https://github.com/nemtrif/utfcpp.git
synced 2025-04-21 12:39:37 +00:00
Compare commits
No commits in common. "master" and "v3.0" have entirely different histories.
29 changed files with 880 additions and 2774 deletions
75
.github/workflows/cmake-multi-platform.yml
vendored
75
.github/workflows/cmake-multi-platform.yml
vendored
|
|
@ -1,75 +0,0 @@
|
|||
# This starter workflow is for a CMake project running on multiple platforms. There is a different starter workflow if you just want a single platform.
|
||||
# See: https://github.com/actions/starter-workflows/blob/main/ci/cmake-single-platform.yml
|
||||
name: CMake on multiple platforms
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "master" ]
|
||||
pull_request:
|
||||
branches: [ "master" ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
# Set fail-fast to false to ensure that feedback is delivered for all matrix combinations. Consider changing this to true when your workflow is stable.
|
||||
fail-fast: false
|
||||
|
||||
# Set up a matrix to run the following 3 configurations:
|
||||
# 1. <Windows, Release, latest MSVC compiler toolchain on the default runner image, default generator>
|
||||
# 2. <Linux, Release, latest GCC compiler toolchain on the default runner image, default generator>
|
||||
# 3. <Linux, Release, latest Clang compiler toolchain on the default runner image, default generator>
|
||||
#
|
||||
# To add more build types (Release, Debug, RelWithDebInfo, etc.) customize the build_type list.
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
build_type: [Release]
|
||||
c_compiler: [gcc, clang, cl]
|
||||
include:
|
||||
- os: windows-latest
|
||||
c_compiler: cl
|
||||
cpp_compiler: cl
|
||||
- os: ubuntu-latest
|
||||
c_compiler: gcc
|
||||
cpp_compiler: g++
|
||||
- os: ubuntu-latest
|
||||
c_compiler: clang
|
||||
cpp_compiler: clang++
|
||||
exclude:
|
||||
- os: windows-latest
|
||||
c_compiler: gcc
|
||||
- os: windows-latest
|
||||
c_compiler: clang
|
||||
- os: ubuntu-latest
|
||||
c_compiler: cl
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set reusable strings
|
||||
# Turn repeated input strings (such as the build output directory) into step outputs. These step outputs can be used throughout the workflow file.
|
||||
id: strings
|
||||
shell: bash
|
||||
run: |
|
||||
echo "build-output-dir=${{ github.workspace }}/tests/build" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Configure CMake
|
||||
# Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
|
||||
# See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
|
||||
run: >
|
||||
cmake -B ${{ steps.strings.outputs.build-output-dir }}
|
||||
-DCMAKE_CXX_COMPILER=${{ matrix.cpp_compiler }}
|
||||
-DCMAKE_C_COMPILER=${{ matrix.c_compiler }}
|
||||
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
|
||||
-S ${{ github.workspace }}
|
||||
|
||||
- name: Build
|
||||
# Build your program with the given configuration. Note that --config is needed because the default Windows generator is a multi-config generator (Visual Studio generator).
|
||||
run: cmake --build ${{ steps.strings.outputs.build-output-dir }} --config ${{ matrix.build_type }}
|
||||
|
||||
- name: Test
|
||||
working-directory: ${{ steps.strings.outputs.build-output-dir }}
|
||||
# Execute tests defined by the CMake configuration. Note that --build-config is needed because the default Windows generator is a multi-config generator (Visual Studio generator).
|
||||
# See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
|
||||
run: ctest -VV --build-config ${{ matrix.build_type }}
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
|
|
@ -1,3 +0,0 @@
|
|||
[submodule "extern/ftest"]
|
||||
path = extern/ftest
|
||||
url = https://github.com/nemtrif/ftest
|
||||
|
|
@ -1,53 +1,44 @@
|
|||
# This file is deprecated and will be removed in a future release
|
||||
# Please see the instructions for installation in README.md file
|
||||
cmake_minimum_required (VERSION 3.0.2)
|
||||
project (utf8cpp VERSION 2.3.6 LANGUAGES CXX)
|
||||
|
||||
cmake_minimum_required (VERSION 3.14...3.27)
|
||||
project (utf8cpp
|
||||
VERSION 4.0.6
|
||||
LANGUAGES CXX
|
||||
DESCRIPTION "C++ portable library for working with utf-8 encoding")
|
||||
|
||||
add_library(${PROJECT_NAME} INTERFACE)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
option(UTF8_TESTS "Enable tests for UTF8-CPP" On)
|
||||
option(UTF8_SAMPLES "Enable building samples for UTF8-CPP" On)
|
||||
|
||||
add_library(utf8cpp INTERFACE)
|
||||
target_include_directories(utf8cpp INTERFACE
|
||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/source>"
|
||||
$<INSTALL_INTERFACE:include/utf8cpp>
|
||||
"$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/source>"
|
||||
$<INSTALL_INTERFACE:include/utf8cpp>
|
||||
)
|
||||
add_library(utf8::cpp ALIAS utf8cpp)
|
||||
|
||||
include(CMakePackageConfigHelpers)
|
||||
write_basic_package_version_file(
|
||||
"${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
|
||||
VERSION ${PROJECT_VERSION}
|
||||
COMPATIBILITY SameMajorVersion
|
||||
ARCH_INDEPENDENT
|
||||
)
|
||||
if(WIN32 AND NOT CYGWIN)
|
||||
set(DEF_INSTALL_CMAKE_DIR CMake)
|
||||
else()
|
||||
include(GNUInstallDirs) # define CMAKE_INSTALL_*
|
||||
set(DEF_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/utf8cpp)
|
||||
endif()
|
||||
|
||||
install(TARGETS ${PROJECT_NAME}
|
||||
EXPORT ${PROJECT_NAME}Targets
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
PUBLIC_HEADER DESTINATION include COMPONENT Development
|
||||
BUNDLE DESTINATION bin COMPONENT Runtime
|
||||
)
|
||||
install(DIRECTORY source/ DESTINATION include/utf8cpp)
|
||||
install(TARGETS utf8cpp EXPORT utf8cppConfig)
|
||||
install(EXPORT utf8cppConfig DESTINATION ${DEF_INSTALL_CMAKE_DIR})
|
||||
|
||||
configure_package_config_file(
|
||||
"${PROJECT_SOURCE_DIR}/${PROJECT_NAME}Config.cmake.in"
|
||||
"${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
|
||||
INSTALL_DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake
|
||||
)
|
||||
if(UTF8_SAMPLES)
|
||||
add_executable(docsample ${PROJECT_SOURCE_DIR}/samples/docsample.cpp)
|
||||
|
||||
install(EXPORT ${PROJECT_NAME}Targets
|
||||
FILE ${PROJECT_NAME}Targets.cmake
|
||||
NAMESPACE ${PROJECT_NAME}::
|
||||
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake)
|
||||
target_link_libraries(docsample PRIVATE utf8::cpp)
|
||||
endif()
|
||||
|
||||
install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
|
||||
"${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake"
|
||||
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake)
|
||||
if(UTF8_TESTS)
|
||||
add_executable(smoke ${PROJECT_SOURCE_DIR}/test_drivers/smoke_test/test.cpp)
|
||||
add_executable(cpp11 ${PROJECT_SOURCE_DIR}/test_drivers/smoke_test/cpp11.cpp)
|
||||
add_executable(negative ${PROJECT_SOURCE_DIR}/test_drivers/negative/negative.cpp)
|
||||
|
||||
install(FILES ${PROJECT_SOURCE_DIR}/source/utf8.h DESTINATION include/utf8cpp)
|
||||
install(DIRECTORY ${PROJECT_SOURCE_DIR}/source/utf8 DESTINATION
|
||||
include/utf8cpp)
|
||||
target_link_libraries(smoke PRIVATE utf8::cpp)
|
||||
target_link_libraries(cpp11 PRIVATE utf8::cpp)
|
||||
target_link_libraries(negative PRIVATE utf8::cpp)
|
||||
|
||||
enable_testing()
|
||||
add_test(smoke_test smoke)
|
||||
add_test(cpp11_test cpp11)
|
||||
add_test(negative_test negative ${PROJECT_SOURCE_DIR}/test_data/negative/utf8_invalid.txt)
|
||||
endif()
|
||||
|
|
|
|||
1
extern/ftest
vendored
1
extern/ftest
vendored
|
|
@ -1 +0,0 @@
|
|||
Subproject commit c4ad4af0946b73ce1a40cbc72205d15d196c7e06
|
||||
64
samples/docsample.cpp
Normal file
64
samples/docsample.cpp
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
#include "../source/utf8.h"
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
using namespace std;
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
if (argc != 2) {
|
||||
cout << "\nUsage: docsample filename\n";
|
||||
return 0;
|
||||
}
|
||||
const char* test_file_path = argv[1];
|
||||
// Open the test file (must be UTF-8 encoded)
|
||||
ifstream fs8(test_file_path);
|
||||
if (!fs8.is_open()) {
|
||||
cout << "Could not open " << test_file_path << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned line_count = 1;
|
||||
string line;
|
||||
// Play with all the lines in the file
|
||||
while (getline(fs8, line)) {
|
||||
// check for invalid utf-8 (for a simple yes/no check, there is also utf8::is_valid function)
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
auto end_it = utf8::find_invalid(line.begin(), line.end());
|
||||
#else
|
||||
string::iterator end_it = utf8::find_invalid(line.begin(), line.end());
|
||||
#endif // C++ 11
|
||||
if (end_it != line.end()) {
|
||||
cout << "Invalid UTF-8 encoding detected at line " << line_count << "\n";
|
||||
cout << "This part is fine: " << string(line.begin(), end_it) << "\n";
|
||||
}
|
||||
// Get the line length (at least for the valid part)
|
||||
int length = utf8::distance(line.begin(), end_it);
|
||||
cout << "Length of line " << line_count << " is " << length << "\n";
|
||||
|
||||
// Convert it to utf-16
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
u16string utf16line = utf8::utf8to16(line);
|
||||
#else
|
||||
vector<unsigned short> utf16line;
|
||||
utf8::utf8to16(line.begin(), end_it, back_inserter(utf16line));
|
||||
#endif // C++ 11
|
||||
// And back to utf-8;
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
string utf8line = utf8::utf16to8(utf16line);
|
||||
#else
|
||||
string utf8line;
|
||||
utf8::utf16to8(utf16line.begin(), utf16line.end(), back_inserter(utf8line));
|
||||
#endif // C++ 11
|
||||
// Confirm that the conversion went OK:
|
||||
if (utf8line != string(line.begin(), end_it))
|
||||
cout << "Error in UTF-16 conversion at line: " << line_count << "\n";
|
||||
|
||||
line_count++;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -28,19 +28,11 @@ DEALINGS IN THE SOFTWARE.
|
|||
#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
/*
|
||||
To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro
|
||||
and set it to one of the values used by the __cplusplus predefined macro.
|
||||
|
||||
For instance,
|
||||
#define UTF_CPP_CPLUSPLUS 199711L
|
||||
will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard.
|
||||
Some library features will be disabled.
|
||||
|
||||
If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus.
|
||||
*/
|
||||
|
||||
#include "utf8/checked.h"
|
||||
#include "utf8/unchecked.h"
|
||||
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
#include "utf8/cpp11.h"
|
||||
#endif // C++ 11 or later
|
||||
|
||||
#endif // header guard
|
||||
|
|
|
|||
|
|
@ -39,62 +39,64 @@ namespace utf8
|
|||
|
||||
// Exceptions that may be thrown from the library functions.
|
||||
class invalid_code_point : public exception {
|
||||
utfchar32_t cp;
|
||||
uint32_t cp;
|
||||
public:
|
||||
invalid_code_point(utfchar32_t codepoint) : cp(codepoint) {}
|
||||
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; }
|
||||
utfchar32_t code_point() const {return cp;}
|
||||
invalid_code_point(uint32_t codepoint) : cp(codepoint) {}
|
||||
virtual const char* what() const throw() { return "Invalid code point"; }
|
||||
uint32_t code_point() const {return cp;}
|
||||
};
|
||||
|
||||
class invalid_utf8 : public exception {
|
||||
utfchar8_t u8;
|
||||
uint8_t u8;
|
||||
public:
|
||||
invalid_utf8 (utfchar8_t u) : u8(u) {}
|
||||
invalid_utf8 (char c) : u8(static_cast<utfchar8_t>(c)) {}
|
||||
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; }
|
||||
utfchar8_t utf8_octet() const {return u8;}
|
||||
invalid_utf8 (uint8_t u) : u8(u) {}
|
||||
virtual const char* what() const throw() { return "Invalid UTF-8"; }
|
||||
uint8_t utf8_octet() const {return u8;}
|
||||
};
|
||||
|
||||
class invalid_utf16 : public exception {
|
||||
utfchar16_t u16;
|
||||
uint16_t u16;
|
||||
public:
|
||||
invalid_utf16 (utfchar16_t u) : u16(u) {}
|
||||
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; }
|
||||
utfchar16_t utf16_word() const {return u16;}
|
||||
invalid_utf16 (uint16_t u) : u16(u) {}
|
||||
virtual const char* what() const throw() { return "Invalid UTF-16"; }
|
||||
uint16_t utf16_word() const {return u16;}
|
||||
};
|
||||
|
||||
class not_enough_room : public exception {
|
||||
public:
|
||||
virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Not enough space"; }
|
||||
virtual const char* what() const throw() { return "Not enough space"; }
|
||||
};
|
||||
|
||||
/// The library API - functions intended to be called by the users
|
||||
|
||||
template <typename octet_iterator>
|
||||
octet_iterator append(utfchar32_t cp, octet_iterator result)
|
||||
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||
{
|
||||
if (!utf8::internal::is_code_point_valid(cp))
|
||||
throw invalid_code_point(cp);
|
||||
|
||||
return internal::append(cp, result);
|
||||
}
|
||||
|
||||
inline void append(utfchar32_t cp, std::string& s)
|
||||
{
|
||||
append(cp, std::back_inserter(s));
|
||||
}
|
||||
|
||||
template <typename word_iterator>
|
||||
word_iterator append16(utfchar32_t cp, word_iterator result)
|
||||
{
|
||||
if (!utf8::internal::is_code_point_valid(cp))
|
||||
throw invalid_code_point(cp);
|
||||
|
||||
return internal::append16(cp, result);
|
||||
if (cp < 0x80) // one octet
|
||||
*(result++) = static_cast<uint8_t>(cp);
|
||||
else if (cp < 0x800) { // two octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else if (cp < 0x10000) { // three octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else { // four octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename output_iterator>
|
||||
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement)
|
||||
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement)
|
||||
{
|
||||
while (start != end) {
|
||||
octet_iterator sequence_start = start;
|
||||
|
|
@ -129,28 +131,14 @@ namespace utf8
|
|||
template <typename octet_iterator, typename output_iterator>
|
||||
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
|
||||
{
|
||||
static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd);
|
||||
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd);
|
||||
return utf8::replace_invalid(start, end, out, replacement_marker);
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(const std::string& s, utfchar32_t replacement)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(const std::string& s)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t next(octet_iterator& it, octet_iterator end)
|
||||
uint32_t next(octet_iterator& it, octet_iterator end)
|
||||
{
|
||||
utfchar32_t cp = 0;
|
||||
uint32_t cp = 0;
|
||||
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
|
||||
switch (err_code) {
|
||||
case internal::UTF8_OK :
|
||||
|
|
@ -160,31 +148,21 @@ namespace utf8
|
|||
case internal::INVALID_LEAD :
|
||||
case internal::INCOMPLETE_SEQUENCE :
|
||||
case internal::OVERLONG_SEQUENCE :
|
||||
throw invalid_utf8(static_cast<utfchar8_t>(*it));
|
||||
throw invalid_utf8(*it);
|
||||
case internal::INVALID_CODE_POINT :
|
||||
throw invalid_code_point(cp);
|
||||
}
|
||||
return cp;
|
||||
}
|
||||
|
||||
template <typename word_iterator>
|
||||
utfchar32_t next16(word_iterator& it, word_iterator end)
|
||||
{
|
||||
utfchar32_t cp = 0;
|
||||
internal::utf_error err_code = utf8::internal::validate_next16(it, end, cp);
|
||||
if (err_code == internal::NOT_ENOUGH_ROOM)
|
||||
throw not_enough_room();
|
||||
return cp;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t peek_next(octet_iterator it, octet_iterator end)
|
||||
uint32_t peek_next(octet_iterator it, octet_iterator end)
|
||||
{
|
||||
return utf8::next(it, end);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t prior(octet_iterator& it, octet_iterator start)
|
||||
uint32_t prior(octet_iterator& it, octet_iterator start)
|
||||
{
|
||||
// can't do much if it == start
|
||||
if (it == start)
|
||||
|
|
@ -227,23 +205,23 @@ namespace utf8
|
|||
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end) {
|
||||
utfchar32_t cp = utf8::internal::mask16(*start++);
|
||||
uint32_t cp = utf8::internal::mask16(*start++);
|
||||
// Take care of surrogate pairs first
|
||||
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||
if (start != end) {
|
||||
const utfchar32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
if (utf8::internal::is_trail_surrogate(trail_surrogate))
|
||||
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||
else
|
||||
throw invalid_utf16(static_cast<utfchar16_t>(trail_surrogate));
|
||||
throw invalid_utf16(static_cast<uint16_t>(trail_surrogate));
|
||||
}
|
||||
else
|
||||
throw invalid_utf16(static_cast<utfchar16_t>(cp));
|
||||
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||
|
||||
}
|
||||
// Lone trail surrogate
|
||||
else if (utf8::internal::is_trail_surrogate(cp))
|
||||
throw invalid_utf16(static_cast<utfchar16_t>(cp));
|
||||
throw invalid_utf16(static_cast<uint16_t>(cp));
|
||||
|
||||
result = utf8::append(cp, result);
|
||||
}
|
||||
|
|
@ -254,13 +232,13 @@ namespace utf8
|
|||
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||
{
|
||||
while (start < end) {
|
||||
const utfchar32_t cp = utf8::next(start, end);
|
||||
uint32_t cp = utf8::next(start, end);
|
||||
if (cp > 0xffff) { //make a surrogate pair
|
||||
*result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
}
|
||||
else
|
||||
*result++ = static_cast<utfchar16_t>(cp);
|
||||
*result++ = static_cast<uint16_t>(cp);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -285,16 +263,11 @@ namespace utf8
|
|||
|
||||
// The iterator class
|
||||
template <typename octet_iterator>
|
||||
class iterator {
|
||||
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||
octet_iterator it;
|
||||
octet_iterator range_start;
|
||||
octet_iterator range_end;
|
||||
public:
|
||||
typedef utfchar32_t value_type;
|
||||
typedef utfchar32_t* pointer;
|
||||
typedef utfchar32_t& reference;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef std::bidirectional_iterator_tag iterator_category;
|
||||
iterator () {}
|
||||
explicit iterator (const octet_iterator& octet_it,
|
||||
const octet_iterator& rangestart,
|
||||
|
|
@ -306,7 +279,7 @@ namespace utf8
|
|||
}
|
||||
// the default "big three" are OK
|
||||
octet_iterator base () const { return it; }
|
||||
utfchar32_t operator * () const
|
||||
uint32_t operator * () const
|
||||
{
|
||||
octet_iterator temp = it;
|
||||
return utf8::next(temp, range_end);
|
||||
|
|
@ -347,13 +320,5 @@ namespace utf8
|
|||
|
||||
} // namespace utf8
|
||||
|
||||
#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later
|
||||
#include "cpp20.h"
|
||||
#elif UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later
|
||||
#include "cpp17.h"
|
||||
#elif UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
|
||||
#include "cpp11.h"
|
||||
#endif // C++ 11 or later
|
||||
|
||||
#endif //header guard
|
||||
|
||||
|
|
|
|||
|
|
@ -29,47 +29,15 @@ DEALINGS IN THE SOFTWARE.
|
|||
#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include <iterator>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
// Determine the C++ standard version.
|
||||
// If the user defines UTF_CPP_CPLUSPLUS, use that.
|
||||
// Otherwise, trust the unreliable predefined macro __cplusplus
|
||||
|
||||
#if !defined UTF_CPP_CPLUSPLUS
|
||||
#define UTF_CPP_CPLUSPLUS __cplusplus
|
||||
#endif
|
||||
|
||||
#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
|
||||
#define UTF_CPP_OVERRIDE override
|
||||
#define UTF_CPP_NOEXCEPT noexcept
|
||||
#define UTF_CPP_STATIC_ASSERT(condition) static_assert(condition, "UTFCPP static assert");
|
||||
#else // C++ 98/03
|
||||
#define UTF_CPP_OVERRIDE
|
||||
#define UTF_CPP_NOEXCEPT throw()
|
||||
// Simulate static_assert:
|
||||
template <bool Condition> struct StaticAssert {static void assert() {int static_assert_impl[(Condition ? 1 : -1)];} };
|
||||
template <> struct StaticAssert<true> {static void assert() {}};
|
||||
#define UTF_CPP_STATIC_ASSERT(condition) StaticAssert<condition>::assert();
|
||||
#endif // C++ 11 or later
|
||||
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
// The typedefs for 8-bit, 16-bit and 32-bit code units
|
||||
#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later
|
||||
#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later
|
||||
typedef char8_t utfchar8_t;
|
||||
#else // C++ 11/14/17
|
||||
typedef unsigned char utfchar8_t;
|
||||
#endif
|
||||
typedef char16_t utfchar16_t;
|
||||
typedef char32_t utfchar32_t;
|
||||
#else // C++ 98/03
|
||||
typedef unsigned char utfchar8_t;
|
||||
typedef unsigned short utfchar16_t;
|
||||
typedef unsigned int utfchar32_t;
|
||||
#endif // C++ 11 or later
|
||||
// The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
|
||||
// You may need to change them to match your system.
|
||||
// These typedefs have the same names as ones from cstdint, or boost/cstdint
|
||||
typedef unsigned char uint8_t;
|
||||
typedef unsigned short uint16_t;
|
||||
typedef unsigned int uint32_t;
|
||||
|
||||
// Helper code - not intended to be directly called by the library users. May be changed at any time
|
||||
namespace internal
|
||||
|
|
@ -77,62 +45,61 @@ namespace internal
|
|||
// Unicode constants
|
||||
// Leading (high) surrogates: 0xd800 - 0xdbff
|
||||
// Trailing (low) surrogates: 0xdc00 - 0xdfff
|
||||
const utfchar16_t LEAD_SURROGATE_MIN = 0xd800u;
|
||||
const utfchar16_t LEAD_SURROGATE_MAX = 0xdbffu;
|
||||
const utfchar16_t TRAIL_SURROGATE_MIN = 0xdc00u;
|
||||
const utfchar16_t TRAIL_SURROGATE_MAX = 0xdfffu;
|
||||
const utfchar16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10)
|
||||
const utfchar32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
|
||||
const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
|
||||
const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
|
||||
const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
|
||||
const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
|
||||
const uint16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10)
|
||||
const uint32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN
|
||||
|
||||
// Maximum valid value for a Unicode code point
|
||||
const utfchar32_t CODE_POINT_MAX = 0x0010ffffu;
|
||||
const uint32_t CODE_POINT_MAX = 0x0010ffffu;
|
||||
|
||||
template<typename octet_type>
|
||||
inline utfchar8_t mask8(octet_type oc)
|
||||
inline uint8_t mask8(octet_type oc)
|
||||
{
|
||||
return static_cast<utfchar8_t>(0xff & oc);
|
||||
return static_cast<uint8_t>(0xff & oc);
|
||||
}
|
||||
template<typename u16_type>
|
||||
inline utfchar16_t mask16(u16_type oc)
|
||||
inline uint16_t mask16(u16_type oc)
|
||||
{
|
||||
return static_cast<utfchar16_t>(0xffff & oc);
|
||||
return static_cast<uint16_t>(0xffff & oc);
|
||||
}
|
||||
|
||||
template<typename octet_type>
|
||||
inline bool is_trail(octet_type oc)
|
||||
{
|
||||
return ((utf8::internal::mask8(oc) >> 6) == 0x2);
|
||||
}
|
||||
|
||||
inline bool is_lead_surrogate(utfchar32_t cp)
|
||||
template <typename u16>
|
||||
inline bool is_lead_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
inline bool is_trail_surrogate(utfchar32_t cp)
|
||||
template <typename u16>
|
||||
inline bool is_trail_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
inline bool is_surrogate(utfchar32_t cp)
|
||||
template <typename u16>
|
||||
inline bool is_surrogate(u16 cp)
|
||||
{
|
||||
return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
|
||||
}
|
||||
|
||||
inline bool is_code_point_valid(utfchar32_t cp)
|
||||
template <typename u32>
|
||||
inline bool is_code_point_valid(u32 cp)
|
||||
{
|
||||
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
|
||||
}
|
||||
|
||||
inline bool is_in_bmp(utfchar32_t cp)
|
||||
{
|
||||
return cp < utfchar32_t(0x10000);
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
int sequence_length(octet_iterator lead_it)
|
||||
inline typename std::iterator_traits<octet_iterator>::difference_type
|
||||
sequence_length(octet_iterator lead_it)
|
||||
{
|
||||
const utfchar8_t lead = utf8::internal::mask8(*lead_it);
|
||||
uint8_t lead = utf8::internal::mask8(*lead_it);
|
||||
if (lead < 0x80)
|
||||
return 1;
|
||||
else if ((lead >> 5) == 0x6)
|
||||
|
|
@ -145,7 +112,8 @@ namespace internal
|
|||
return 0;
|
||||
}
|
||||
|
||||
inline bool is_overlong_sequence(utfchar32_t cp, int length)
|
||||
template <typename octet_difference_type>
|
||||
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
|
||||
{
|
||||
if (cp < 0x80) {
|
||||
if (length != 1)
|
||||
|
|
@ -159,6 +127,7 @@ namespace internal
|
|||
if (length != 3)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -166,7 +135,7 @@ namespace internal
|
|||
|
||||
/// Helper for get_sequence_x
|
||||
template <typename octet_iterator>
|
||||
utf_error increase_safely(octet_iterator& it, const octet_iterator end)
|
||||
utf_error increase_safely(octet_iterator& it, octet_iterator end)
|
||||
{
|
||||
if (++it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -177,11 +146,11 @@ namespace internal
|
|||
return UTF8_OK;
|
||||
}
|
||||
|
||||
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
|
||||
#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}
|
||||
|
||||
/// get_sequence_x functions decode utf-8 sequences of the length x
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
|
||||
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -192,7 +161,7 @@ namespace internal
|
|||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
|
||||
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -207,7 +176,7 @@ namespace internal
|
|||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
|
||||
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -220,13 +189,13 @@ namespace internal
|
|||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = static_cast<utfchar32_t>(code_point + ((*it) & 0x3f));
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
|
||||
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -239,11 +208,11 @@ namespace internal
|
|||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = static_cast<utfchar32_t>(code_point + ((utf8::internal::mask8(*it) << 6) & 0xfff));
|
||||
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||
|
||||
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
|
||||
|
||||
code_point = static_cast<utfchar32_t>(code_point + ((*it) & 0x3f));
|
||||
code_point += (*it) & 0x3f;
|
||||
|
||||
return UTF8_OK;
|
||||
}
|
||||
|
|
@ -251,7 +220,7 @@ namespace internal
|
|||
#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
|
||||
|
||||
template <typename octet_iterator>
|
||||
utf_error validate_next(octet_iterator& it, octet_iterator end, utfchar32_t& code_point)
|
||||
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
|
||||
{
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
|
|
@ -260,9 +229,10 @@ namespace internal
|
|||
// Of course, it does not make much sense with i.e. stream iterators
|
||||
octet_iterator original_it = it;
|
||||
|
||||
utfchar32_t cp = 0;
|
||||
uint32_t cp = 0;
|
||||
// Determine the sequence length based on the lead octet
|
||||
const int length = utf8::internal::sequence_length(it);
|
||||
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
|
||||
const octet_difference_type length = utf8::internal::sequence_length(it);
|
||||
|
||||
// Get trail octets and calculate the code point
|
||||
utf_error err = UTF8_OK;
|
||||
|
|
@ -306,138 +276,16 @@ namespace internal
|
|||
|
||||
template <typename octet_iterator>
|
||||
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
|
||||
utfchar32_t ignored;
|
||||
uint32_t ignored;
|
||||
return utf8::internal::validate_next(it, end, ignored);
|
||||
}
|
||||
|
||||
template <typename word_iterator>
|
||||
utf_error validate_next16(word_iterator& it, word_iterator end, utfchar32_t& code_point)
|
||||
{
|
||||
// Make sure the iterator dereferences a large enough type
|
||||
typedef typename std::iterator_traits<word_iterator>::value_type word_type;
|
||||
UTF_CPP_STATIC_ASSERT(sizeof(word_type) >= sizeof(utfchar16_t));
|
||||
// Check the edge case:
|
||||
if (it == end)
|
||||
return NOT_ENOUGH_ROOM;
|
||||
// Save the original value of it so we can go back in case of failure
|
||||
// Of course, it does not make much sense with i.e. stream iterators
|
||||
word_iterator original_it = it;
|
||||
|
||||
utf_error err = UTF8_OK;
|
||||
|
||||
const utfchar16_t first_word = *it++;
|
||||
if (!is_surrogate(first_word)) {
|
||||
code_point = first_word;
|
||||
return UTF8_OK;
|
||||
}
|
||||
else {
|
||||
if (it == end)
|
||||
err = NOT_ENOUGH_ROOM;
|
||||
else if (is_lead_surrogate(first_word)) {
|
||||
const utfchar16_t second_word = *it++;
|
||||
if (is_trail_surrogate(second_word)) {
|
||||
code_point = static_cast<utfchar32_t>(first_word << 10) + second_word + SURROGATE_OFFSET;
|
||||
return UTF8_OK;
|
||||
} else
|
||||
err = INCOMPLETE_SEQUENCE;
|
||||
|
||||
} else {
|
||||
err = INVALID_LEAD;
|
||||
}
|
||||
}
|
||||
// error branch
|
||||
it = original_it;
|
||||
return err;
|
||||
}
|
||||
|
||||
// Internal implementation of both checked and unchecked append() function
|
||||
// This function will be invoked by the overloads below, as they will know
|
||||
// the octet_type.
|
||||
template <typename octet_iterator, typename octet_type>
|
||||
octet_iterator append(utfchar32_t cp, octet_iterator result) {
|
||||
if (cp < 0x80) // one octet
|
||||
*(result++) = static_cast<octet_type>(cp);
|
||||
else if (cp < 0x800) { // two octets
|
||||
*(result++) = static_cast<octet_type>((cp >> 6) | 0xc0);
|
||||
*(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else if (cp < 0x10000) { // three octets
|
||||
*(result++) = static_cast<octet_type>((cp >> 12) | 0xe0);
|
||||
*(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else { // four octets
|
||||
*(result++) = static_cast<octet_type>((cp >> 18) | 0xf0);
|
||||
*(result++) = static_cast<octet_type>(((cp >> 12) & 0x3f)| 0x80);
|
||||
*(result++) = static_cast<octet_type>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<octet_type>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// One of the following overloads will be invoked from the API calls
|
||||
|
||||
// A simple (but dangerous) case: the caller appends byte(s) to a char array
|
||||
inline char* append(utfchar32_t cp, char* result) {
|
||||
return append<char*, char>(cp, result);
|
||||
}
|
||||
|
||||
// Hopefully, most common case: the caller uses back_inserter
|
||||
// i.e. append(cp, std::back_inserter(str));
|
||||
template<typename container_type>
|
||||
std::back_insert_iterator<container_type> append
|
||||
(utfchar32_t cp, std::back_insert_iterator<container_type> result) {
|
||||
return append<std::back_insert_iterator<container_type>,
|
||||
typename container_type::value_type>(cp, result);
|
||||
}
|
||||
|
||||
// The caller uses some other kind of output operator - not covered above
|
||||
// Note that in this case we are not able to determine octet_type
|
||||
// so we assume it's utfchar8_t; that can cause a conversion warning if we are wrong.
|
||||
template <typename octet_iterator>
|
||||
octet_iterator append(utfchar32_t cp, octet_iterator result) {
|
||||
return append<octet_iterator, utfchar8_t>(cp, result);
|
||||
}
|
||||
|
||||
// Internal implementation of both checked and unchecked append16() function
|
||||
// This function will be invoked by the overloads below, as they will know
|
||||
// the word_type.
|
||||
template <typename word_iterator, typename word_type>
|
||||
word_iterator append16(utfchar32_t cp, word_iterator result) {
|
||||
UTF_CPP_STATIC_ASSERT(sizeof(word_type) >= sizeof(utfchar16_t));
|
||||
if (is_in_bmp(cp))
|
||||
*(result++) = static_cast<word_type>(cp);
|
||||
else {
|
||||
// Code points from the supplementary planes are encoded via surrogate pairs
|
||||
*(result++) = static_cast<word_type>(LEAD_OFFSET + (cp >> 10));
|
||||
*(result++) = static_cast<word_type>(TRAIL_SURROGATE_MIN + (cp & 0x3FF));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Hopefully, most common case: the caller uses back_inserter
|
||||
// i.e. append16(cp, std::back_inserter(str));
|
||||
template<typename container_type>
|
||||
std::back_insert_iterator<container_type> append16
|
||||
(utfchar32_t cp, std::back_insert_iterator<container_type> result) {
|
||||
return append16<std::back_insert_iterator<container_type>,
|
||||
typename container_type::value_type>(cp, result);
|
||||
}
|
||||
|
||||
// The caller uses some other kind of output operator - not covered above
|
||||
// Note that in this case we are not able to determine word_type
|
||||
// so we assume it's utfchar16_t; that can cause a conversion warning if we are wrong.
|
||||
template <typename word_iterator>
|
||||
word_iterator append16(utfchar32_t cp, word_iterator result) {
|
||||
return append16<word_iterator, utfchar16_t>(cp, result);
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
/// The library API - functions intended to be called by the users
|
||||
|
||||
// Byte order mark
|
||||
const utfchar8_t bom[] = {0xef, 0xbb, 0xbf};
|
||||
const uint8_t bom[] = {0xef, 0xbb, 0xbf};
|
||||
|
||||
template <typename octet_iterator>
|
||||
octet_iterator find_invalid(octet_iterator start, octet_iterator end)
|
||||
|
|
@ -451,36 +299,12 @@ namespace internal
|
|||
return result;
|
||||
}
|
||||
|
||||
inline const char* find_invalid(const char* str)
|
||||
{
|
||||
const char* end = str + std::strlen(str);
|
||||
return find_invalid(str, end);
|
||||
}
|
||||
|
||||
inline std::size_t find_invalid(const std::string& s)
|
||||
{
|
||||
std::string::const_iterator invalid = find_invalid(s.begin(), s.end());
|
||||
return (invalid == s.end()) ? std::string::npos : static_cast<std::size_t>(invalid - s.begin());
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline bool is_valid(octet_iterator start, octet_iterator end)
|
||||
{
|
||||
return (utf8::find_invalid(start, end) == end);
|
||||
}
|
||||
|
||||
inline bool is_valid(const char* str)
|
||||
{
|
||||
return (*(utf8::find_invalid(str)) == '\0');
|
||||
}
|
||||
|
||||
inline bool is_valid(const std::string& s)
|
||||
{
|
||||
return is_valid(s.begin(), s.end());
|
||||
}
|
||||
|
||||
|
||||
|
||||
template <typename octet_iterator>
|
||||
inline bool starts_with_bom (octet_iterator it, octet_iterator end)
|
||||
{
|
||||
|
|
@ -489,12 +313,7 @@ namespace internal
|
|||
((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
|
||||
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
|
||||
);
|
||||
}
|
||||
|
||||
inline bool starts_with_bom(const std::string& s)
|
||||
{
|
||||
return starts_with_bom(s.begin(), s.end());
|
||||
}
|
||||
}
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
|
|
|||
|
|
@ -26,44 +26,78 @@ DEALINGS IN THE SOFTWARE.
|
|||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1
|
||||
#define UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1
|
||||
#define UTF8_FOR_CPP_a184c22_cd012_11e8_a8d5_f2801f1b9fd1
|
||||
|
||||
#include "checked.h"
|
||||
#include <string>
|
||||
#include <cassert>
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
inline void append16(utfchar32_t cp, std::u16string& s)
|
||||
|
||||
void append(char32_t cp, std::string& s)
|
||||
{
|
||||
append16(cp, std::back_inserter(s));
|
||||
append(uint32_t(cp), std::back_inserter(s));
|
||||
}
|
||||
|
||||
inline std::string utf16to8(const std::u16string& s)
|
||||
std::string utf16to8(const std::u16string& s)
|
||||
{
|
||||
std::string result;
|
||||
utf16to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u16string utf8to16(const std::string& s)
|
||||
std::u16string utf8to16(const std::string& s)
|
||||
{
|
||||
std::u16string result;
|
||||
utf8to16(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::string utf32to8(const std::u32string& s)
|
||||
std::string utf32to8(const std::u32string& s)
|
||||
{
|
||||
std::string result;
|
||||
utf32to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u32string utf8to32(const std::string& s)
|
||||
std::u32string utf8to32(const std::string& s)
|
||||
{
|
||||
std::u32string result;
|
||||
utf8to32(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
std::size_t find_invalid(const std::string& s)
|
||||
{
|
||||
std::string::const_iterator invalid = find_invalid(s.begin(), s.end());
|
||||
return (invalid == s.end()) ? std::string::npos : (invalid - s.begin());
|
||||
}
|
||||
|
||||
bool is_valid(const std::string& s)
|
||||
{
|
||||
return is_valid(s.begin(), s.end());
|
||||
}
|
||||
|
||||
std::string replace_invalid(const std::string& s, char32_t replacement)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string replace_invalid(const std::string& s)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
bool starts_with_bom(const std::string& s)
|
||||
{
|
||||
return starts_with_bom(s.begin(), s.end());
|
||||
}
|
||||
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
// Copyright 2018 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9
|
||||
#define UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9
|
||||
|
||||
#include "cpp11.h"
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
inline std::string utf16to8(std::u16string_view s)
|
||||
{
|
||||
std::string result;
|
||||
utf16to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u16string utf8to16(std::string_view s)
|
||||
{
|
||||
std::u16string result;
|
||||
utf8to16(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::string utf32to8(std::u32string_view s)
|
||||
{
|
||||
std::string result;
|
||||
utf32to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u32string utf8to32(std::string_view s)
|
||||
{
|
||||
std::u32string result;
|
||||
utf8to32(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::size_t find_invalid(std::string_view s)
|
||||
{
|
||||
std::string_view::const_iterator invalid = find_invalid(s.begin(), s.end());
|
||||
return (invalid == s.end()) ? std::string_view::npos : static_cast<std::size_t>(invalid - s.begin());
|
||||
}
|
||||
|
||||
inline bool is_valid(std::string_view s)
|
||||
{
|
||||
return is_valid(s.begin(), s.end());
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(std::string_view s, char32_t replacement)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(std::string_view s)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline bool starts_with_bom(std::string_view s)
|
||||
{
|
||||
return starts_with_bom(s.begin(), s.end());
|
||||
}
|
||||
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
// Copyright 2022 Nemanja Trifunovic
|
||||
|
||||
/*
|
||||
Permission is hereby granted, free of charge, to any person or organization
|
||||
obtaining a copy of the software and accompanying documentation covered by
|
||||
this license (the "Software") to use, reproduce, display, distribute,
|
||||
execute, and transmit the Software, and to prepare derivative works of the
|
||||
Software, and to permit third-parties to whom the Software is furnished to
|
||||
do so, all subject to the following:
|
||||
|
||||
The copyright notices in the Software and this entire statement, including
|
||||
the above license grant, this restriction and the following disclaimer,
|
||||
must be included in all copies of the Software, in whole or in part, and
|
||||
all derivative works of the Software, unless such copies or derivative
|
||||
works are solely in the form of machine-executable object code generated by
|
||||
a source language processor.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
|
||||
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
|
||||
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9
|
||||
#define UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9
|
||||
|
||||
#include "cpp17.h"
|
||||
|
||||
namespace utf8
|
||||
{
|
||||
inline std::u8string utf16tou8(const std::u16string& s)
|
||||
{
|
||||
std::u8string result;
|
||||
utf16to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u8string utf16tou8(std::u16string_view s)
|
||||
{
|
||||
std::u8string result;
|
||||
utf16to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u16string utf8to16(const std::u8string& s)
|
||||
{
|
||||
std::u16string result;
|
||||
utf8to16(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u16string utf8to16(const std::u8string_view& s)
|
||||
{
|
||||
std::u16string result;
|
||||
utf8to16(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u8string utf32tou8(const std::u32string& s)
|
||||
{
|
||||
std::u8string result;
|
||||
utf32to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u8string utf32tou8(const std::u32string_view& s)
|
||||
{
|
||||
std::u8string result;
|
||||
utf32to8(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u32string utf8to32(const std::u8string& s)
|
||||
{
|
||||
std::u32string result;
|
||||
utf8to32(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u32string utf8to32(const std::u8string_view& s)
|
||||
{
|
||||
std::u32string result;
|
||||
utf8to32(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::size_t find_invalid(const std::u8string& s)
|
||||
{
|
||||
std::u8string::const_iterator invalid = find_invalid(s.begin(), s.end());
|
||||
return (invalid == s.end()) ? std::string_view::npos : static_cast<std::size_t>(invalid - s.begin());
|
||||
}
|
||||
|
||||
inline bool is_valid(const std::u8string& s)
|
||||
{
|
||||
return is_valid(s.begin(), s.end());
|
||||
}
|
||||
|
||||
inline std::u8string replace_invalid(const std::u8string& s, char32_t replacement)
|
||||
{
|
||||
std::u8string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::u8string replace_invalid(const std::u8string& s)
|
||||
{
|
||||
std::u8string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
inline bool starts_with_bom(const std::u8string& s)
|
||||
{
|
||||
return starts_with_bom(s.begin(), s.end());
|
||||
}
|
||||
|
||||
} // namespace utf8
|
||||
|
||||
#endif // header guard
|
||||
|
||||
|
|
@ -32,79 +32,37 @@ DEALINGS IN THE SOFTWARE.
|
|||
|
||||
namespace utf8
|
||||
{
|
||||
namespace unchecked
|
||||
namespace unchecked
|
||||
{
|
||||
template <typename octet_iterator>
|
||||
octet_iterator append(utfchar32_t cp, octet_iterator result)
|
||||
octet_iterator append(uint32_t cp, octet_iterator result)
|
||||
{
|
||||
return internal::append(cp, result);
|
||||
}
|
||||
|
||||
template <typename word_iterator>
|
||||
word_iterator append16(utfchar32_t cp, word_iterator result)
|
||||
{
|
||||
return internal::append16(cp, result);
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename output_iterator>
|
||||
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement)
|
||||
{
|
||||
while (start != end) {
|
||||
octet_iterator sequence_start = start;
|
||||
internal::utf_error err_code = utf8::internal::validate_next(start, end);
|
||||
switch (err_code) {
|
||||
case internal::UTF8_OK :
|
||||
for (octet_iterator it = sequence_start; it != start; ++it)
|
||||
*out++ = *it;
|
||||
break;
|
||||
case internal::NOT_ENOUGH_ROOM:
|
||||
out = utf8::unchecked::append(replacement, out);
|
||||
start = end;
|
||||
break;
|
||||
case internal::INVALID_LEAD:
|
||||
out = utf8::unchecked::append(replacement, out);
|
||||
++start;
|
||||
break;
|
||||
case internal::INCOMPLETE_SEQUENCE:
|
||||
case internal::OVERLONG_SEQUENCE:
|
||||
case internal::INVALID_CODE_POINT:
|
||||
out = utf8::unchecked::append(replacement, out);
|
||||
++start;
|
||||
// just one replacement mark for the sequence
|
||||
while (start != end && utf8::internal::is_trail(*start))
|
||||
++start;
|
||||
break;
|
||||
}
|
||||
if (cp < 0x80) // one octet
|
||||
*(result++) = static_cast<uint8_t>(cp);
|
||||
else if (cp < 0x800) { // two octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 6) | 0xc0);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else if (cp < 0x10000) { // three octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 12) | 0xe0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
else { // four octets
|
||||
*(result++) = static_cast<uint8_t>((cp >> 18) | 0xf0);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 12) & 0x3f)| 0x80);
|
||||
*(result++) = static_cast<uint8_t>(((cp >> 6) & 0x3f) | 0x80);
|
||||
*(result++) = static_cast<uint8_t>((cp & 0x3f) | 0x80);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename output_iterator>
|
||||
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out)
|
||||
{
|
||||
static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd);
|
||||
return utf8::unchecked::replace_invalid(start, end, out, replacement_marker);
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(const std::string& s, utfchar32_t replacement)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement);
|
||||
return result;
|
||||
}
|
||||
|
||||
inline std::string replace_invalid(const std::string& s)
|
||||
{
|
||||
std::string result;
|
||||
replace_invalid(s.begin(), s.end(), std::back_inserter(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t next(octet_iterator& it)
|
||||
uint32_t next(octet_iterator& it)
|
||||
{
|
||||
utfchar32_t cp = utf8::internal::mask8(*it);
|
||||
switch (utf8::internal::sequence_length(it)) {
|
||||
uint32_t cp = utf8::internal::mask8(*it);
|
||||
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it);
|
||||
switch (length) {
|
||||
case 1:
|
||||
break;
|
||||
case 2:
|
||||
|
|
@ -115,15 +73,15 @@ namespace utf8
|
|||
++it;
|
||||
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
|
||||
++it;
|
||||
cp = static_cast<utfchar32_t>(cp + ((*it) & 0x3f));
|
||||
cp += (*it) & 0x3f;
|
||||
break;
|
||||
case 4:
|
||||
++it;
|
||||
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
|
||||
++it;
|
||||
cp = static_cast<utfchar32_t>(cp + ((utf8::internal::mask8(*it) << 6) & 0xfff));
|
||||
cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
|
||||
++it;
|
||||
cp = static_cast<utfchar32_t>(cp + ((*it) & 0x3f));
|
||||
cp += (*it) & 0x3f;
|
||||
break;
|
||||
}
|
||||
++it;
|
||||
|
|
@ -131,22 +89,13 @@ namespace utf8
|
|||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t peek_next(octet_iterator it)
|
||||
uint32_t peek_next(octet_iterator it)
|
||||
{
|
||||
return utf8::unchecked::next(it);
|
||||
}
|
||||
|
||||
template <typename word_iterator>
|
||||
utfchar32_t next16(word_iterator& it)
|
||||
{
|
||||
utfchar32_t cp = utf8::internal::mask16(*it++);
|
||||
if (utf8::internal::is_lead_surrogate(cp))
|
||||
return (cp << 10) + *it++ + utf8::internal::SURROGATE_OFFSET;
|
||||
return cp;
|
||||
}
|
||||
|
||||
template <typename octet_iterator>
|
||||
utfchar32_t prior(octet_iterator& it)
|
||||
uint32_t prior(octet_iterator& it)
|
||||
{
|
||||
while (utf8::internal::is_trail(*(--it))) ;
|
||||
octet_iterator temp = it;
|
||||
|
|
@ -154,7 +103,7 @@ namespace utf8
|
|||
}
|
||||
|
||||
template <typename octet_iterator, typename distance_type>
|
||||
void advance(octet_iterator& it, distance_type n)
|
||||
void advance (octet_iterator& it, distance_type n)
|
||||
{
|
||||
const distance_type zero(0);
|
||||
if (n < zero) {
|
||||
|
|
@ -170,7 +119,7 @@ namespace utf8
|
|||
|
||||
template <typename octet_iterator>
|
||||
typename std::iterator_traits<octet_iterator>::difference_type
|
||||
distance(octet_iterator first, octet_iterator last)
|
||||
distance (octet_iterator first, octet_iterator last)
|
||||
{
|
||||
typename std::iterator_traits<octet_iterator>::difference_type dist;
|
||||
for (dist = 0; first < last; ++dist)
|
||||
|
|
@ -179,15 +128,13 @@ namespace utf8
|
|||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end) {
|
||||
utfchar32_t cp = utf8::internal::mask16(*start++);
|
||||
// Take care of surrogate pairs first
|
||||
uint32_t cp = utf8::internal::mask16(*start++);
|
||||
// Take care of surrogate pairs first
|
||||
if (utf8::internal::is_lead_surrogate(cp)) {
|
||||
if (start == end)
|
||||
return result;
|
||||
utfchar32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
uint32_t trail_surrogate = utf8::internal::mask16(*start++);
|
||||
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
|
||||
}
|
||||
result = utf8::unchecked::append(cp, result);
|
||||
|
|
@ -196,22 +143,22 @@ namespace utf8
|
|||
}
|
||||
|
||||
template <typename u16bit_iterator, typename octet_iterator>
|
||||
u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result)
|
||||
{
|
||||
while (start < end) {
|
||||
utfchar32_t cp = utf8::unchecked::next(start);
|
||||
uint32_t cp = utf8::unchecked::next(start);
|
||||
if (cp > 0xffff) { //make a surrogate pair
|
||||
*result++ = static_cast<utfchar16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<utfchar16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET);
|
||||
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN);
|
||||
}
|
||||
else
|
||||
*result++ = static_cast<utfchar16_t>(cp);
|
||||
*result++ = static_cast<uint16_t>(cp);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result)
|
||||
{
|
||||
while (start != end)
|
||||
result = utf8::unchecked::append(*(start++), result);
|
||||
|
|
@ -220,7 +167,7 @@ namespace utf8
|
|||
}
|
||||
|
||||
template <typename octet_iterator, typename u32bit_iterator>
|
||||
u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result)
|
||||
{
|
||||
while (start < end)
|
||||
(*result++) = utf8::unchecked::next(start);
|
||||
|
|
@ -230,19 +177,14 @@ namespace utf8
|
|||
|
||||
// The iterator class
|
||||
template <typename octet_iterator>
|
||||
class iterator {
|
||||
class iterator : public std::iterator <std::bidirectional_iterator_tag, uint32_t> {
|
||||
octet_iterator it;
|
||||
public:
|
||||
typedef utfchar32_t value_type;
|
||||
typedef utfchar32_t* pointer;
|
||||
typedef utfchar32_t& reference;
|
||||
typedef std::ptrdiff_t difference_type;
|
||||
typedef std::bidirectional_iterator_tag iterator_category;
|
||||
iterator () {}
|
||||
explicit iterator (const octet_iterator& octet_it): it(octet_it) {}
|
||||
// the default "big three" are OK
|
||||
octet_iterator base () const { return it; }
|
||||
utfchar32_t operator * () const
|
||||
uint32_t operator * () const
|
||||
{
|
||||
octet_iterator temp = it;
|
||||
return utf8::unchecked::next(temp);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#include "utf8.h"
|
||||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
|
||||
#include <string>
|
||||
|
|
@ -17,13 +17,13 @@ int main(int argc, char** argv)
|
|||
test_file_path = argv[1];
|
||||
else {
|
||||
cout << "Wrong number of arguments" << endl;
|
||||
return 1;
|
||||
exit(0);
|
||||
}
|
||||
// Open the test file
|
||||
ifstream fs8(test_file_path.c_str());
|
||||
if (!fs8.is_open()) {
|
||||
cout << "Could not open " << test_file_path << endl;
|
||||
return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Read it line by line
|
||||
|
|
@ -38,22 +38,16 @@ int main(int argc, char** argv)
|
|||
bool expected_valid = (find(INVALID_LINES, INVALID_LINES_END, line_count) == INVALID_LINES_END);
|
||||
// Print out lines that contain unexpected invalid UTF-8
|
||||
if (!is_valid(line.begin(), line.end())) {
|
||||
if (expected_valid) {
|
||||
if (expected_valid)
|
||||
cout << "Unexpected invalid utf-8 at line " << line_count << '\n';
|
||||
return 1;
|
||||
}
|
||||
|
||||
// try fixing it:
|
||||
string fixed_line;
|
||||
replace_invalid(line.begin(), line.end(), back_inserter(fixed_line));
|
||||
if (!is_valid(fixed_line.begin(), fixed_line.end())) {
|
||||
if (!is_valid(fixed_line.begin(), fixed_line.end()))
|
||||
cout << "replace_invalid() resulted in an invalid utf-8 at line " << line_count << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
else if (!expected_valid) {
|
||||
else if (!expected_valid)
|
||||
cout << "Invalid utf-8 NOT detected at line " << line_count << '\n';
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
78
test_drivers/smoke_test/cpp11.cpp
Normal file
78
test_drivers/smoke_test/cpp11.cpp
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
int main()
|
||||
{
|
||||
string u;
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
//append
|
||||
|
||||
append(0x0448, u);
|
||||
assert (u[0] == char(0xd1) && u[1] == char(0x88) && u.length() == 2);
|
||||
|
||||
u.clear();
|
||||
append(0x65e5, u);
|
||||
assert (u[0] == char(0xe6) && u[1] == char(0x97) && u[2] == char(0xa5) && u.length() == 3);
|
||||
|
||||
u.clear();
|
||||
append(0x3044, u);
|
||||
assert (u[0] == char(0xe3) && u[1] == char(0x81) && u[2] == char(0x84) && u.length() == 3);
|
||||
|
||||
u.clear();
|
||||
append(0x10346, u);
|
||||
assert (u[0] == char(0xf0) && u[1] == char(0x90) && u[2] == char(0x8d) && u[3] == char(0x86) && u.length() == 4);
|
||||
|
||||
//utf16to8
|
||||
u16string utf16string = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
u.clear();
|
||||
u = utf16to8(utf16string);
|
||||
assert (u.size() == 10);
|
||||
|
||||
//utf8to16
|
||||
string utf8_with_surrogates = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
u16string utf16result = utf8to16(utf8_with_surrogates);
|
||||
assert (utf16result.length() == 4);
|
||||
assert (utf16result[2] == 0xd834);
|
||||
assert (utf16result[3] == 0xdd1e);
|
||||
|
||||
// utf32to8
|
||||
u32string utf32string = {0x448, 0x65E5, 0x10346};
|
||||
string utf8result = utf32to8(utf32string);
|
||||
assert (utf8result.size() == 9);
|
||||
|
||||
// utf8to32
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
u32string utf32result = utf8to32(twochars);
|
||||
assert (utf32result.size() == 2);
|
||||
|
||||
//find_invalid
|
||||
string utf_invalid = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
auto invalid = find_invalid(utf_invalid);
|
||||
assert (invalid == 5);
|
||||
|
||||
//is_valid
|
||||
bool bvalid = is_valid(utf_invalid);
|
||||
assert (bvalid == false);
|
||||
bvalid = is_valid(utf8_with_surrogates);
|
||||
assert (bvalid == true);
|
||||
|
||||
//replace_invalid
|
||||
string invalid_sequence = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
string replace_invalid_result = replace_invalid(invalid_sequence, '?');
|
||||
bvalid = is_valid(replace_invalid_result);
|
||||
assert (bvalid);
|
||||
const string fixed_invalid_sequence = "a????z";
|
||||
assert (fixed_invalid_sequence == replace_invalid_result);
|
||||
|
||||
//starts_with_bom
|
||||
string byte_order_mark = {char(0xef), char(0xbb), char(0xbf)};
|
||||
bool bbom = starts_with_bom(byte_order_mark);
|
||||
assert (bbom == true);
|
||||
string threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
bool no_bbom = starts_with_bom(threechars);
|
||||
assert (no_bbom == false);
|
||||
|
||||
|
||||
#endif // C++ 11 or later
|
||||
}
|
||||
273
test_drivers/smoke_test/test.cpp
Normal file
273
test_drivers/smoke_test/test.cpp
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
#include <cstring>
|
||||
#include <cassert>
|
||||
#include <vector>
|
||||
#include "../../source/utf8.h"
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
int main()
|
||||
{
|
||||
//append
|
||||
unsigned char u[5] = {0,0,0,0,0};
|
||||
|
||||
append(0x0448, u);
|
||||
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x65e5, u);
|
||||
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x3044, u);
|
||||
assert (u[0] == 0xe3 && u[1] == 0x81 && u[2] == 0x84 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x10346, u);
|
||||
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||
|
||||
|
||||
//next
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars;
|
||||
int cp = next(w, twochars + 6);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars + 3);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars;
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars + 4);
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 7);
|
||||
cp = next(w, threechars + 9);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 9);
|
||||
|
||||
//peek_next
|
||||
const char* const cw = twochars;
|
||||
cp = peek_next(cw, cw + 6);
|
||||
assert (cp == 0x65e5);
|
||||
assert (cw == twochars);
|
||||
|
||||
//prior
|
||||
w = twochars + 3;
|
||||
cp = prior (w, twochars);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars);
|
||||
|
||||
w = threechars + 9;
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 7);
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 4);
|
||||
cp = prior(w, threechars);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars);
|
||||
|
||||
// advance
|
||||
w = threechars;
|
||||
advance(w, 2, threechars + 9);
|
||||
assert(w == threechars + 7);
|
||||
advance(w, -2, threechars);
|
||||
assert(w == threechars);
|
||||
advance(w, 3, threechars + 9);
|
||||
assert(w == threechars + 9);
|
||||
advance(w, -2, threechars);
|
||||
assert(w == threechars + 4);
|
||||
advance(w, -1, threechars);
|
||||
assert(w == threechars);
|
||||
|
||||
// distance
|
||||
size_t dist = utf8::distance(twochars, twochars + 5);
|
||||
assert (dist == 2);
|
||||
|
||||
// utf32to8
|
||||
int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
|
||||
vector<char> utf8result;
|
||||
utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 9);
|
||||
// try it with the return value;
|
||||
char* utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 9);
|
||||
|
||||
//utf8to32
|
||||
vector<int> utf32result;
|
||||
utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
assert (utf32result.size() == 2);
|
||||
// try it with the return value;
|
||||
int* utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||
assert (utf32_end == &utf32result[0] + 2);
|
||||
|
||||
//utf16to8
|
||||
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
utf8result.clear();
|
||||
utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 10);
|
||||
// try it with the return value;
|
||||
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 10);
|
||||
|
||||
//utf8to16
|
||||
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
vector <unsigned short> utf16result;
|
||||
utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
assert (utf16result.size() == 4);
|
||||
assert (utf16result[2] == 0xd834);
|
||||
assert (utf16result[3] == 0xdd1e);
|
||||
// try it with the return value;
|
||||
unsigned short* utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||
assert (utf16_end == &utf16result[0] + 4);
|
||||
|
||||
//find_invalid
|
||||
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
|
||||
assert (invalid == utf_invalid + 5);
|
||||
|
||||
//is_valid
|
||||
bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
|
||||
assert (bvalid == false);
|
||||
bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
|
||||
assert (bvalid == true);
|
||||
|
||||
//starts_with_bom
|
||||
unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
|
||||
bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
|
||||
assert (bbom == true);
|
||||
bool no_bbom = starts_with_bom(threechars, threechars + sizeof(threechars));
|
||||
assert (no_bbom == false);
|
||||
|
||||
//replace_invalid
|
||||
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
vector<char> replace_invalid_result;
|
||||
replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), std::back_inserter(replace_invalid_result), '?');
|
||||
bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
|
||||
assert (bvalid);
|
||||
const char fixed_invalid_sequence[] = "a????z";
|
||||
assert (sizeof(fixed_invalid_sequence) == replace_invalid_result.size());
|
||||
assert (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
|
||||
|
||||
// iterator
|
||||
utf8::iterator<const char*> it(threechars, threechars, threechars + 9);
|
||||
utf8::iterator<const char*> it2 = it;
|
||||
assert (it2 == it);
|
||||
assert (*it == 0x10346);
|
||||
assert (*(++it) == 0x65e5);
|
||||
assert ((*it++) == 0x65e5);
|
||||
assert (*it == 0x0448);
|
||||
assert (it != it2);
|
||||
utf8::iterator<const char*> endit (threechars + 9, threechars, threechars + 9);
|
||||
assert (++it == endit);
|
||||
assert (*(--it) == 0x0448);
|
||||
assert ((*it--) == 0x0448);
|
||||
assert (*it == 0x65e5);
|
||||
assert (--it == utf8::iterator<const char*>(threechars, threechars, threechars + 9));
|
||||
assert (*it == 0x10346);
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
//// Unchecked variants
|
||||
//////////////////////////////////////////////////////////
|
||||
|
||||
//append
|
||||
memset(u, 0, 5);
|
||||
append(0x0448, u);
|
||||
assert (u[0] == 0xd1 && u[1] == 0x88 && u[2] == 0 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x65e5, u);
|
||||
assert (u[0] == 0xe6 && u[1] == 0x97 && u[2] == 0xa5 && u[3] == 0 && u[4] == 0);
|
||||
|
||||
append(0x10346, u);
|
||||
assert (u[0] == 0xf0 && u[1] == 0x90 && u[2] == 0x8d && u[3] == 0x86 && u[4] == 0);
|
||||
|
||||
//next
|
||||
w = twochars;
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == twochars + 3);
|
||||
|
||||
w = threechars;
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x10346);
|
||||
assert (w == threechars + 4);
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x65e5);
|
||||
assert (w == threechars + 7);
|
||||
cp = unchecked::next(w);
|
||||
assert (cp == 0x0448);
|
||||
assert (w == threechars + 9);
|
||||
|
||||
//peek_next
|
||||
cp = unchecked::peek_next(cw);
|
||||
assert (cp == 0x65e5);
|
||||
assert (cw == twochars);
|
||||
|
||||
// advance
|
||||
w = threechars;
|
||||
unchecked::advance(w, 2);
|
||||
assert(w == threechars + 7);
|
||||
unchecked::advance(w, -2);
|
||||
assert(w == threechars);
|
||||
unchecked::advance(w, 3);
|
||||
assert(w == threechars + 9);
|
||||
unchecked::advance(w, -2);
|
||||
assert(w == threechars + 4);
|
||||
unchecked::advance(w, -1);
|
||||
assert(w == threechars);
|
||||
|
||||
// distance
|
||||
dist = unchecked::distance(twochars, twochars + 5);
|
||||
assert (dist == 2);
|
||||
|
||||
// utf32to8
|
||||
utf8result.clear();
|
||||
unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 9);
|
||||
// try it with the return value;
|
||||
utf8_end = utf32to8(utf32string, utf32string + 3, &utf8result[0]);
|
||||
assert(utf8_end == &utf8result[0] + 9);
|
||||
|
||||
//utf8to32
|
||||
utf32result.clear();
|
||||
unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
assert (utf32result.size() == 2);
|
||||
// try it with the return value;
|
||||
utf32_end = utf8to32(twochars, twochars + 5, &utf32result[0]);
|
||||
assert (utf32_end == &utf32result[0] + 2);
|
||||
|
||||
//utf16to8
|
||||
utf8result.clear();
|
||||
unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
assert (utf8result.size() == 10);
|
||||
// try it with the return value;
|
||||
utf8_end = utf16to8 (utf16string, utf16string + 5, &utf8result[0]);
|
||||
assert (utf8_end == &utf8result[0] + 10);
|
||||
|
||||
//utf8to16
|
||||
utf16result.clear();
|
||||
unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
assert (utf16result.size() == 4);
|
||||
assert (utf16result[2] == 0xd834);
|
||||
assert (utf16result[3] == 0xdd1e);
|
||||
// try it with the return value;
|
||||
utf16_end = utf8to16 (utf8_with_surrogates, utf8_with_surrogates + 9, &utf16result[0]);
|
||||
assert (utf16_end == &utf16result[0] + 4);
|
||||
|
||||
// iterator
|
||||
utf8::unchecked::iterator<const char*> un_it(threechars);
|
||||
utf8::unchecked::iterator<const char*> un_it2 = un_it;
|
||||
assert (un_it2 == un_it);
|
||||
assert (*un_it == 0x10346);
|
||||
assert (*(++un_it) == 0x65e5);
|
||||
assert ((*un_it++) == 0x65e5);
|
||||
assert (un_it != un_it2);
|
||||
assert (*un_it == 0x0448);
|
||||
utf8::unchecked::iterator<const char*> un_endit (threechars + 9);
|
||||
assert (++un_it == un_endit);
|
||||
assert (*(--un_it) == 0x0448);
|
||||
assert ((*un_it--) == 0x0448);
|
||||
assert (*un_it == 0x65e5);
|
||||
assert (--un_it == utf8::unchecked::iterator<const char*>(threechars));
|
||||
assert (*un_it == 0x10346);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
cmake_minimum_required (VERSION 3.5)
|
||||
project(utfcpptests LANGUAGES CXX)
|
||||
enable_testing()
|
||||
|
||||
add_library(${PROJECT_NAME} INTERFACE)
|
||||
|
||||
include_directories("${PROJECT_SOURCE_DIR}/../source")
|
||||
|
||||
if (MSVC)
|
||||
# warning level 4
|
||||
add_compile_options(/W4)
|
||||
else()
|
||||
# additional warnings
|
||||
add_compile_options(-Wall -Wextra -Wpedantic -Wconversion -Wsign-conversion)
|
||||
endif()
|
||||
|
||||
add_executable(negative negative.cpp)
|
||||
add_executable(cpp11 test_cpp11.cpp)
|
||||
add_executable(cpp17 test_cpp17.cpp)
|
||||
add_executable(cpp20 test_cpp20.cpp)
|
||||
add_executable(apitests apitests.cpp)
|
||||
|
||||
add_executable(noexceptionstests noexceptionstests.cpp)
|
||||
|
||||
target_compile_options(noexceptionstests PUBLIC -fno-exceptions)
|
||||
|
||||
set_target_properties(negative apitests noexceptionstests
|
||||
PROPERTIES
|
||||
CXX_STANDARD 98
|
||||
CXX_STANDARD_REQUIRED YES
|
||||
CXX_EXTENSIONS NO)
|
||||
|
||||
set_target_properties(cpp11
|
||||
PROPERTIES
|
||||
CXX_STANDARD 11
|
||||
CXX_STANDARD_REQUIRED YES
|
||||
CXX_EXTENSIONS NO)
|
||||
|
||||
set_target_properties(cpp17
|
||||
PROPERTIES
|
||||
CXX_STANDARD 17
|
||||
CXX_STANDARD_REQUIRED YES
|
||||
CXX_EXTENSIONS NO)
|
||||
|
||||
set_target_properties(cpp20
|
||||
PROPERTIES
|
||||
CXX_STANDARD 20
|
||||
CXX_STANDARD_REQUIRED YES
|
||||
CXX_EXTENSIONS NO)
|
||||
|
||||
|
||||
add_test(negative_test negative ${PROJECT_SOURCE_DIR}/test_data/utf8_invalid.txt)
|
||||
add_test(cpp11_test cpp11)
|
||||
add_test(cpp17_test cpp17)
|
||||
add_test(cpp20_test cpp20)
|
||||
add_test(api_test apitests)
|
||||
add_test(noexceptions_test noexceptionstests)
|
||||
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
#include "../extern/ftest/ftest.h"
|
||||
|
||||
#include "test_checked_api.h"
|
||||
#include "test_checked_iterator.h"
|
||||
#include "test_unchecked_api.h"
|
||||
#include "test_unchecked_iterator.h"
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
FROM gcc:12.2
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y make g++ cmake git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
#include "../extern/ftest/ftest.h"
|
||||
|
||||
#include "test_unchecked_api.h"
|
||||
#include "test_unchecked_iterator.h"
|
||||
|
|
@ -1,233 +0,0 @@
|
|||
#ifndef UTF8_FOR_CPP_TEST_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_TEST_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "utf8.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
|
||||
TEST(CheckedAPITests, test_append)
|
||||
{
|
||||
unsigned char u[5] = {0,0,0,0,0};
|
||||
append(0x0448, u);
|
||||
EXPECT_EQ (u[0], 0xd1);
|
||||
EXPECT_EQ (u[1], 0x88);
|
||||
EXPECT_EQ (u[2], 0);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
append(0x65e5, u);
|
||||
EXPECT_EQ (u[0], 0xe6);
|
||||
EXPECT_EQ (u[1], 0x97);
|
||||
EXPECT_EQ (u[2], 0xa5);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
append(0x3044, u);
|
||||
EXPECT_EQ (u[0], 0xe3);
|
||||
EXPECT_EQ (u[1], 0x81);
|
||||
EXPECT_EQ (u[2], 0x84);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
append(0x10346, u);
|
||||
EXPECT_EQ (u[0], 0xf0);
|
||||
EXPECT_EQ (u[1], 0x90);
|
||||
EXPECT_EQ (u[2], 0x8d);
|
||||
EXPECT_EQ (u[3], 0x86);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
// Ensure no warnings with plain char
|
||||
char c[2] = {0,0};
|
||||
append('a', c);
|
||||
EXPECT_EQ (c[0], 'a');
|
||||
EXPECT_EQ (c[1], 0);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_append16)
|
||||
{
|
||||
utfchar16_t u[5] = {0,0};
|
||||
append16(0x0448, u);
|
||||
EXPECT_EQ (u[0], 0x0448);
|
||||
EXPECT_EQ (u[1], 0x0000);
|
||||
|
||||
append16(0x65e5, u);
|
||||
EXPECT_EQ (u[0], 0x65e5);
|
||||
EXPECT_EQ (u[1], 0x0000);
|
||||
|
||||
append16(0x10346, u);
|
||||
EXPECT_EQ (u[0], 0xd800);
|
||||
EXPECT_EQ (u[1], 0xdf46);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_next)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars;
|
||||
unsigned int cp = next(w, twochars + 6);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, twochars + 3);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars;
|
||||
|
||||
cp = next(w, threechars + 9);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, threechars + 4);
|
||||
|
||||
cp = next(w, threechars + 9);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, threechars + 7);
|
||||
|
||||
cp = next(w, threechars + 9);
|
||||
EXPECT_EQ (cp, 0x0448);
|
||||
EXPECT_EQ (w, threechars + 9);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_next16)
|
||||
{
|
||||
const utfchar16_t u[3] = {0x65e5, 0xd800, 0xdf46};
|
||||
const utfchar16_t* w = u;
|
||||
utf8::utfchar32_t cp = next16(w, w + 3);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, u + 1);
|
||||
|
||||
cp = next16(w, w + 2);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, u + 3);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_peek_next)
|
||||
{
|
||||
const char* const cw = "\xe6\x97\xa5\xd1\x88";
|
||||
unsigned int cp = peek_next(cw, cw + 6);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_prior)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars + 3;
|
||||
unsigned int cp = prior (w, twochars);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, twochars);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars + 9;
|
||||
cp = prior(w, threechars);
|
||||
EXPECT_EQ (cp, 0x0448);
|
||||
EXPECT_EQ (w, threechars + 7);
|
||||
cp = prior(w, threechars);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, threechars + 4);
|
||||
cp = prior(w, threechars);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, threechars);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_advance)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = threechars;
|
||||
advance(w, 2, threechars + 9);
|
||||
EXPECT_EQ(w, threechars + 7);
|
||||
advance(w, -2, threechars);
|
||||
EXPECT_EQ(w, threechars);
|
||||
advance(w, 3, threechars + 9);
|
||||
EXPECT_EQ(w, threechars + 9);
|
||||
advance(w, -2, threechars);
|
||||
EXPECT_EQ(w, threechars + 4);
|
||||
advance(w, -1, threechars);
|
||||
EXPECT_EQ(w, threechars);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_distance)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
size_t dist = static_cast<size_t>(utf8::distance(twochars, twochars + 5));
|
||||
EXPECT_EQ (dist, 2);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_utf32to8)
|
||||
{
|
||||
unsigned int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
|
||||
string utf8result;
|
||||
utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
EXPECT_EQ (utf8result.size(), 9);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_utf8to32)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
vector<unsigned int> utf32result;
|
||||
utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
EXPECT_EQ (utf32result.size(), 2);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_utf16to8)
|
||||
{
|
||||
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
string utf8result;
|
||||
utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
EXPECT_EQ (utf8result.size(), 10);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_utf8to16)
|
||||
{
|
||||
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
vector <unsigned short> utf16result;
|
||||
utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
EXPECT_EQ (utf16result.size(), 4);
|
||||
EXPECT_EQ (utf16result[2], 0xd834);
|
||||
EXPECT_EQ (utf16result[3], 0xdd1e);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_replace_invalid)
|
||||
{
|
||||
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
vector<char> replace_invalid_result;
|
||||
replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), std::back_inserter(replace_invalid_result), '?');
|
||||
bool bvalid = is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
|
||||
EXPECT_TRUE (bvalid);
|
||||
const char fixed_invalid_sequence[] = "a????z";
|
||||
EXPECT_EQ (sizeof(fixed_invalid_sequence), replace_invalid_result.size());
|
||||
EXPECT_TRUE (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_find_invalid)
|
||||
{
|
||||
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
const char* invalid = find_invalid(utf_invalid, utf_invalid + 6);
|
||||
EXPECT_EQ (invalid, utf_invalid + 5);
|
||||
invalid = find_invalid(utf_invalid);
|
||||
EXPECT_EQ (invalid, utf_invalid + 5);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_is_valid)
|
||||
{
|
||||
char utf_invalid[] = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
bool bvalid = is_valid(utf_invalid, utf_invalid + 6);
|
||||
EXPECT_FALSE (bvalid);
|
||||
bvalid = is_valid(utf_invalid);
|
||||
EXPECT_FALSE (bvalid);
|
||||
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
bvalid = is_valid(utf8_with_surrogates, utf8_with_surrogates + 9);
|
||||
EXPECT_TRUE (bvalid);
|
||||
bvalid = is_valid(utf8_with_surrogates);
|
||||
EXPECT_TRUE (bvalid);
|
||||
}
|
||||
|
||||
TEST(CheckedAPITests, test_starts_with_bom)
|
||||
{
|
||||
unsigned char byte_order_mark[] = {0xef, 0xbb, 0xbf};
|
||||
bool bbom = starts_with_bom(byte_order_mark, byte_order_mark + sizeof(byte_order_mark));
|
||||
EXPECT_TRUE (bbom);
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
bool no_bbom = starts_with_bom(threechars, threechars + sizeof(threechars));
|
||||
EXPECT_FALSE (no_bbom);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
#ifndef UTF8_FOR_CPP_TEST_CHECKED_ITERATOR_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_TEST_CHECKED_ITERATOR_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "utf8.h"
|
||||
|
||||
using namespace utf8;
|
||||
|
||||
|
||||
TEST(CheckedIteratrTests, test_increment)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
utf8::iterator<const char*> it(threechars, threechars, threechars + 9);
|
||||
utf8::iterator<const char*> it2 = it;
|
||||
EXPECT_EQ (it2, it);
|
||||
EXPECT_EQ (*it, 0x10346);
|
||||
EXPECT_EQ (*(++it), 0x65e5);
|
||||
EXPECT_EQ ((*it++), 0x65e5);
|
||||
EXPECT_EQ (*it, 0x0448);
|
||||
EXPECT_NE (it, it2);
|
||||
utf8::iterator<const char*> endit (threechars + 9, threechars, threechars + 9);
|
||||
EXPECT_EQ (++it, endit);
|
||||
}
|
||||
|
||||
TEST(CheckedIteratrTests, test_decrement)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
utf8::iterator<const char*> it(threechars+9, threechars, threechars + 9);
|
||||
EXPECT_EQ (*(--it), 0x0448);
|
||||
EXPECT_EQ ((*it--), 0x0448);
|
||||
EXPECT_EQ (*it, 0x65e5);
|
||||
EXPECT_EQ (--it, utf8::iterator<const char*>(threechars, threechars, threechars + 9));
|
||||
EXPECT_EQ (*it, 0x10346);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
#include "../extern/ftest/ftest.h"
|
||||
#include "utf8.h"
|
||||
#include <string>
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
#if __cplusplus >= 201103L // C++ 11 or later
|
||||
|
||||
TEST(CPP11APITests, test_append)
|
||||
{
|
||||
string u;
|
||||
append(0x0448, u);
|
||||
EXPECT_EQ (u[0], char(0xd1));
|
||||
EXPECT_EQ (u[1], char(0x88));
|
||||
EXPECT_EQ (u.length(), 2);
|
||||
|
||||
u.clear();
|
||||
append(0x65e5, u);
|
||||
EXPECT_EQ (u[0], char(0xe6));
|
||||
EXPECT_EQ (u[1], char(0x97));
|
||||
EXPECT_EQ (u[2], char(0xa5));
|
||||
EXPECT_EQ (u.length(), 3);
|
||||
|
||||
u.clear();
|
||||
append(0x3044, u);
|
||||
EXPECT_EQ (u[0], char(0xe3));
|
||||
EXPECT_EQ (u[1], char(0x81));
|
||||
EXPECT_EQ (u[2], char(0x84));
|
||||
EXPECT_EQ (u.length(), 3);
|
||||
|
||||
u.clear();
|
||||
append(0x10346, u);
|
||||
EXPECT_EQ (u[0], char(0xf0));
|
||||
EXPECT_EQ (u[1], char(0x90));
|
||||
EXPECT_EQ (u[2], char(0x8d));
|
||||
EXPECT_EQ (u[3], char(0x86));
|
||||
EXPECT_EQ (u.length(), 4);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_append16)
|
||||
{
|
||||
u16string u;
|
||||
append16(0x0448, u);
|
||||
EXPECT_EQ (u[0], char16_t(0x0448));
|
||||
EXPECT_EQ (u.length(), 1);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_utf16to8)
|
||||
{
|
||||
u16string utf16string = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
string u = utf16to8(utf16string);
|
||||
EXPECT_EQ (u.size(), 10);
|
||||
|
||||
u16string h16 = u"h!";
|
||||
string h8;
|
||||
utf8::unchecked::utf16to8(h16.begin(), h16.end(), std::back_inserter(h8));
|
||||
EXPECT_EQ (h8, "h!");
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_utf8to16)
|
||||
{
|
||||
string utf8_with_surrogates = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
u16string utf16result = utf8to16(utf8_with_surrogates);
|
||||
EXPECT_EQ (utf16result.size(), 4);
|
||||
EXPECT_EQ (utf16result[2], 0xd834);
|
||||
EXPECT_EQ (utf16result[3], 0xdd1e);
|
||||
// Just to make sure it compiles with string literals
|
||||
utf8to16(u8"simple");
|
||||
utf8to16("simple");
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_utf32to8)
|
||||
{
|
||||
u32string utf32string = {0x448, 0x65E5, 0x10346};
|
||||
string utf8result = utf32to8(utf32string);
|
||||
EXPECT_EQ (utf8result.size(), 9);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_utf8to32)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
u32string utf32result = utf8to32(twochars);
|
||||
EXPECT_EQ (utf32result.size(), 2);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_find_invalid)
|
||||
{
|
||||
string utf_invalid = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
auto invalid = find_invalid(utf_invalid);
|
||||
EXPECT_EQ (invalid, 5);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_is_valid)
|
||||
{
|
||||
string utf_invalid = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
bool bvalid = is_valid(utf_invalid);
|
||||
EXPECT_FALSE (bvalid);
|
||||
string utf8_with_surrogates = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
bvalid = is_valid(utf8_with_surrogates);
|
||||
EXPECT_TRUE (bvalid);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_replace_invalid)
|
||||
{
|
||||
string invalid_sequence = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
string replace_invalid_result = replace_invalid(invalid_sequence, '?');
|
||||
bool bvalid = is_valid(replace_invalid_result);
|
||||
EXPECT_TRUE (bvalid);
|
||||
const string fixed_invalid_sequence = "a????z";
|
||||
EXPECT_EQ(fixed_invalid_sequence, replace_invalid_result);
|
||||
}
|
||||
|
||||
TEST(CPP11APITests, test_starts_with_bom)
|
||||
{
|
||||
string byte_order_mark = {char(0xef), char(0xbb), char(0xbf)};
|
||||
bool bbom = starts_with_bom(byte_order_mark);
|
||||
EXPECT_TRUE (bbom);
|
||||
string threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
bool no_bbom = starts_with_bom(threechars);
|
||||
EXPECT_FALSE (no_bbom);
|
||||
}
|
||||
#endif // C++ 11 or later
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
#include "../extern/ftest/ftest.h"
|
||||
#include "utf8.h"
|
||||
#include <string>
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
#if __cplusplus >= 201703L // C++ 17 or later
|
||||
|
||||
|
||||
TEST(CPP17APITests, test_utf16to8)
|
||||
{
|
||||
u16string utf16string = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
u16string_view utf16stringview(utf16string);
|
||||
string u = utf16to8(utf16stringview);
|
||||
EXPECT_EQ (u.size(), 10);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_utf8to16)
|
||||
{
|
||||
string_view utf8_with_surrogates = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
u16string utf16result = utf8to16(utf8_with_surrogates);
|
||||
EXPECT_EQ (utf16result.size(), 4);
|
||||
EXPECT_EQ (utf16result[2], 0xd834);
|
||||
EXPECT_EQ (utf16result[3], 0xdd1e);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_utf32to8)
|
||||
{
|
||||
u32string utf32string = {0x448, 0x65E5, 0x10346};
|
||||
u32string_view utf32stringview(utf32string);
|
||||
string utf8result = utf32to8(utf32stringview);
|
||||
EXPECT_EQ (utf8result.size(), 9);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_utf8to32)
|
||||
{
|
||||
string_view twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
u32string utf32result = utf8to32(twochars);
|
||||
EXPECT_EQ (utf32result.size(), 2);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_find_invalid)
|
||||
{
|
||||
string_view utf_invalid = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
auto invalid = find_invalid(utf_invalid);
|
||||
EXPECT_EQ (invalid, 5);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_is_valid)
|
||||
{
|
||||
string_view utf_invalid = "\xe6\x97\xa5\xd1\x88\xfa";
|
||||
bool bvalid = is_valid(utf_invalid);
|
||||
EXPECT_FALSE (bvalid);
|
||||
string_view utf8_with_surrogates = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
bvalid = is_valid(utf8_with_surrogates);
|
||||
EXPECT_TRUE (bvalid);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_replace_invalid)
|
||||
{
|
||||
string_view invalid_sequence = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
string replace_invalid_result = replace_invalid(invalid_sequence, '?');
|
||||
bool bvalid = is_valid(replace_invalid_result);
|
||||
EXPECT_TRUE (bvalid);
|
||||
const string fixed_invalid_sequence = "a????z";
|
||||
EXPECT_EQ(fixed_invalid_sequence, replace_invalid_result);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, test_starts_with_bom)
|
||||
{
|
||||
string byte_order_mark = {char(0xef), char(0xbb), char(0xbf)};
|
||||
string_view byte_order_mark_view(byte_order_mark);
|
||||
bool bbom = starts_with_bom(byte_order_mark_view);
|
||||
EXPECT_TRUE (bbom);
|
||||
string_view threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
bool no_bbom = starts_with_bom(threechars);
|
||||
EXPECT_FALSE (no_bbom);
|
||||
}
|
||||
|
||||
TEST(CPP17APITests, string_class_and_literals)
|
||||
{
|
||||
const char* twochars = u8"ab";
|
||||
EXPECT_TRUE (is_valid(twochars));
|
||||
const string two_chars_string(twochars);
|
||||
EXPECT_TRUE (is_valid(two_chars_string));
|
||||
}
|
||||
|
||||
#endif // C++ 11 or later
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
#include "../extern/ftest/ftest.h"
|
||||
#define UTF_CPP_CPLUSPLUS 202002L
|
||||
#include "utf8.h"
|
||||
#include <string>
|
||||
using namespace utf8;
|
||||
using namespace std;
|
||||
|
||||
TEST(CPP20APITests, test_utf16tou8)
|
||||
{
|
||||
u16string utf16string = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
u16string_view utf16stringview{utf16string};
|
||||
u8string u = utf16tou8(utf16string);
|
||||
EXPECT_EQ (u.size(), 10);
|
||||
u = utf16tou8(utf16stringview);
|
||||
EXPECT_EQ (u.size(), 10);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, tes20t_utf8to16)
|
||||
{
|
||||
u8string utf8_with_surrogates{u8"\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e"};
|
||||
u16string utf16result = utf8to16(utf8_with_surrogates);
|
||||
EXPECT_EQ (utf16result.size(), 4);
|
||||
EXPECT_EQ (utf16result[2], 0xd834);
|
||||
EXPECT_EQ (utf16result[3], 0xdd1e);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_utf32tou8)
|
||||
{
|
||||
u32string utf32string = {0x448, 0x65E5, 0x10346};
|
||||
u32string_view utf32stringview{utf32string};
|
||||
u8string utf8result = utf32tou8(utf32stringview);
|
||||
EXPECT_EQ (utf8result.size(), 9);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_utf8to32)
|
||||
{
|
||||
u8string twochars = u8"\xe6\x97\xa5\xd1\x88";
|
||||
u32string utf32result = utf8to32(twochars);
|
||||
EXPECT_EQ (utf32result.size(), 2);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_find_invalid)
|
||||
{
|
||||
u8string utf_invalid = u8"\xe6\x97\xa5\xd1\x88\xfa";
|
||||
auto invalid = find_invalid(utf_invalid);
|
||||
EXPECT_EQ (invalid, 5);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_is_valid)
|
||||
{
|
||||
u8string utf_invalid = u8"\xe6\x97\xa5\xd1\x88\xfa";
|
||||
bool bvalid = is_valid(utf_invalid);
|
||||
EXPECT_FALSE (bvalid);
|
||||
u8string utf8_with_surrogates = u8"\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
bvalid = is_valid(utf8_with_surrogates);
|
||||
EXPECT_TRUE (bvalid);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_replace_invalid)
|
||||
{
|
||||
u8string invalid_sequence = u8"a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
u8string replace_invalid_result = replace_invalid(invalid_sequence, u8'?');
|
||||
bool bvalid = is_valid(replace_invalid_result);
|
||||
EXPECT_TRUE (bvalid);
|
||||
const u8string fixed_invalid_sequence = u8"a????z";
|
||||
EXPECT_EQ(fixed_invalid_sequence, replace_invalid_result);
|
||||
}
|
||||
|
||||
TEST(CPP20APITests, test_starts_with_bom)
|
||||
{
|
||||
u8string byte_order_mark = u8"\xef\xbb\xbf";
|
||||
bool bbom = starts_with_bom(byte_order_mark);
|
||||
EXPECT_TRUE (bbom);
|
||||
u8string threechars = u8"\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
bool no_bbom = starts_with_bom(threechars);
|
||||
EXPECT_FALSE (no_bbom);
|
||||
}
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
#ifndef UTF8_FOR_CPP_TEST_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_TEST_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "utf8/unchecked.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using namespace std;
|
||||
|
||||
TEST(UnCheckedAPITests, test_append)
|
||||
{
|
||||
unsigned char u[5] = {0,0,0,0,0};
|
||||
utf8::unchecked::append(0x0448, u);
|
||||
EXPECT_EQ (u[0], 0xd1);
|
||||
EXPECT_EQ (u[1], 0x88);
|
||||
EXPECT_EQ (u[2], 0);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
utf8::unchecked::append(0x65e5, u);
|
||||
EXPECT_EQ (u[0], 0xe6);
|
||||
EXPECT_EQ (u[1], 0x97);
|
||||
EXPECT_EQ (u[2], 0xa5);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
utf8::unchecked::append(0x3044, u);
|
||||
EXPECT_EQ (u[0], 0xe3);
|
||||
EXPECT_EQ (u[1], 0x81);
|
||||
EXPECT_EQ (u[2], 0x84);
|
||||
EXPECT_EQ (u[3], 0);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
|
||||
utf8::unchecked::append(0x10346, u);
|
||||
EXPECT_EQ (u[0], 0xf0);
|
||||
EXPECT_EQ (u[1], 0x90);
|
||||
EXPECT_EQ (u[2], 0x8d);
|
||||
EXPECT_EQ (u[3], 0x86);
|
||||
EXPECT_EQ (u[4], 0);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_append16)
|
||||
{
|
||||
unsigned short u[5] = {0,0};
|
||||
utf8::unchecked::append16(0x0448, u);
|
||||
EXPECT_EQ (u[0], 0x0448);
|
||||
EXPECT_EQ (u[1], 0x0000);
|
||||
|
||||
utf8::unchecked::append16(0x65e5, u);
|
||||
EXPECT_EQ (u[0], 0x65e5);
|
||||
EXPECT_EQ (u[1], 0x0000);
|
||||
|
||||
utf8::unchecked::append16(0x10346, u);
|
||||
EXPECT_EQ (u[0], 0xd800);
|
||||
EXPECT_EQ (u[1], 0xdf46);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_next)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars;
|
||||
unsigned int cp = utf8::unchecked::next(w);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, twochars + 3);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars;
|
||||
|
||||
cp = utf8::unchecked::next(w);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, threechars + 4);
|
||||
|
||||
cp = utf8::unchecked::next(w);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, threechars + 7);
|
||||
|
||||
cp = utf8::unchecked::next(w);
|
||||
EXPECT_EQ (cp, 0x0448);
|
||||
EXPECT_EQ (w, threechars + 9);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_next16)
|
||||
{
|
||||
const utf8::utfchar16_t u[3] = {0x65e5, 0xd800, 0xdf46};
|
||||
const utf8::utfchar16_t* w = u;
|
||||
utf8::utfchar32_t cp = utf8::unchecked::next16(w);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, u + 1);
|
||||
|
||||
cp = utf8::unchecked::next16(w);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, u + 3);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_peek_next)
|
||||
{
|
||||
const char* const cw = "\xe6\x97\xa5\xd1\x88";
|
||||
unsigned int cp = utf8::unchecked::peek_next(cw);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_prior)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = twochars + 3;
|
||||
unsigned int cp = utf8::unchecked::prior (w);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, twochars);
|
||||
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
w = threechars + 9;
|
||||
cp = utf8::unchecked::prior(w);
|
||||
EXPECT_EQ (cp, 0x0448);
|
||||
EXPECT_EQ (w, threechars + 7);
|
||||
cp = utf8::unchecked::prior(w);
|
||||
EXPECT_EQ (cp, 0x65e5);
|
||||
EXPECT_EQ (w, threechars + 4);
|
||||
cp = utf8::unchecked::prior(w);
|
||||
EXPECT_EQ (cp, 0x10346);
|
||||
EXPECT_EQ (w, threechars);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_advance)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
const char* w = threechars;
|
||||
utf8::unchecked::advance(w, 2);
|
||||
EXPECT_EQ(w, threechars + 7);
|
||||
utf8::unchecked::advance(w, -2);
|
||||
EXPECT_EQ(w, threechars);
|
||||
utf8::unchecked::advance(w, 3);
|
||||
EXPECT_EQ(w, threechars + 9);
|
||||
utf8::unchecked::advance(w, -2);
|
||||
EXPECT_EQ(w, threechars + 4);
|
||||
utf8::unchecked::advance(w, -1);
|
||||
EXPECT_EQ(w, threechars);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_distance)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
size_t dist = static_cast<size_t>(utf8::unchecked::distance(twochars, twochars + 5));
|
||||
EXPECT_EQ (dist, 2);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_utf32to8)
|
||||
{
|
||||
unsigned int utf32string[] = {0x448, 0x65E5, 0x10346, 0};
|
||||
string utf8result;
|
||||
utf8::unchecked::utf32to8(utf32string, utf32string + 3, back_inserter(utf8result));
|
||||
EXPECT_EQ (utf8result.size(), 9);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_utf8to32)
|
||||
{
|
||||
const char* twochars = "\xe6\x97\xa5\xd1\x88";
|
||||
vector<unsigned int> utf32result;
|
||||
utf8::unchecked::utf8to32(twochars, twochars + 5, back_inserter(utf32result));
|
||||
EXPECT_EQ (utf32result.size(), 2);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_utf16to8)
|
||||
{
|
||||
unsigned short utf16string[] = {0x41, 0x0448, 0x65e5, 0xd834, 0xdd1e};
|
||||
string utf8result;
|
||||
utf8::unchecked::utf16to8(utf16string, utf16string + 5, back_inserter(utf8result));
|
||||
EXPECT_EQ (utf8result.size(), 10);
|
||||
|
||||
utf8result.clear();
|
||||
unsigned short highsurrogateonly[] = {0xd800};
|
||||
utf8::unchecked::utf16to8(highsurrogateonly, highsurrogateonly + 1, back_inserter(utf8result));
|
||||
EXPECT_TRUE(true); // we didn't crash
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_utf8to16)
|
||||
{
|
||||
char utf8_with_surrogates[] = "\xe6\x97\xa5\xd1\x88\xf0\x9d\x84\x9e";
|
||||
vector <unsigned short> utf16result;
|
||||
utf8::unchecked::utf8to16(utf8_with_surrogates, utf8_with_surrogates + 9, back_inserter(utf16result));
|
||||
EXPECT_EQ (utf16result.size(), 4);
|
||||
EXPECT_EQ (utf16result[2], 0xd834);
|
||||
EXPECT_EQ (utf16result[3], 0xdd1e);
|
||||
}
|
||||
|
||||
TEST(UnCheckedAPITests, test_replace_invalid)
|
||||
{
|
||||
char invalid_sequence[] = "a\x80\xe0\xa0\xc0\xaf\xed\xa0\x80z";
|
||||
vector<char> replace_invalid_result;
|
||||
utf8::unchecked::replace_invalid (invalid_sequence, invalid_sequence + sizeof(invalid_sequence), std::back_inserter(replace_invalid_result), '?');
|
||||
bool bvalid = utf8::is_valid(replace_invalid_result.begin(), replace_invalid_result.end());
|
||||
EXPECT_TRUE (bvalid);
|
||||
const char fixed_invalid_sequence[] = "a????z";
|
||||
EXPECT_EQ (sizeof(fixed_invalid_sequence), replace_invalid_result.size());
|
||||
EXPECT_TRUE (std::equal(replace_invalid_result.begin(), replace_invalid_result.begin() + sizeof(fixed_invalid_sequence), fixed_invalid_sequence));
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
#ifndef UTF8_FOR_CPP_TEST_UNCHECKED_ITERATOR_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
#define UTF8_FOR_CPP_TEST_UNCHECKED_ITERATOR_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
|
||||
|
||||
#include "utf8/unchecked.h"
|
||||
|
||||
using namespace utf8::unchecked;
|
||||
|
||||
|
||||
TEST(UnCheckedIteratrTests, test_increment)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
utf8::unchecked::iterator<const char*> it(threechars);
|
||||
utf8::unchecked::iterator<const char*> it2 = it;
|
||||
EXPECT_EQ (it2, it);
|
||||
EXPECT_EQ (*it, 0x10346);
|
||||
EXPECT_EQ (*(++it), 0x65e5);
|
||||
EXPECT_EQ ((*it++), 0x65e5);
|
||||
EXPECT_EQ (*it, 0x0448);
|
||||
EXPECT_NE (it, it2);
|
||||
utf8::unchecked::iterator<const char*> endit (threechars + 9);
|
||||
EXPECT_EQ (++it, endit);
|
||||
}
|
||||
|
||||
TEST(UnCheckedIteratrTests, test_decrement)
|
||||
{
|
||||
const char* threechars = "\xf0\x90\x8d\x86\xe6\x97\xa5\xd1\x88";
|
||||
utf8::unchecked::iterator<const char*> it(threechars+9);
|
||||
EXPECT_EQ (*(--it), 0x0448);
|
||||
EXPECT_EQ ((*it--), 0x0448);
|
||||
EXPECT_EQ (*it, 0x65e5);
|
||||
EXPECT_EQ (--it, utf8::unchecked::iterator<const char*>(threechars));
|
||||
EXPECT_EQ (*it, 0x10346);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
@PACKAGE_INIT@
|
||||
|
||||
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
|
||||
check_required_components("@PROJECT_NAME@")
|
||||
|
||||
if(NOT TARGET utf8::cpp)
|
||||
add_library(utf8::cpp ALIAS utf8cpp::utf8cpp)
|
||||
endif()
|
||||
Loading…
Add table
Reference in a new issue