Updated utf8proc to the most recent version.
This commit is contained in:
Alexander Zolotarev
parent
71ef49263f
commit
7c7825b4b1
43 changed files with 18825 additions and 16171 deletions
33
3rdparty/utf8proc/CMakeLists.txt
vendored
Executable file
33
3rdparty/utf8proc/CMakeLists.txt
vendored
Executable file
|
|
@ -0,0 +1,33 @@
|
||||||
|
cmake_minimum_required (VERSION 2.8)
|
||||||
|
|
||||||
|
include (utils.cmake)
|
||||||
|
|
||||||
|
disallow_intree_builds()
|
||||||
|
|
||||||
|
project (utf8proc C)
|
||||||
|
|
||||||
|
# This is the ABI version number, which may differ from the
|
||||||
|
# API version number (defined in utf8proc.h).
|
||||||
|
# Be sure to also update these in Makefile and MANIFEST!
|
||||||
|
set(SO_MAJOR 2)
|
||||||
|
set(SO_MINOR 0)
|
||||||
|
set(SO_PATCH 2)
|
||||||
|
|
||||||
|
add_definitions (
|
||||||
|
-DUTF8PROC_EXPORTS
|
||||||
|
)
|
||||||
|
|
||||||
|
if (NOT MSVC)
|
||||||
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
add_library (utf8proc
|
||||||
|
utf8proc.c
|
||||||
|
utf8proc.h
|
||||||
|
)
|
||||||
|
|
||||||
|
set_target_properties (utf8proc PROPERTIES
|
||||||
|
POSITION_INDEPENDENT_CODE ON
|
||||||
|
VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
|
||||||
|
SOVERSION ${SO_MAJOR}
|
||||||
|
)
|
||||||
128
3rdparty/utf8proc/Changelog
vendored
128
3rdparty/utf8proc/Changelog
vendored
|
|
@ -1,128 +0,0 @@
|
||||||
Changelog
|
|
||||||
|
|
||||||
2006-06-02:
|
|
||||||
- initial release of version 0.1
|
|
||||||
|
|
||||||
2006-06-05:
|
|
||||||
- changed behaviour of PostgreSQL function to return NULL in case of
|
|
||||||
invalid input, rather than raising an exceptional condition
|
|
||||||
- improved efficiency of PostgreSQL function (no transformation to C string
|
|
||||||
is done)
|
|
||||||
|
|
||||||
2006-06-20:
|
|
||||||
- added -fpic compiler flag in Makefile
|
|
||||||
- fixed bug in the C code for the ruby library (usage of non-existent
|
|
||||||
function)
|
|
||||||
|
|
||||||
Release of version 0.2
|
|
||||||
|
|
||||||
|
|
||||||
2006-07-18:
|
|
||||||
- changed normalization from NFC to NFKC for postgresql unifold function
|
|
||||||
|
|
||||||
2006-08-04:
|
|
||||||
- added support to mark the beginning of a grapheme cluster with 0xFF
|
|
||||||
(option: CHARBOUND)
|
|
||||||
- added the ruby method String#chars, which is returning an array of UTF-8
|
|
||||||
encoded grapheme clusters
|
|
||||||
- added NLF2LF transformation in postgresql unifold function
|
|
||||||
- added the DECOMPOSE option, if you neither use COMPOSE or DECOMPOSE, no
|
|
||||||
normalization will be performed (different from previous versions)
|
|
||||||
- using integer constants rather than C-strings for character properties
|
|
||||||
- fixed (hopefully) a problem with the ruby library on Mac OS X, which
|
|
||||||
occured when compiler optimization was switched on
|
|
||||||
|
|
||||||
Release of version 0.3
|
|
||||||
|
|
||||||
|
|
||||||
2006-09-17:
|
|
||||||
- added the LUMP option, which lumps certain characters together
|
|
||||||
(see lump.txt) (also used for the PostgreSQL "unifold" function)
|
|
||||||
- added the STRIPMARK option, which strips marking characters
|
|
||||||
(or marks of composed characters)
|
|
||||||
- deprecated ruby method String#char_ary in favour of String#utf8chars
|
|
||||||
|
|
||||||
Release of version 1.0
|
|
||||||
|
|
||||||
|
|
||||||
2006-09-20:
|
|
||||||
- included a gem file for the ruby version of the library
|
|
||||||
|
|
||||||
Release of version 1.0.1
|
|
||||||
|
|
||||||
|
|
||||||
2006-09-21:
|
|
||||||
- included a check in Integer#utf8, which raises an exception, if the given
|
|
||||||
code-point is invalid because of being too high (this was missing yet)
|
|
||||||
|
|
||||||
2006-12-26:
|
|
||||||
- added support for PostgreSQL version 8.2
|
|
||||||
|
|
||||||
Release of version 1.0.2
|
|
||||||
|
|
||||||
|
|
||||||
2007-03-16:
|
|
||||||
- Fixed a bug in the ruby library, which caused an error, when splitting an
|
|
||||||
empty string at grapheme cluster boundaries (method String#utf8chars).
|
|
||||||
|
|
||||||
Release of version 1.0.3
|
|
||||||
|
|
||||||
|
|
||||||
2007-06-25:
|
|
||||||
- Added a new PostgreSQL function 'unistrip', which behaves like 'unifold',
|
|
||||||
but also removes all character marks (e.g. accents).
|
|
||||||
|
|
||||||
2007-07-22:
|
|
||||||
- Changed license from BSD to MIT style.
|
|
||||||
- Added a new function 'utf8proc_codepoint_valid' to the C library.
|
|
||||||
- Changed compiler flags in Makefile from -g -O0 to -O2
|
|
||||||
- The ruby script, which was used to build the utf8proc_data.c file, is now
|
|
||||||
included in the distribution.
|
|
||||||
|
|
||||||
Release of version 1.1.1
|
|
||||||
|
|
||||||
|
|
||||||
2007-07-25:
|
|
||||||
- Fixed a serious bug in the data file generator, which caused characters
|
|
||||||
being treated incorrectly, when stripping default ignorable characters or
|
|
||||||
calculating grapheme cluster boundaries.
|
|
||||||
|
|
||||||
Release of version 1.1.2
|
|
||||||
|
|
||||||
|
|
||||||
2008-10-04:
|
|
||||||
- Added a function utf8proc_version returning a string containing the version
|
|
||||||
number of the library.
|
|
||||||
- Included a target libutf8proc.dylib for MacOSX.
|
|
||||||
|
|
||||||
2009-05-01:
|
|
||||||
- PostgreSQL 8.3 compatibility (use of SET_VARSIZE macro)
|
|
||||||
|
|
||||||
Release of version 1.1.3
|
|
||||||
|
|
||||||
|
|
||||||
2009-06-14:
|
|
||||||
- replaced C++ style comments for compatibility reasons
|
|
||||||
- added typecasts to suppress compiler warnings
|
|
||||||
- removed redundant source files for ruby-gemfile generation
|
|
||||||
|
|
||||||
2009-08-19:
|
|
||||||
- Changed copyright notice for Public Software Group e. V.
|
|
||||||
- Minor changes in the README file
|
|
||||||
- Release of version 1.1.4
|
|
||||||
|
|
||||||
2009-08-20:
|
|
||||||
- Use RSTRING_PTR() and RSTRING_LEN() instead of RSTRING()->ptr and
|
|
||||||
RSTRING()->len for ruby1.9 compatibility (and #define them, if not
|
|
||||||
existent)
|
|
||||||
|
|
||||||
2009-10-02:
|
|
||||||
- Patches for compatibility with Microsoft Visual Studio
|
|
||||||
|
|
||||||
2009-10-08:
|
|
||||||
- Fixes to make utf8proc usable in C++ programs
|
|
||||||
|
|
||||||
2009-10-16:
|
|
||||||
- Release of version 1.1.5
|
|
||||||
|
|
||||||
2009-10-08:
|
|
||||||
1528
3rdparty/utf8proc/Doxyfile
vendored
Executable file
1528
3rdparty/utf8proc/Doxyfile
vendored
Executable file
File diff suppressed because it is too large
Load diff
41
3rdparty/utf8proc/LICENSE → 3rdparty/utf8proc/LICENSE.md
vendored
Normal file → Executable file
41
3rdparty/utf8proc/LICENSE → 3rdparty/utf8proc/LICENSE.md
vendored
Normal file → Executable file
|
|
@ -1,5 +1,13 @@
|
||||||
|
## utf8proc license ##
|
||||||
|
|
||||||
Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
**utf8proc** is a software package originally developed
|
||||||
|
by Jan Behrens and the rest of the Public Software Group, who
|
||||||
|
deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers. Like the original utf8proc,
|
||||||
|
whose copyright and license statements are reproduced below, all new
|
||||||
|
work on the utf8proc library is licensed under the [MIT "expat"
|
||||||
|
license](http://opensource.org/licenses/MIT):
|
||||||
|
|
||||||
|
*Copyright © 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a
|
Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
copy of this software and associated documentation files (the "Software"),
|
copy of this software and associated documentation files (the "Software"),
|
||||||
|
|
@ -19,14 +27,37 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
DEALINGS IN THE SOFTWARE.
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
## Original utf8proc license ##
|
||||||
|
|
||||||
|
*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
copy of this software and associated documentation files (the "Software"),
|
||||||
|
to deal in the Software without restriction, including without limitation
|
||||||
|
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
Software is furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
## Unicode data license ##
|
||||||
|
|
||||||
This software distribution contains derived data from a modified version of
|
This software distribution contains derived data from a modified version of
|
||||||
the Unicode data files. The following license applies to that data:
|
the Unicode data files. The following license applies to that data:
|
||||||
|
|
||||||
COPYRIGHT AND PERMISSION NOTICE
|
**COPYRIGHT AND PERMISSION NOTICE**
|
||||||
|
|
||||||
Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
||||||
under the Terms of Use in http://www.unicode.org/copyright.html.
|
under the Terms of Use in http://www.unicode.org/copyright.html.*
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a
|
Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
copy of the Unicode data files and any associated documentation (the "Data
|
copy of the Unicode data files and any associated documentation (the "Data
|
||||||
|
|
@ -57,8 +88,6 @@ not be used in advertising or otherwise to promote the sale, use or other
|
||||||
dealings in these Data Files or Software without prior written
|
dealings in these Data Files or Software without prior written
|
||||||
authorization of the copyright holder.
|
authorization of the copyright holder.
|
||||||
|
|
||||||
|
|
||||||
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
|
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
|
||||||
registered in some jurisdictions. All other trademarks and registered
|
registered in some jurisdictions. All other trademarks and registered
|
||||||
trademarks mentioned herein are the property of their respective owners.
|
trademarks mentioned herein are the property of their respective owners.
|
||||||
|
|
||||||
7
3rdparty/utf8proc/MANIFEST
vendored
Executable file
7
3rdparty/utf8proc/MANIFEST
vendored
Executable file
|
|
@ -0,0 +1,7 @@
|
||||||
|
include/
|
||||||
|
include/utf8proc.h
|
||||||
|
lib/
|
||||||
|
lib/libutf8proc.a
|
||||||
|
lib/libutf8proc.so -> libutf8proc.so.2.0.2
|
||||||
|
lib/libutf8proc.so.2 -> libutf8proc.so.2.0.2
|
||||||
|
lib/libutf8proc.so.2.0.2
|
||||||
156
3rdparty/utf8proc/Makefile
vendored
Normal file → Executable file
156
3rdparty/utf8proc/Makefile
vendored
Normal file → Executable file
|
|
@ -1,68 +1,146 @@
|
||||||
# libutf8proc Makefile
|
# libutf8proc Makefile
|
||||||
|
|
||||||
|
# programs
|
||||||
|
MAKE=make
|
||||||
|
AR?=ar
|
||||||
|
CC?=gcc
|
||||||
|
INSTALL=install
|
||||||
|
FIND=find
|
||||||
|
|
||||||
# settings
|
# compiler settings
|
||||||
|
CFLAGS ?= -O2
|
||||||
|
PICFLAG = -fPIC
|
||||||
|
C99FLAG = -std=c99
|
||||||
|
WCFLAGS = -Wall -pedantic
|
||||||
|
UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS
|
||||||
|
|
||||||
cflags = -O2 -std=c99 -pedantic -Wall -fpic $(CFLAGS)
|
# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
|
||||||
cc = $(CC) $(cflags)
|
# from the utf8proc version number because it indicates ABI compatibility,
|
||||||
|
# not API compatibility: MAJOR should be incremented whenever *binary*
|
||||||
|
# compatibility is broken, even if the API is backward-compatible.
|
||||||
|
# The API version number is defined in utf8proc.h.
|
||||||
|
# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
|
||||||
|
MAJOR=2
|
||||||
|
MINOR=0
|
||||||
|
PATCH=2
|
||||||
|
|
||||||
|
OS := $(shell uname)
|
||||||
|
ifeq ($(OS),Darwin) # MacOS X
|
||||||
|
SHLIB_EXT = dylib
|
||||||
|
SHLIB_VERS_EXT = $(MAJOR).dylib
|
||||||
|
else # GNU/Linux, at least (Windows should probably use cmake)
|
||||||
|
SHLIB_EXT = so
|
||||||
|
SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
|
||||||
|
endif
|
||||||
|
|
||||||
|
# installation directories (for 'make install')
|
||||||
|
prefix=/usr/local
|
||||||
|
libdir=$(prefix)/lib
|
||||||
|
includedir=$(prefix)/include
|
||||||
|
|
||||||
# meta targets
|
# meta targets
|
||||||
|
|
||||||
c-library: libutf8proc.a libutf8proc.so
|
.PHONY: all clean data update manifest install
|
||||||
|
|
||||||
ruby-library: ruby/utf8proc_native.so
|
all: libutf8proc.a libutf8proc.$(SHLIB_EXT)
|
||||||
|
|
||||||
pgsql-library: pgsql/utf8proc_pgsql.so
|
clean:
|
||||||
|
rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
|
||||||
|
ifneq ($(OS),Darwin)
|
||||||
|
rm -f libutf8proc.so.$(MAJOR)
|
||||||
|
endif
|
||||||
|
rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case
|
||||||
|
rm -rf MANIFEST.new tmp
|
||||||
|
$(MAKE) -C bench clean
|
||||||
|
$(MAKE) -C data clean
|
||||||
|
|
||||||
all: c-library ruby-library ruby-gem pgsql-library
|
data: data/utf8proc_data.c.new
|
||||||
|
|
||||||
clean::
|
update: data/utf8proc_data.c.new
|
||||||
rm -f utf8proc.o libutf8proc.a libutf8proc.so
|
cp -f data/utf8proc_data.c.new utf8proc_data.c
|
||||||
cd ruby/ && test -e Makefile && (make clean && rm -f Makefile) || true
|
|
||||||
rm -Rf ruby/gem/lib ruby/gem/ext
|
manifest: MANIFEST.new
|
||||||
rm -f ruby/gem/utf8proc-*.gem
|
|
||||||
cd pgsql/ && make clean
|
|
||||||
|
|
||||||
# real targets
|
# real targets
|
||||||
|
|
||||||
|
data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
|
||||||
|
$(MAKE) -C data utf8proc_data.c.new
|
||||||
|
|
||||||
utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
|
utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
|
||||||
$(cc) -c -o utf8proc.o utf8proc.c
|
$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c
|
||||||
|
|
||||||
libutf8proc.a: utf8proc.o
|
libutf8proc.a: utf8proc.o
|
||||||
rm -f libutf8proc.a
|
rm -f libutf8proc.a
|
||||||
ar rs libutf8proc.a utf8proc.o
|
$(AR) rs libutf8proc.a utf8proc.o
|
||||||
|
|
||||||
libutf8proc.so: utf8proc.o
|
libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
|
||||||
$(cc) -shared -o libutf8proc.so utf8proc.o
|
$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
|
||||||
chmod a-x libutf8proc.so
|
chmod a-x $@
|
||||||
|
|
||||||
libutf8proc.dylib: utf8proc.o
|
libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
|
||||||
$(cc) -dynamiclib -o $@ $^ -install_name $(libdir)/$@
|
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
|
||||||
|
ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)
|
||||||
|
|
||||||
ruby/Makefile: ruby/extconf.rb
|
libutf8proc.$(MAJOR).dylib: utf8proc.o
|
||||||
cd ruby && ruby extconf.rb
|
$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)
|
||||||
|
|
||||||
ruby/utf8proc_native.so: utf8proc.h utf8proc.c utf8proc_data.c \
|
libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
|
||||||
ruby/utf8proc_native.c ruby/Makefile
|
ln -f -s libutf8proc.$(MAJOR).dylib $@
|
||||||
cd ruby && make
|
|
||||||
|
|
||||||
ruby/gem/lib/utf8proc.rb: ruby/utf8proc.rb
|
install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
|
||||||
test -e ruby/gem/lib || mkdir ruby/gem/lib
|
mkdir -m 755 -p $(DESTDIR)$(includedir)
|
||||||
cp ruby/utf8proc.rb ruby/gem/lib/
|
$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
|
||||||
|
mkdir -m 755 -p $(DESTDIR)$(libdir)
|
||||||
|
$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
|
||||||
|
$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
|
||||||
|
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
|
||||||
|
ifneq ($(OS),Darwin)
|
||||||
|
ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
|
||||||
|
endif
|
||||||
|
|
||||||
ruby/gem/ext/extconf.rb: ruby/extconf.rb
|
MANIFEST.new:
|
||||||
test -e ruby/gem/ext || mkdir ruby/gem/ext
|
rm -rf tmp
|
||||||
cp ruby/extconf.rb ruby/gem/ext/
|
$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
|
||||||
|
$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
|
||||||
|
rm -rf tmp
|
||||||
|
|
||||||
ruby/gem/ext/utf8proc_native.c: utf8proc.h utf8proc_data.c utf8proc.c ruby/utf8proc_native.c
|
# Test programs
|
||||||
test -e ruby/gem/ext || mkdir ruby/gem/ext
|
|
||||||
cat utf8proc.h utf8proc_data.c utf8proc.c ruby/utf8proc_native.c | grep -v '#include "utf8proc.h"' | grep -v '#include "utf8proc_data.c"' | grep -v '#include "../utf8proc.c"' > ruby/gem/ext/utf8proc_native.c
|
|
||||||
|
|
||||||
ruby-gem:: ruby/gem/lib/utf8proc.rb ruby/gem/ext/extconf.rb ruby/gem/ext/utf8proc_native.c
|
data/NormalizationTest.txt:
|
||||||
cd ruby/gem && gem build utf8proc.gemspec
|
$(MAKE) -C data NormalizationTest.txt
|
||||||
|
|
||||||
pgsql/utf8proc_pgsql.so: utf8proc.h utf8proc.c utf8proc_data.c \
|
data/GraphemeBreakTest.txt:
|
||||||
pgsql/utf8proc_pgsql.c
|
$(MAKE) -C data GraphemeBreakTest.txt
|
||||||
cd pgsql && make
|
|
||||||
|
|
||||||
|
test/tests.o: test/tests.c test/tests.h utf8proc.h
|
||||||
|
$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
|
||||||
|
|
||||||
|
test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
|
||||||
|
$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
|
||||||
|
|
||||||
|
check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
|
||||||
|
$(MAKE) -C bench
|
||||||
|
test/normtest data/NormalizationTest.txt
|
||||||
|
test/graphemetest data/GraphemeBreakTest.txt
|
||||||
|
test/charwidth
|
||||||
|
test/valid
|
||||||
|
test/iterate
|
||||||
|
test/case
|
||||||
|
|
|
||||||
281
3rdparty/utf8proc/NEWS.md
vendored
Executable file
281
3rdparty/utf8proc/NEWS.md
vendored
Executable file
|
|
@ -0,0 +1,281 @@
|
||||||
|
# utf8proc release history #
|
||||||
|
|
||||||
|
## Version 2.0.2 ##
|
||||||
|
|
||||||
|
2016-07-27:
|
||||||
|
|
||||||
|
- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
|
||||||
|
since MSVC does not understand this flag and it is occasionally useful to
|
||||||
|
build using MSVC through the `Makefile` ([#79]).
|
||||||
|
|
||||||
|
- Use a different variable name for a nested loop in `bench/bench.c`, and
|
||||||
|
declare it in a C89 way rather than inside the `for` to avoid "error:
|
||||||
|
'for' loop initial declarations are only allowed in C99 mode" ([#80]).
|
||||||
|
|
||||||
|
## Version 2.0.1 ##
|
||||||
|
|
||||||
|
2016-07-13:
|
||||||
|
|
||||||
|
- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
|
||||||
|
|
||||||
|
- Tests now use versioned Unicode files, so they will no longer
|
||||||
|
break when a new version of Unicode is released ([#78]).
|
||||||
|
|
||||||
|
## Version 2.0 ##
|
||||||
|
|
||||||
|
2016-07-13:
|
||||||
|
|
||||||
|
- Updated for Unicode 9.0 ([#70]).
|
||||||
|
|
||||||
|
- New `utf8proc_grapheme_break_stateful` to handle the complicated
|
||||||
|
grapheme-breaking rules in Unicode 9. The old `utf8proc_grapheme_break`
|
||||||
|
is still provided, but may incorrectly identify grapheme breaks
|
||||||
|
in some Unicode-9 sequences.
|
||||||
|
|
||||||
|
- Smaller Unicode tables ([#62], [#68]). This required changes
|
||||||
|
in the `utf8proc_property_t` structure, which breaks backward
|
||||||
|
compatibility if you access this `struct` directly. The
|
||||||
|
functions in the API remain backward-compatible, however.
|
||||||
|
|
||||||
|
- Buffer overrun fix ([#66]).
|
||||||
|
|
||||||
|
## Version 1.3.1 ##
|
||||||
|
|
||||||
|
2015-11-02:
|
||||||
|
|
||||||
|
- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
|
||||||
|
|
||||||
|
- Install relative symbolic links for shared libraries ([#58]).
|
||||||
|
|
||||||
|
- Enable and fix compiler warnings ([#55], [#58]).
|
||||||
|
|
||||||
|
- Add missing files to `make clean` ([#58]).
|
||||||
|
|
||||||
|
## Version 1.3 ##
|
||||||
|
|
||||||
|
2015-07-06:
|
||||||
|
|
||||||
|
- Updated for Unicode 8.0 ([#45]).
|
||||||
|
|
||||||
|
- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
|
||||||
|
replacements for `towlower` and `towupper` in the C library ([#40]).
|
||||||
|
|
||||||
|
- Don't treat Unicode "non-characters" as invalid, and improved
|
||||||
|
validity checking in general ([#35]).
|
||||||
|
|
||||||
|
- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
|
||||||
|
to avoid collisions with other libraries ([#32]).
|
||||||
|
|
||||||
|
- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
|
||||||
|
|
||||||
|
- Fix build breakage in the benchmark routines.
|
||||||
|
|
||||||
|
- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
|
||||||
|
compilation flags can be selectively overridden, and in particular
|
||||||
|
so that `CFLAGS` can be changed without accidentally eliminating
|
||||||
|
necessary flags like `-fPIC` and `-std=c99` ([#43]).
|
||||||
|
|
||||||
|
- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
|
||||||
|
the Unicode 8 character categories ([#47]).
|
||||||
|
|
||||||
|
## Version 1.2 ##
|
||||||
|
|
||||||
|
2015-03-28:
|
||||||
|
|
||||||
|
- Updated for Unicode 7.0 ([#6]).
|
||||||
|
|
||||||
|
- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
|
||||||
|
there is a grapheme break between `c1` and `c2` ([#20]).
|
||||||
|
|
||||||
|
- New function `utf8proc_charwidth(c)` that returns the number of
|
||||||
|
column-positions that should be required for `c`; essentially a
|
||||||
|
portable replacment for `wcwidth(c)` ([#27]).
|
||||||
|
|
||||||
|
- New function `utf8proc_category(c)` that returns the Unicode
|
||||||
|
category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
|
||||||
|
Also, a function `utf8proc_category_string(c)` that returns the Unicode
|
||||||
|
category of `c` as a two-character string.
|
||||||
|
|
||||||
|
- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
|
||||||
|
easier compilation on Windows ([#28]).
|
||||||
|
|
||||||
|
- Various `Makefile` improvements: a `make check` target to perform
|
||||||
|
tests ([#13]), `make install`, a rule to automate updating the Unicode
|
||||||
|
tables, etcetera.
|
||||||
|
|
||||||
|
- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
|
||||||
|
|
||||||
|
- C++/MSVC compatibility ([#17]).
|
||||||
|
|
||||||
|
- Most `#defined` constants are now `enums` ([#29]).
|
||||||
|
|
||||||
|
- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
|
||||||
|
`UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
|
||||||
|
detection of the API version.
|
||||||
|
|
||||||
|
- Doxygen-formatted documentation ([#29]).
|
||||||
|
|
||||||
|
- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
|
||||||
|
|
||||||
|
## Version 1.1.6 ##
|
||||||
|
|
||||||
|
2013-11-27:
|
||||||
|
|
||||||
|
- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
|
||||||
|
|
||||||
|
## Version 1.1.5 ##
|
||||||
|
|
||||||
|
2009-08-20:
|
||||||
|
|
||||||
|
- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
|
||||||
|
`RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
|
||||||
|
existent)
|
||||||
|
|
||||||
|
2009-10-02:
|
||||||
|
|
||||||
|
- Patches for compatibility with Microsoft Visual Studio
|
||||||
|
|
||||||
|
2009-10-08:
|
||||||
|
|
||||||
|
- Fixes to make utf8proc usable in C++ programs
|
||||||
|
|
||||||
|
2009-10-16:
|
||||||
|
|
||||||
|
## Version 1.1.4 ##
|
||||||
|
|
||||||
|
2009-06-14:
|
||||||
|
|
||||||
|
- replaced C++ style comments for compatibility reasons
|
||||||
|
- added typecasts to suppress compiler warnings
|
||||||
|
- removed redundant source files for ruby-gemfile generation
|
||||||
|
|
||||||
|
2009-08-19:
|
||||||
|
|
||||||
|
- Changed copyright notice for Public Software Group e. V.
|
||||||
|
- Minor changes in the `README` file
|
||||||
|
|
||||||
|
## Version 1.1.3 ##
|
||||||
|
|
||||||
|
2008-10-04:
|
||||||
|
|
||||||
|
- Added a function `utf8proc_version` returning a string containing the version
|
||||||
|
number of the library.
|
||||||
|
- Included a target `libutf8proc.dylib` for MacOSX.
|
||||||
|
|
||||||
|
2009-05-01:
|
||||||
|
- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
|
||||||
|
|
||||||
|
## Version 1.1.2 ##
|
||||||
|
|
||||||
|
2007-07-25:
|
||||||
|
|
||||||
|
- Fixed a serious bug in the data file generator, which caused characters
|
||||||
|
being treated incorrectly, when stripping default ignorable characters or
|
||||||
|
calculating grapheme cluster boundaries.
|
||||||
|
|
||||||
|
## Version 1.1.1 ##
|
||||||
|
|
||||||
|
2007-06-25:
|
||||||
|
|
||||||
|
- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
|
||||||
|
but also removes all character marks (e.g. accents).
|
||||||
|
|
||||||
|
2007-07-22:
|
||||||
|
|
||||||
|
- Changed license from BSD to MIT style.
|
||||||
|
- Added a new function `utf8proc_codepoint_valid` to the C library.
|
||||||
|
- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
|
||||||
|
- The ruby script, which was used to build the `utf8proc_data.c` file, is now
|
||||||
|
included in the distribution.
|
||||||
|
|
||||||
|
## Version 1.0.3 ##
|
||||||
|
|
||||||
|
2007-03-16:
|
||||||
|
|
||||||
|
- Fixed a bug in the ruby library, which caused an error, when splitting an
|
||||||
|
empty string at grapheme cluster boundaries (method `String#utf8chars`).
|
||||||
|
|
||||||
|
## Version 1.0.2 ##
|
||||||
|
|
||||||
|
2006-09-21:
|
||||||
|
|
||||||
|
- included a check in `Integer#utf8`, which raises an exception, if the given
|
||||||
|
code-point is invalid because of being too high (this was missing yet)
|
||||||
|
|
||||||
|
2006-12-26:
|
||||||
|
|
||||||
|
- added support for PostgreSQL version 8.2
|
||||||
|
|
||||||
|
## Version 1.0.1 ##
|
||||||
|
|
||||||
|
2006-09-20:
|
||||||
|
|
||||||
|
- included a gem file for the ruby version of the library
|
||||||
|
|
||||||
|
Release of version 1.0.1
|
||||||
|
|
||||||
|
## Version 1.0 ##
|
||||||
|
|
||||||
|
2006-09-17:
|
||||||
|
|
||||||
|
- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
|
||||||
|
- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
|
||||||
|
- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
|
||||||
|
|
||||||
|
## Version 0.3 ##
|
||||||
|
|
||||||
|
2006-07-18:
|
||||||
|
|
||||||
|
- changed normalization from NFC to NFKC for postgresql unifold function
|
||||||
|
|
||||||
|
2006-08-04:
|
||||||
|
|
||||||
|
- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
|
||||||
|
- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
|
||||||
|
- added `NLF2LF` transformation in postgresql `unifold` function
|
||||||
|
- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
|
||||||
|
- using integer constants rather than C-strings for character properties
|
||||||
|
- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
|
||||||
|
|
||||||
|
## Version 0.2 ##
|
||||||
|
|
||||||
|
2006-06-05:
|
||||||
|
|
||||||
|
- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
|
||||||
|
- improved efficiency of PostgreSQL function (no transformation to C string is done)
|
||||||
|
|
||||||
|
2006-06-20:
|
||||||
|
|
||||||
|
- added -fpic compiler flag in Makefile
|
||||||
|
- fixed bug in the C code for the ruby library (usage of non-existent function)
|
||||||
|
|
||||||
|
## Version 0.1 ##
|
||||||
|
|
||||||
|
2006-06-02: initial release of version 0.1
|
||||||
|
|
||||||
|
[#6]: https://github.com/JuliaLang/utf8proc/issues/6
|
||||||
|
[#13]: https://github.com/JuliaLang/utf8proc/issues/13
|
||||||
|
[#17]: https://github.com/JuliaLang/utf8proc/issues/17
|
||||||
|
[#20]: https://github.com/JuliaLang/utf8proc/issues/20
|
||||||
|
[#22]: https://github.com/JuliaLang/utf8proc/issues/22
|
||||||
|
[#24]: https://github.com/JuliaLang/utf8proc/issues/24
|
||||||
|
[#27]: https://github.com/JuliaLang/utf8proc/issues/27
|
||||||
|
[#28]: https://github.com/JuliaLang/utf8proc/issues/28
|
||||||
|
[#29]: https://github.com/JuliaLang/utf8proc/issues/29
|
||||||
|
[#32]: https://github.com/JuliaLang/utf8proc/issues/32
|
||||||
|
[#35]: https://github.com/JuliaLang/utf8proc/issues/35
|
||||||
|
[#40]: https://github.com/JuliaLang/utf8proc/issues/40
|
||||||
|
[#43]: https://github.com/JuliaLang/utf8proc/issues/43
|
||||||
|
[#45]: https://github.com/JuliaLang/utf8proc/issues/45
|
||||||
|
[#47]: https://github.com/JuliaLang/utf8proc/issues/47
|
||||||
|
[#51]: https://github.com/JuliaLang/utf8proc/issues/51
|
||||||
|
[#55]: https://github.com/JuliaLang/utf8proc/issues/55
|
||||||
|
[#58]: https://github.com/JuliaLang/utf8proc/issues/58
|
||||||
|
[#62]: https://github.com/JuliaLang/utf8proc/issues/62
|
||||||
|
[#66]: https://github.com/JuliaLang/utf8proc/issues/66
|
||||||
|
[#68]: https://github.com/JuliaLang/utf8proc/issues/68
|
||||||
|
[#70]: https://github.com/JuliaLang/utf8proc/issues/70
|
||||||
|
[#77]: https://github.com/JuliaLang/utf8proc/issues/77
|
||||||
|
[#78]: https://github.com/JuliaLang/utf8proc/issues/78
|
||||||
|
[#79]: https://github.com/JuliaLang/utf8proc/issues/79
|
||||||
|
[#80]: https://github.com/JuliaLang/utf8proc/issues/80
|
||||||
116
3rdparty/utf8proc/README
vendored
116
3rdparty/utf8proc/README
vendored
|
|
@ -1,116 +0,0 @@
|
||||||
|
|
||||||
Please read the LICENSE file, which is shipping with this software.
|
|
||||||
|
|
||||||
|
|
||||||
*** QUICK START ***
|
|
||||||
|
|
||||||
For compilation of the C library call "make c-library", for compilation of
|
|
||||||
the ruby library call "make ruby-library" and for compilation of the
|
|
||||||
PostgreSQL extension call "make pgsql-library".
|
|
||||||
|
|
||||||
For ruby you can also create a gem-file by calling "make ruby-gem".
|
|
||||||
|
|
||||||
"make all" can be used to build everything, but both ruby and PostgreSQL
|
|
||||||
installations are required in this case.
|
|
||||||
|
|
||||||
|
|
||||||
*** GENERAL INFORMATION ***
|
|
||||||
|
|
||||||
The C library is found in this directory after successful compilation and
|
|
||||||
is named "libutf8proc.a" and "libutf8proc.so". The ruby library consists of
|
|
||||||
the files "utf8proc.rb" and "utf8proc_native.so", which are found in the
|
|
||||||
subdirectory "ruby/". If you chose to create a gem-file it is placed in the
|
|
||||||
"ruby/gem" directory. The PostgreSQL extension is named "utf8proc_pgsql.so"
|
|
||||||
and resides in the "pgsql/" directory.
|
|
||||||
|
|
||||||
Both the ruby library and the PostgreSQL extension are built as stand-alone
|
|
||||||
libraries and are therefore not dependent the dynamic version of the
|
|
||||||
C library files, but this behaviour might change in future releases.
|
|
||||||
|
|
||||||
The Unicode version being supported is 5.0.0.
|
|
||||||
Note: Version 4.1.0 of Unicode Standard Annex #29 was used, as
|
|
||||||
version 5.0.0 had not been available at the time of implementation.
|
|
||||||
|
|
||||||
For Unicode normalizations, the following options have to be used:
|
|
||||||
Normalization Form C: STABLE, COMPOSE
|
|
||||||
Normalization Form D: STABLE, DECOMPOSE
|
|
||||||
Normalization Form KC: STABLE, COMPOSE, COMPAT
|
|
||||||
Normalization Form KD: STABLE, DECOMPOSE, COMPAT
|
|
||||||
|
|
||||||
|
|
||||||
*** C LIBRARY ***
|
|
||||||
|
|
||||||
The documentation for the C library is found in the utf8proc.h header file.
|
|
||||||
"utf8proc_map" is most likely function you will be using for mapping UTF-8
|
|
||||||
strings, unless you want to allocate memory yourself.
|
|
||||||
|
|
||||||
|
|
||||||
*** RUBY API ***
|
|
||||||
|
|
||||||
The ruby library adds the methods "utf8map" and "utf8map!" to the String
|
|
||||||
class, and the method "utf8" to the Integer class.
|
|
||||||
|
|
||||||
The String#utf8map method does the same as the "utf8proc_map" C function.
|
|
||||||
Options for the mapping procedure are passed as symbols, i.e:
|
|
||||||
"Hello".utf8map(:casefold) => "hello"
|
|
||||||
|
|
||||||
The descriptions of all options are found in the C header file
|
|
||||||
"utf8proc.h". Please notice that the according symbols in ruby are all
|
|
||||||
lowercase.
|
|
||||||
|
|
||||||
String#utf8map! is the destructive function in the meaning that the string
|
|
||||||
is replaced by the result.
|
|
||||||
|
|
||||||
There are shortcuts for the 4 normalization forms specified by Unicode:
|
|
||||||
String#utf8nfd, String#utf8nfd!,
|
|
||||||
String#utf8nfc, String#utf8nfc!,
|
|
||||||
String#utf8nfkd, String#utf8nfkd!,
|
|
||||||
String#utf8nfkc, String#utf8nfkc!
|
|
||||||
|
|
||||||
The method Integer#utf8 returns a UTF-8 string, which is containing the
|
|
||||||
unicode char given by the code point.
|
|
||||||
0x000A.utf8 => "\n"
|
|
||||||
0x2028.utf8 => "\342\200\250"
|
|
||||||
|
|
||||||
|
|
||||||
*** POSTGRESQL API ***
|
|
||||||
|
|
||||||
For PostgreSQL there are two SQL functions supplied named "unifold" and
|
|
||||||
"unistrip". These functions function can be used to prepare index fields in
|
|
||||||
order to be folded in a way where string-comparisons make more sense, e.g.
|
|
||||||
where "bathtub" == "bath<soft hyphen>tub"
|
|
||||||
or "Hello World" == "hello world".
|
|
||||||
|
|
||||||
CREATE TABLE people (
|
|
||||||
id serial8 primary key,
|
|
||||||
name text,
|
|
||||||
CHECK (unifold(name) NOTNULL)
|
|
||||||
);
|
|
||||||
CREATE INDEX name_idx ON people (unifold(name));
|
|
||||||
SELECT * FROM people WHERE unifold(name) = unifold('John Doe');
|
|
||||||
|
|
||||||
The function "unistrip" removes character marks like accents or diaeresis,
|
|
||||||
while "unifold" keeps then.
|
|
||||||
|
|
||||||
NOTICE: The outputs of the function can change between releases, as
|
|
||||||
utf8proc does not follow a versioning stability policy. You have to
|
|
||||||
rebuild your database indicies, if you upgrade to a newer version
|
|
||||||
of utf8proc.
|
|
||||||
|
|
||||||
|
|
||||||
*** TODO ***
|
|
||||||
|
|
||||||
- detect stable code points and process segments independently in order to
|
|
||||||
save memory
|
|
||||||
- do a quick check before normalizing strings to optimize speed
|
|
||||||
- support stream processing
|
|
||||||
|
|
||||||
|
|
||||||
*** CONTACT ***
|
|
||||||
|
|
||||||
If you find any bugs or experience difficulties in compiling this software,
|
|
||||||
please contact us:
|
|
||||||
|
|
||||||
Project page: http://www.public-software-group.org/utf8proc
|
|
||||||
|
|
||||||
|
|
||||||
69
3rdparty/utf8proc/README.md
vendored
Executable file
69
3rdparty/utf8proc/README.md
vendored
Executable file
|
|
@ -0,0 +1,69 @@
|
||||||
|
# utf8proc
|
||||||
|
[](https://travis-ci.org/JuliaLang/utf8proc)
|
||||||
|
[](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
|
||||||
|
|
||||||
|
|
||||||
|
[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
|
||||||
|
library that provides Unicode normalization, case-folding, and other
|
||||||
|
operations for data in the [UTF-8
|
||||||
|
encoding](http://en.wikipedia.org/wiki/UTF-8). It was [initially
|
||||||
|
developed](http://www.public-software-group.org/utf8proc) by Jan
|
||||||
|
Behrens and the rest of the [Public Software
|
||||||
|
Group](http://www.public-software-group.org/), who deserve *nearly all
|
||||||
|
of the credit* for this package. With the blessing of the Public
|
||||||
|
Software Group, the [Julia developers](http://julialang.org/) have
|
||||||
|
taken over development of utf8proc, since the original developers have
|
||||||
|
moved to other projects.
|
||||||
|
|
||||||
|
(utf8proc is used for basic Unicode
|
||||||
|
support in the [Julia language](http://julialang.org/), and the Julia
|
||||||
|
developers became involved because they wanted to add Unicode 7 support and other features.)
|
||||||
|
|
||||||
|
(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
|
||||||
|
We removed those from utf8proc in order to focus exclusively on the C
|
||||||
|
library for the time being, but plan to add them back in or release them as separate packages.)
|
||||||
|
|
||||||
|
The utf8proc package is licensed under the
|
||||||
|
free/open-source [MIT "expat"
|
||||||
|
license](http://opensource.org/licenses/MIT) (plus certain Unicode
|
||||||
|
data governed by the similarly permissive [Unicode data
|
||||||
|
license](http://www.unicode.org/copyright.html#Exhibit1)); please see
|
||||||
|
the included `LICENSE.md` file for more detailed information.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
For compilation of the C library run `make`.
|
||||||
|
|
||||||
|
## General Information
|
||||||
|
|
||||||
|
The C library is found in this directory after successful compilation
|
||||||
|
and is named `libutf8proc.a` (for the static library) and
|
||||||
|
`libutf8proc.so` (for the dynamic library).
|
||||||
|
|
||||||
|
The Unicode version supported is 9.0.0.
|
||||||
|
|
||||||
|
For Unicode normalizations, the following options are used:
|
||||||
|
|
||||||
|
* Normalization Form C: `STABLE`, `COMPOSE`
|
||||||
|
* Normalization Form D: `STABLE`, `DECOMPOSE`
|
||||||
|
* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
|
||||||
|
* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
|
||||||
|
|
||||||
|
## C Library
|
||||||
|
|
||||||
|
The documentation for the C library is found in the `utf8proc.h` header file.
|
||||||
|
`utf8proc_map` is function you will most likely be using for mapping UTF-8
|
||||||
|
strings, unless you want to allocate memory yourself.
|
||||||
|
|
||||||
|
## To Do
|
||||||
|
|
||||||
|
See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
|
||||||
|
|
||||||
|
## Contact
|
||||||
|
|
||||||
|
Bug reports, feature requests, and other queries can be filed at
|
||||||
|
the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
|
||||||
|
|
||||||
|
## See also
|
||||||
|
|
||||||
|
An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.
|
||||||
42
3rdparty/utf8proc/appveyor.yml
vendored
Executable file
42
3rdparty/utf8proc/appveyor.yml
vendored
Executable file
|
|
@ -0,0 +1,42 @@
|
||||||
|
branches:
|
||||||
|
only:
|
||||||
|
- master
|
||||||
|
- /release-.*/
|
||||||
|
|
||||||
|
notifications:
|
||||||
|
- provider: Email
|
||||||
|
on_build_success: false
|
||||||
|
on_build_failure: false
|
||||||
|
on_build_status_changed: false
|
||||||
|
|
||||||
|
build_script:
|
||||||
|
- ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
|
||||||
|
https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
|
||||||
|
Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
|
||||||
|
throw "There are newer queued builds for this pull request, failing early." }
|
||||||
|
- mkdir msvc_static
|
||||||
|
- cd msvc_static
|
||||||
|
- cmake ..
|
||||||
|
- cmake --build .
|
||||||
|
- mkdir ..\msvc_shared
|
||||||
|
- cd ..\msvc_shared
|
||||||
|
- cmake .. -DBUILD_SHARED_LIBS=ON
|
||||||
|
- cmake --build .
|
||||||
|
- set PATH=C:\MinGW\bin;%PATH%
|
||||||
|
- C:\MinGW\msys\1.0\bin\sh --login -c "
|
||||||
|
cd /c/projects/utf8proc &&
|
||||||
|
mkdir mingw_static &&
|
||||||
|
cd mingw_static &&
|
||||||
|
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
|
||||||
|
make &&
|
||||||
|
mkdir ../mingw_shared &&
|
||||||
|
cd ../mingw_shared &&
|
||||||
|
cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
|
||||||
|
make
|
||||||
|
"
|
||||||
|
|
||||||
|
on_finish:
|
||||||
|
# Uncomment the following line for interactive debugging, which
|
||||||
|
# will print login data for a temporary remote session after the
|
||||||
|
# build. This requires an RDP version 6 client, e.g., FreeRDP.
|
||||||
|
#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
|
||||||
39
3rdparty/utf8proc/bench/Makefile
vendored
Executable file
39
3rdparty/utf8proc/bench/Makefile
vendored
Executable file
|
|
@ -0,0 +1,39 @@
|
||||||
|
CURL=curl
|
||||||
|
|
||||||
|
CC = cc
|
||||||
|
CFLAGS = -O2 -std=c99 -pedantic -Wall
|
||||||
|
|
||||||
|
all: bench
|
||||||
|
|
||||||
|
LIBUTF8PROC = ../utf8proc.o
|
||||||
|
|
||||||
|
bench: bench.o util.o $(LIBUTF8PROC)
|
||||||
|
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
|
||||||
|
|
||||||
|
DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
|
||||||
|
DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
|
||||||
|
|
||||||
|
$(DATAFILES):
|
||||||
|
$(CURL) -O $(DATAURL)/$@
|
||||||
|
|
||||||
|
bench.out: $(DATAFILES) bench
|
||||||
|
./bench -nfkc $(DATAFILES) > $@
|
||||||
|
|
||||||
|
# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
|
||||||
|
icu: icu.o util.o
|
||||||
|
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
|
||||||
|
|
||||||
|
icu.out: $(DATAFILES) icu
|
||||||
|
./icu $(DATAFILES) > $@
|
||||||
|
|
||||||
|
unistring: unistring.o util.o
|
||||||
|
$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
|
||||||
|
|
||||||
|
unistring.out: $(DATAFILES) unistring
|
||||||
|
./unistring $(DATAFILES) > $@
|
||||||
|
|
||||||
|
.c.o:
|
||||||
|
$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf *.o *.txt bench *.out icu unistring
|
||||||
56
3rdparty/utf8proc/bench/bench.c
vendored
Executable file
56
3rdparty/utf8proc/bench/bench.c
vendored
Executable file
|
|
@ -0,0 +1,56 @@
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
#include "utf8proc.h"
|
||||||
|
#include "util.h"
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int i, j;
|
||||||
|
int options = 0;
|
||||||
|
|
||||||
|
for (i = 1; i < argc; ++i) {
|
||||||
|
if (!strcmp(argv[i], "-nfkc")) {
|
||||||
|
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfkd")) {
|
||||||
|
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfc")) {
|
||||||
|
options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfd")) {
|
||||||
|
options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-casefold")) {
|
||||||
|
options |= UTF8PROC_CASEFOLD;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (argv[i][0] == '-') {
|
||||||
|
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len;
|
||||||
|
uint8_t *src = readfile(argv[i], &len);
|
||||||
|
if (!src) {
|
||||||
|
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
uint8_t *dest;
|
||||||
|
mytime start = gettime();
|
||||||
|
for (j = 0; j < 100; ++j) {
|
||||||
|
utf8proc_map(src, len, &dest, options);
|
||||||
|
free(dest);
|
||||||
|
}
|
||||||
|
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||||
|
free(src);
|
||||||
|
}
|
||||||
|
|
||||||
|
return EXIT_SUCCESS;
|
||||||
|
}
|
||||||
61
3rdparty/utf8proc/bench/icu.c
vendored
Executable file
61
3rdparty/utf8proc/bench/icu.c
vendored
Executable file
|
|
@ -0,0 +1,61 @@
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
/* ICU4C */
|
||||||
|
#include <unicode/utypes.h>
|
||||||
|
#include <unicode/ustring.h>
|
||||||
|
#include <unicode/ucnv.h>
|
||||||
|
#include <unicode/unorm2.h>
|
||||||
|
|
||||||
|
#include "util.h"
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
|
||||||
|
UErrorCode err;
|
||||||
|
UConverter *uc = ucnv_open("UTF8", &err);
|
||||||
|
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||||
|
|
||||||
|
const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
|
||||||
|
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||||
|
|
||||||
|
for (i = 1; i < argc; ++i) {
|
||||||
|
if (argv[i][0] == '-') {
|
||||||
|
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len;
|
||||||
|
uint8_t *src = readfile(argv[i], &len);
|
||||||
|
if (!src) {
|
||||||
|
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* convert UTF8 data to ICU's UTF16 */
|
||||||
|
UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
|
||||||
|
ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
|
||||||
|
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||||
|
size_t ulen = u_strlen(usrc);
|
||||||
|
|
||||||
|
/* ICU's insane normalization API requires you to
|
||||||
|
know the size of the destination buffer in advance,
|
||||||
|
or alternatively to repeatly try normalizing and
|
||||||
|
double the buffer size until it succeeds. Here, I just
|
||||||
|
allocate a huge destination buffer to avoid the issue. */
|
||||||
|
UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
|
||||||
|
|
||||||
|
mytime start = gettime();
|
||||||
|
for (int i = 0; i < 100; ++i) {
|
||||||
|
unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
|
||||||
|
if (U_FAILURE(err)) return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||||
|
free(udest);
|
||||||
|
free(usrc);
|
||||||
|
free(src);
|
||||||
|
}
|
||||||
|
|
||||||
|
return EXIT_SUCCESS;
|
||||||
|
}
|
||||||
60
3rdparty/utf8proc/bench/unistring.c
vendored
Executable file
60
3rdparty/utf8proc/bench/unistring.c
vendored
Executable file
|
|
@ -0,0 +1,60 @@
|
||||||
|
/* comparitive benchmark of GNU libunistring */
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/* libunistring */
|
||||||
|
#include <unistr.h>
|
||||||
|
#include <uninorm.h>
|
||||||
|
|
||||||
|
#include "util.h"
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
uninorm_t nf = UNINORM_NFKC;
|
||||||
|
|
||||||
|
for (i = 1; i < argc; ++i) {
|
||||||
|
if (!strcmp(argv[i], "-nfkc")) {
|
||||||
|
nf = UNINORM_NFKC;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfkd")) {
|
||||||
|
nf = UNINORM_NFKD;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfc")) {
|
||||||
|
nf = UNINORM_NFC;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!strcmp(argv[i], "-nfd")) {
|
||||||
|
nf = UNINORM_NFD;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (argv[i][0] == '-') {
|
||||||
|
fprintf(stderr, "unrecognized option: %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t len;
|
||||||
|
uint8_t *src = readfile(argv[i], &len);
|
||||||
|
if (!src) {
|
||||||
|
fprintf(stderr, "error reading %s\n", argv[i]);
|
||||||
|
return EXIT_FAILURE;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t destlen;
|
||||||
|
uint8_t *dest;
|
||||||
|
mytime start = gettime();
|
||||||
|
for (int i = 0; i < 100; ++i) {
|
||||||
|
dest = u8_normalize(nf, src, len, NULL, &destlen);
|
||||||
|
if (!dest) return EXIT_FAILURE;
|
||||||
|
free(dest);
|
||||||
|
}
|
||||||
|
printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
|
||||||
|
free(src);
|
||||||
|
}
|
||||||
|
|
||||||
|
return EXIT_SUCCESS;
|
||||||
|
}
|
||||||
39
3rdparty/utf8proc/bench/util.c
vendored
Executable file
39
3rdparty/utf8proc/bench/util.c
vendored
Executable file
|
|
@ -0,0 +1,39 @@
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
#include "util.h"
|
||||||
|
|
||||||
|
/* read file named FILENAME into an array of *len bytes,
|
||||||
|
returning NULL on error */
|
||||||
|
uint8_t *readfile(const char *filename, size_t *len)
|
||||||
|
{
|
||||||
|
*len = 0;
|
||||||
|
struct stat st;
|
||||||
|
if (0 != stat(filename, &st)) return NULL;
|
||||||
|
*len = st.st_size;
|
||||||
|
FILE *f = fopen(filename, "r");
|
||||||
|
if (!f) return NULL;
|
||||||
|
uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
|
||||||
|
if (!s) return NULL;
|
||||||
|
if (fread(s, 1, *len, f) != *len) {
|
||||||
|
free(s);
|
||||||
|
s = NULL;
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
mytime gettime(void) {
|
||||||
|
mytime t;
|
||||||
|
gettimeofday(&t, NULL);
|
||||||
|
return t;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* time difference in seconds */
|
||||||
|
double elapsed(mytime t1, mytime t0)
|
||||||
|
{
|
||||||
|
return (double)(t1.tv_sec - t0.tv_sec) +
|
||||||
|
(double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
|
||||||
|
}
|
||||||
|
|
||||||
22
3rdparty/utf8proc/bench/util.h
vendored
Executable file
22
3rdparty/utf8proc/bench/util.h
vendored
Executable file
|
|
@ -0,0 +1,22 @@
|
||||||
|
#ifndef UTIL_H
|
||||||
|
#define UTIL_H 1
|
||||||
|
|
||||||
|
#include <inttypes.h>
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
uint8_t *readfile(const char *filename, size_t *len);
|
||||||
|
|
||||||
|
typedef struct timeval mytime;
|
||||||
|
mytime gettime(void);
|
||||||
|
double elapsed(mytime t1, mytime t0);
|
||||||
|
|
||||||
|
#ifdef __cplusplus
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#endif /* UTIL_H */
|
||||||
66
3rdparty/utf8proc/data/Makefile
vendored
Executable file
66
3rdparty/utf8proc/data/Makefile
vendored
Executable file
|
|
@ -0,0 +1,66 @@
|
||||||
|
# Unicode data generation rules. Except for the test data files, most
|
||||||
|
# users will not use these Makefile rules, which are primarily to re-generate
|
||||||
|
# unicode_data.c when we get a new Unicode version or charwidth data; they
|
||||||
|
# require ruby, fontforge, and julia to be installed.
|
||||||
|
|
||||||
|
# programs
|
||||||
|
CURL=curl
|
||||||
|
RUBY=ruby
|
||||||
|
PERL=perl
|
||||||
|
MAKE=make
|
||||||
|
JULIA=julia
|
||||||
|
FONTFORGE=fontforge
|
||||||
|
CURLFLAGS = --retry 5 --location
|
||||||
|
|
||||||
|
.PHONY: clean
|
||||||
|
|
||||||
|
.DELETE_ON_ERROR:
|
||||||
|
|
||||||
|
utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
|
||||||
|
$(RUBY) data_generator.rb < UnicodeData.txt > $@
|
||||||
|
|
||||||
|
# GNU Unifont version for font metric calculations:
|
||||||
|
UNIFONT_VERSION=9.0.01
|
||||||
|
|
||||||
|
unifont.ttf:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
|
||||||
|
|
||||||
|
unifont_upper.ttf:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
|
||||||
|
|
||||||
|
%.sfd: %.ttf
|
||||||
|
$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
|
||||||
|
|
||||||
|
CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
|
||||||
|
$(JULIA) charwidths.jl > $@
|
||||||
|
|
||||||
|
# Unicode data version
|
||||||
|
UNICODE_VERSION=9.0.0
|
||||||
|
|
||||||
|
UnicodeData.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
|
||||||
|
|
||||||
|
EastAsianWidth.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
|
||||||
|
|
||||||
|
GraphemeBreakProperty.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
|
||||||
|
|
||||||
|
DerivedCoreProperties.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
|
||||||
|
|
||||||
|
CompositionExclusions.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
|
||||||
|
|
||||||
|
CaseFolding.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
|
||||||
|
|
||||||
|
NormalizationTest.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
|
||||||
|
|
||||||
|
GraphemeBreakTest.txt:
|
||||||
|
$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
|
||||||
|
rm -f utf8proc_data.c.new
|
||||||
190
3rdparty/utf8proc/data/charwidths.jl
vendored
Executable file
190
3rdparty/utf8proc/data/charwidths.jl
vendored
Executable file
|
|
@ -0,0 +1,190 @@
|
||||||
|
# Following work by @jiahao, we compute character widths using a combination of
|
||||||
|
# * advance widths from GNU Unifont (advance width 512 = 1 en)
|
||||||
|
# * UAX 11: East Asian Width
|
||||||
|
# * a few exceptions as needed
|
||||||
|
# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
|
||||||
|
#
|
||||||
|
# Requires Julia (obviously) and FontForge.
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Julia 0.3/0.4 compatibility (taken from Compat package)
|
||||||
|
if VERSION < v"0.4.0-dev+1387"
|
||||||
|
typealias AbstractString String
|
||||||
|
end
|
||||||
|
if VERSION < v"0.4.0-dev+1419"
|
||||||
|
const UInt32 = Uint32
|
||||||
|
end
|
||||||
|
if VERSION < v"0.4.0-dev+3874"
|
||||||
|
Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
|
||||||
|
end
|
||||||
|
|
||||||
|
CharWidths = Dict{Int,Int}()
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Use ../libutf8proc for category codes, rather than the one in Julia,
|
||||||
|
# to minimize bootstrapping complexity when a new version of Unicode comes out.
|
||||||
|
catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
|
||||||
|
|
||||||
|
# use Base.UTF8proc module to get category codes constants, since
|
||||||
|
# we won't change these in utf8proc.
|
||||||
|
import Base.UTF8proc
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Use a default width of 1 for all character categories that are
|
||||||
|
# letter/symbol/number-like. This can be overriden by Unifont or UAX 11
|
||||||
|
# below, but provides a useful nonzero fallback for new codepoints when
|
||||||
|
# a new Unicode version has been released but Unifont hasn't been updated yet.
|
||||||
|
|
||||||
|
zerowidth = Set{Int}() # categories that may contain zero-width chars
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
|
||||||
|
push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
|
||||||
|
for c in 0x0000:0x110000
|
||||||
|
if catcode(c) ∉ zerowidth
|
||||||
|
CharWidths[c] = 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Widths from GNU Unifont
|
||||||
|
|
||||||
|
#Read sfdfile for character widths
|
||||||
|
function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
|
||||||
|
state=:seekchar
|
||||||
|
lineno = 0
|
||||||
|
codepoint = width = nothing
|
||||||
|
for line in readlines(open(filename))
|
||||||
|
lineno += 1
|
||||||
|
if state==:seekchar #StartChar: nonmarkingreturn
|
||||||
|
if contains(line, "StartChar: ")
|
||||||
|
codepoint = nothing
|
||||||
|
width = nothing
|
||||||
|
state = :readdata
|
||||||
|
end
|
||||||
|
elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
|
||||||
|
contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
|
||||||
|
contains(line, "Width:") && (width = parse(Int, split(line)[2]))
|
||||||
|
if codepoint!=nothing && width!=nothing && codepoint >= 0
|
||||||
|
w=div(width, 512) # 512 units to the en
|
||||||
|
if w > 0
|
||||||
|
# only add nonzero widths, since (1) the default is zero
|
||||||
|
# and (2) this circumvents some apparent bugs in Unifont
|
||||||
|
# (https://savannah.gnu.org/bugs/index.php?45395)
|
||||||
|
CharWidths[codepoint] = w
|
||||||
|
end
|
||||||
|
state = :seekchar
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
CharWidths
|
||||||
|
end
|
||||||
|
CharWidths=parsesfd("unifont.sfd", CharWidths)
|
||||||
|
CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Widths from UAX #11: East Asian Width
|
||||||
|
# .. these take precedence over the Unifont width for all codepoints
|
||||||
|
# listed explicitly as wide/full/narrow/half-width
|
||||||
|
|
||||||
|
for line in readlines(open("EastAsianWidth.txt"))
|
||||||
|
#Strip comments
|
||||||
|
line[1] == '#' && continue
|
||||||
|
precomment = split(line, '#')[1]
|
||||||
|
#Parse code point range and width code
|
||||||
|
tokens = split(precomment, ';')
|
||||||
|
length(tokens) >= 2 || continue
|
||||||
|
charrange = tokens[1]
|
||||||
|
width = strip(tokens[2])
|
||||||
|
#Parse code point range into Julia UnitRange
|
||||||
|
rangetokens = split(charrange, "..")
|
||||||
|
charstart = parse(UInt32, "0x"*rangetokens[1])
|
||||||
|
charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
|
||||||
|
|
||||||
|
#Assign widths
|
||||||
|
for c in charstart:charend
|
||||||
|
if width=="W" || width=="F" # wide or full
|
||||||
|
CharWidths[c]=2
|
||||||
|
elseif width=="Na"|| width=="H" # narrow or half
|
||||||
|
CharWidths[c]=1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# A few exceptions to the above cases, found by manual comparison
|
||||||
|
# to other wcwidth functions and similar checks.
|
||||||
|
|
||||||
|
for c in keys(CharWidths)
|
||||||
|
cat = catcode(c)
|
||||||
|
|
||||||
|
# make sure format control character (category Cf) have width 0,
|
||||||
|
# except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
|
||||||
|
if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
|
||||||
|
CharWidths[c]=0
|
||||||
|
end
|
||||||
|
|
||||||
|
# Unifont has nonzero width for a number of non-spacing combining
|
||||||
|
# characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
|
||||||
|
# the variation selectors
|
||||||
|
if cat==UTF8proc.UTF8PROC_CATEGORY_MN
|
||||||
|
CharWidths[c]=0
|
||||||
|
end
|
||||||
|
|
||||||
|
# We also assign width of zero to unassigned and private-use
|
||||||
|
# codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
|
||||||
|
# but since these are nonstandard it seems questionable to recognize them).
|
||||||
|
if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
|
||||||
|
CharWidths[c]=0
|
||||||
|
end
|
||||||
|
|
||||||
|
# for some reason, Unifont has width-2 glyphs for ASCII control chars
|
||||||
|
if cat==UTF8proc.UTF8PROC_CATEGORY_CC
|
||||||
|
CharWidths[c]=0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
#By definition, should have zero width (on the same line)
|
||||||
|
#0x002028 '
' category: Zl name: LINE SEPARATOR/
|
||||||
|
#0x002029 '
' category: Zp name: PARAGRAPH SEPARATOR/
|
||||||
|
CharWidths[0x2028]=0
|
||||||
|
CharWidths[0x2029]=0
|
||||||
|
|
||||||
|
#By definition, should be narrow = width of 1 en space
|
||||||
|
#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
|
||||||
|
CharWidths[0x202f]=1
|
||||||
|
|
||||||
|
#By definition, should be wide = width of 1 em space
|
||||||
|
#0x002001 ' ' category: Zs name: EM QUAD/
|
||||||
|
#0x002003 ' ' category: Zs name: EM SPACE/
|
||||||
|
CharWidths[0x2001]=2
|
||||||
|
CharWidths[0x2003]=2
|
||||||
|
|
||||||
|
#############################################################################
|
||||||
|
# Output (to a file or pipe) for processing by data_generator.rb
|
||||||
|
# ... don't bother to output zero widths since that will be the default.
|
||||||
|
|
||||||
|
firstc = 0x000000
|
||||||
|
lastv = 0
|
||||||
|
uhex(c) = uppercase(hex(c,4))
|
||||||
|
for c in 0x0000:0x110000
|
||||||
|
v = get(CharWidths, c, 0)
|
||||||
|
if v != lastv || c == 0x110000
|
||||||
|
v < 4 || error("invalid charwidth $v for $c")
|
||||||
|
if firstc+1 < c
|
||||||
|
println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
|
||||||
|
else
|
||||||
|
println(uhex(firstc), "; ", lastv)
|
||||||
|
end
|
||||||
|
firstc = c
|
||||||
|
lastv = v
|
||||||
|
end
|
||||||
|
end
|
||||||
411
3rdparty/utf8proc/data/data_generator.rb
vendored
Executable file
411
3rdparty/utf8proc/data/data_generator.rb
vendored
Executable file
|
|
@ -0,0 +1,411 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
# This file was used to generate the 'unicode_data.c' file by parsing the
|
||||||
|
# Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
|
||||||
|
# It is included for informational purposes only and not intended for
|
||||||
|
# production use.
|
||||||
|
|
||||||
|
|
||||||
|
# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
# copy of this software and associated documentation files (the "Software"),
|
||||||
|
# to deal in the Software without restriction, including without limitation
|
||||||
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
# and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
# Software is furnished to do so, subject to the following conditions:
|
||||||
|
#
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
# DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
|
||||||
|
# This file contains derived data from a modified version of the
|
||||||
|
# Unicode data files. The following license applies to that data:
|
||||||
|
#
|
||||||
|
# COPYRIGHT AND PERMISSION NOTICE
|
||||||
|
#
|
||||||
|
# Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
||||||
|
# under the Terms of Use in http://www.unicode.org/copyright.html.
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
# copy of the Unicode data files and any associated documentation (the "Data
|
||||||
|
# Files") or Unicode software and any associated documentation (the
|
||||||
|
# "Software") to deal in the Data Files or Software without restriction,
|
||||||
|
# including without limitation the rights to use, copy, modify, merge,
|
||||||
|
# publish, distribute, and/or sell copies of the Data Files or Software, and
|
||||||
|
# to permit persons to whom the Data Files or Software are furnished to do
|
||||||
|
# so, provided that (a) the above copyright notice(s) and this permission
|
||||||
|
# notice appear with all copies of the Data Files or Software, (b) both the
|
||||||
|
# above copyright notice(s) and this permission notice appear in associated
|
||||||
|
# documentation, and (c) there is clear notice in each modified Data File or
|
||||||
|
# in the Software as well as in the documentation associated with the Data
|
||||||
|
# File(s) or Software that the data or software has been modified.
|
||||||
|
#
|
||||||
|
# THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
||||||
|
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
|
||||||
|
# THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
|
||||||
|
# INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
|
||||||
|
# CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
|
||||||
|
# USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
||||||
|
# TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
||||||
|
# PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
||||||
|
#
|
||||||
|
# Except as contained in this notice, the name of a copyright holder shall
|
||||||
|
# not be used in advertising or otherwise to promote the sale, use or other
|
||||||
|
# dealings in these Data Files or Software without prior written
|
||||||
|
# authorization of the copyright holder.
|
||||||
|
|
||||||
|
|
||||||
|
$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
|
||||||
|
$ignorable = []
|
||||||
|
$ignorable_list.each_line do |entry|
|
||||||
|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
|
||||||
|
$1.hex.upto($2.hex) { |e2| $ignorable << e2 }
|
||||||
|
elsif entry =~ /^[0-9A-F]+/
|
||||||
|
$ignorable << $&.hex
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
|
||||||
|
$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
|
||||||
|
$grapheme_boundclass_list.each_line do |entry|
|
||||||
|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
|
||||||
|
$1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
|
||||||
|
elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
|
||||||
|
$grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
$charwidth_list = File.read("CharWidths.txt")
|
||||||
|
$charwidth = Hash.new(0)
|
||||||
|
$charwidth_list.each_line do |entry|
|
||||||
|
if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
|
||||||
|
$1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
|
||||||
|
elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
|
||||||
|
$charwidth[$1.hex] = $2.to_i
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
|
||||||
|
$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
|
||||||
|
|
||||||
|
$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
|
||||||
|
$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
|
||||||
|
|
||||||
|
$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
|
||||||
|
$case_folding = {}
|
||||||
|
$case_folding_string.chomp.split("\n").each do |line|
|
||||||
|
next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
|
||||||
|
$case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
|
||||||
|
end
|
||||||
|
|
||||||
|
$int_array = []
|
||||||
|
$int_array_indicies = {}
|
||||||
|
|
||||||
|
def str2c(string, prefix)
|
||||||
|
return "0" if string.nil?
|
||||||
|
return "UTF8PROC_#{prefix}_#{string.upcase}"
|
||||||
|
end
|
||||||
|
def pushary(array)
|
||||||
|
idx = $int_array_indicies[array]
|
||||||
|
unless idx
|
||||||
|
$int_array_indicies[array] = $int_array.length
|
||||||
|
idx = $int_array.length
|
||||||
|
array.each { |entry| $int_array << entry }
|
||||||
|
end
|
||||||
|
return idx
|
||||||
|
end
|
||||||
|
def cpary2utf16encoded(array)
|
||||||
|
return array.flat_map { |cp|
|
||||||
|
if (cp <= 0xFFFF)
|
||||||
|
raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
|
||||||
|
cp
|
||||||
|
else
|
||||||
|
temp = cp - 0x10000
|
||||||
|
[(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
|
||||||
|
end
|
||||||
|
}
|
||||||
|
end
|
||||||
|
def cpary2c(array)
|
||||||
|
return "UINT16_MAX" if array.nil? || array.length == 0
|
||||||
|
lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ...
|
||||||
|
array = cpary2utf16encoded(array)
|
||||||
|
if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
|
||||||
|
array = [lencode] + array
|
||||||
|
lencode = 7
|
||||||
|
end
|
||||||
|
idx = pushary(array)
|
||||||
|
raise "Array index out of bound" if idx > 0x1FFF
|
||||||
|
return "#{idx | (lencode << 13)}"
|
||||||
|
end
|
||||||
|
def singlecpmap(cp)
|
||||||
|
return "UINT16_MAX" if cp == nil
|
||||||
|
idx = pushary(cpary2utf16encoded([cp]))
|
||||||
|
raise "Array index out of bound" if idx > 0xFFFF
|
||||||
|
return "#{idx}"
|
||||||
|
end
|
||||||
|
|
||||||
|
class UnicodeChar
|
||||||
|
attr_accessor :code, :name, :category, :combining_class, :bidi_class,
|
||||||
|
:decomp_type, :decomp_mapping,
|
||||||
|
:bidi_mirrored,
|
||||||
|
:uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
|
||||||
|
#caches:
|
||||||
|
:c_entry_index, :c_decomp_mapping, :c_case_folding
|
||||||
|
def initialize(line)
|
||||||
|
raise "Could not parse input." unless line =~ /^
|
||||||
|
([0-9A-F]+); # code
|
||||||
|
([^;]+); # name
|
||||||
|
([A-Z]+); # general category
|
||||||
|
([0-9]+); # canonical combining class
|
||||||
|
([A-Z]+); # bidi class
|
||||||
|
(<([A-Z]*)>)? # decomposition type
|
||||||
|
((\ ?[0-9A-F]+)*); # decompomposition mapping
|
||||||
|
([0-9]*); # decimal digit
|
||||||
|
([0-9]*); # digit
|
||||||
|
([^;]*); # numeric
|
||||||
|
([YN]*); # bidi mirrored
|
||||||
|
([^;]*); # unicode 1.0 name
|
||||||
|
([^;]*); # iso comment
|
||||||
|
([0-9A-F]*); # simple uppercase mapping
|
||||||
|
([0-9A-F]*); # simple lowercase mapping
|
||||||
|
([0-9A-F]*)$/ix # simple titlecase mapping
|
||||||
|
@code = $1.hex
|
||||||
|
@name = $2
|
||||||
|
@category = $3
|
||||||
|
@combining_class = Integer($4)
|
||||||
|
@bidi_class = $5
|
||||||
|
@decomp_type = $7
|
||||||
|
@decomp_mapping = ($8=='') ? nil :
|
||||||
|
$8.split.collect { |element| element.hex }
|
||||||
|
@bidi_mirrored = ($13=='Y') ? true : false
|
||||||
|
@uppercase_mapping = ($16=='') ? nil : $16.hex
|
||||||
|
@lowercase_mapping = ($17=='') ? nil : $17.hex
|
||||||
|
@titlecase_mapping = ($18=='') ? nil : $18.hex
|
||||||
|
end
|
||||||
|
def case_folding
|
||||||
|
$case_folding[code]
|
||||||
|
end
|
||||||
|
def c_entry(comb_indicies)
|
||||||
|
" " <<
|
||||||
|
"{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
|
||||||
|
"#{str2c bidi_class, 'BIDI_CLASS'}, " <<
|
||||||
|
"#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
|
||||||
|
"#{c_decomp_mapping}, " <<
|
||||||
|
"#{c_case_folding}, " <<
|
||||||
|
"#{singlecpmap uppercase_mapping }, " <<
|
||||||
|
"#{singlecpmap lowercase_mapping }, " <<
|
||||||
|
"#{singlecpmap titlecase_mapping }, " <<
|
||||||
|
"#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
|
||||||
|
"#{bidi_mirrored}, " <<
|
||||||
|
"#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
|
||||||
|
"#{$ignorable.include?(code)}, " <<
|
||||||
|
"#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
|
||||||
|
"#{$charwidth[code]}, 0, " <<
|
||||||
|
"#{$grapheme_boundclass[code]}},\n"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
chars = []
|
||||||
|
char_hash = {}
|
||||||
|
|
||||||
|
while gets
|
||||||
|
if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
|
||||||
|
first = $1.hex
|
||||||
|
gets
|
||||||
|
char = UnicodeChar.new($_)
|
||||||
|
raise "No last character of sequence found." unless
|
||||||
|
$_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
|
||||||
|
last = $1.hex
|
||||||
|
name = "<#{$2}>"
|
||||||
|
for i in first..last
|
||||||
|
char_clone = char.clone
|
||||||
|
char_clone.code = i
|
||||||
|
char_clone.name = name
|
||||||
|
char_hash[char_clone.code] = char_clone
|
||||||
|
chars << char_clone
|
||||||
|
end
|
||||||
|
else
|
||||||
|
char = UnicodeChar.new($_)
|
||||||
|
char_hash[char.code] = char
|
||||||
|
chars << char
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
comb1st_indicies = {}
|
||||||
|
comb2nd_indicies = {}
|
||||||
|
comb2nd_indicies_sorted_keys = []
|
||||||
|
comb2nd_indicies_nonbasic = {}
|
||||||
|
comb_array = []
|
||||||
|
|
||||||
|
chars.each do |char|
|
||||||
|
if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
|
||||||
|
char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
|
||||||
|
char_hash[char.decomp_mapping[0]].combining_class == 0 and
|
||||||
|
not $exclusions.include?(char.code)
|
||||||
|
|
||||||
|
dm0 = char.decomp_mapping[0]
|
||||||
|
dm1 = char.decomp_mapping[1]
|
||||||
|
unless comb1st_indicies[dm0]
|
||||||
|
comb1st_indicies[dm0] = comb1st_indicies.keys.length
|
||||||
|
end
|
||||||
|
unless comb2nd_indicies[dm1]
|
||||||
|
comb2nd_indicies_sorted_keys << dm1
|
||||||
|
comb2nd_indicies[dm1] = comb2nd_indicies.keys.length
|
||||||
|
end
|
||||||
|
comb_array[comb1st_indicies[dm0]] ||= []
|
||||||
|
raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
|
||||||
|
comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
|
||||||
|
|
||||||
|
comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
|
||||||
|
end
|
||||||
|
char.c_decomp_mapping = cpary2c(char.decomp_mapping)
|
||||||
|
char.c_case_folding = cpary2c(char.case_folding)
|
||||||
|
end
|
||||||
|
|
||||||
|
comb_indicies = {}
|
||||||
|
cumoffset = 0
|
||||||
|
comb1st_indicies_lastoffsets = []
|
||||||
|
comb1st_indicies_firstoffsets = []
|
||||||
|
comb1st_indicies.each do |dm0, index|
|
||||||
|
first = nil
|
||||||
|
last = nil
|
||||||
|
offset = 0
|
||||||
|
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
|
||||||
|
if comb_array[index][b]
|
||||||
|
first = offset unless first
|
||||||
|
last = offset
|
||||||
|
last += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||||
|
end
|
||||||
|
offset += 1
|
||||||
|
offset += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||||
|
end
|
||||||
|
comb1st_indicies_firstoffsets[index] = first
|
||||||
|
comb1st_indicies_lastoffsets[index] = last
|
||||||
|
raise "double index" if comb_indicies[dm0]
|
||||||
|
comb_indicies[dm0] = cumoffset
|
||||||
|
cumoffset += last - first + 1 + 2
|
||||||
|
end
|
||||||
|
|
||||||
|
offset = 0
|
||||||
|
comb2nd_indicies_sorted_keys.each do |dm1|
|
||||||
|
raise "double index" if comb_indicies[dm1]
|
||||||
|
comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
|
||||||
|
raise "too large comb index" if comb2nd_indicies[dm1] + offset > 0x4000
|
||||||
|
if comb2nd_indicies_nonbasic[dm1]
|
||||||
|
comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
|
||||||
|
offset += 1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
properties_indicies = {}
|
||||||
|
properties = []
|
||||||
|
chars.each do |char|
|
||||||
|
c_entry = char.c_entry(comb_indicies)
|
||||||
|
char.c_entry_index = properties_indicies[c_entry]
|
||||||
|
unless char.c_entry_index
|
||||||
|
properties_indicies[c_entry] = properties.length
|
||||||
|
char.c_entry_index = properties.length
|
||||||
|
properties << c_entry
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
stage1 = []
|
||||||
|
stage2 = []
|
||||||
|
for code in 0...0x110000
|
||||||
|
next unless code % 0x100 == 0
|
||||||
|
stage2_entry = []
|
||||||
|
for code2 in code...(code+0x100)
|
||||||
|
if char_hash[code2]
|
||||||
|
stage2_entry << (char_hash[code2].c_entry_index + 1)
|
||||||
|
else
|
||||||
|
stage2_entry << 0
|
||||||
|
end
|
||||||
|
end
|
||||||
|
old_index = stage2.index(stage2_entry)
|
||||||
|
if old_index
|
||||||
|
stage1 << (old_index * 0x100)
|
||||||
|
else
|
||||||
|
stage1 << (stage2.length * 0x100)
|
||||||
|
stage2 << stage2_entry
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
$stdout << "const utf8proc_uint16_t utf8proc_sequences[] = {\n "
|
||||||
|
i = 0
|
||||||
|
$int_array.each do |entry|
|
||||||
|
i += 1
|
||||||
|
if i == 8
|
||||||
|
i = 0
|
||||||
|
$stdout << "\n "
|
||||||
|
end
|
||||||
|
$stdout << entry << ", "
|
||||||
|
end
|
||||||
|
$stdout << "};\n\n"
|
||||||
|
|
||||||
|
$stdout << "const utf8proc_uint16_t utf8proc_stage1table[] = {\n "
|
||||||
|
i = 0
|
||||||
|
stage1.each do |entry|
|
||||||
|
i += 1
|
||||||
|
if i == 8
|
||||||
|
i = 0
|
||||||
|
$stdout << "\n "
|
||||||
|
end
|
||||||
|
$stdout << entry << ", "
|
||||||
|
end
|
||||||
|
$stdout << "};\n\n"
|
||||||
|
|
||||||
|
$stdout << "const utf8proc_uint16_t utf8proc_stage2table[] = {\n "
|
||||||
|
i = 0
|
||||||
|
stage2.flatten.each do |entry|
|
||||||
|
i += 1
|
||||||
|
if i == 8
|
||||||
|
i = 0
|
||||||
|
$stdout << "\n "
|
||||||
|
end
|
||||||
|
$stdout << entry << ", "
|
||||||
|
end
|
||||||
|
$stdout << "};\n\n"
|
||||||
|
|
||||||
|
$stdout << "const utf8proc_property_t utf8proc_properties[] = {\n"
|
||||||
|
$stdout << " {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
|
||||||
|
properties.each { |line|
|
||||||
|
$stdout << line
|
||||||
|
}
|
||||||
|
$stdout << "};\n\n"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
$stdout << "const utf8proc_uint16_t utf8proc_combinations[] = {\n "
|
||||||
|
i = 0
|
||||||
|
comb1st_indicies.keys.each_index do |a|
|
||||||
|
offset = 0
|
||||||
|
$stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
|
||||||
|
comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
|
||||||
|
break if offset > comb1st_indicies_lastoffsets[a]
|
||||||
|
if offset >= comb1st_indicies_firstoffsets[a]
|
||||||
|
i += 1
|
||||||
|
if i == 8
|
||||||
|
i = 0
|
||||||
|
$stdout << "\n "
|
||||||
|
end
|
||||||
|
v = comb_array[a][b] ? comb_array[a][b] : 0
|
||||||
|
$stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
|
||||||
|
$stdout << (v & 0xFFFF) << ", "
|
||||||
|
end
|
||||||
|
offset += 1
|
||||||
|
offset += 1 if comb2nd_indicies_nonbasic[dm1]
|
||||||
|
end
|
||||||
|
$stdout << "\n"
|
||||||
|
end
|
||||||
|
$stdout << "};\n\n"
|
||||||
|
|
||||||
1624
3rdparty/utf8proc/data_generator.rb
vendored
1624
3rdparty/utf8proc/data_generator.rb
vendored
File diff suppressed because it is too large
Load diff
3
3rdparty/utf8proc/lump.txt → 3rdparty/utf8proc/lump.md
vendored
Normal file → Executable file
3
3rdparty/utf8proc/lump.txt → 3rdparty/utf8proc/lump.md
vendored
Normal file → Executable file
|
|
@ -1,3 +1,4 @@
|
||||||
|
```
|
||||||
U+0020 <-- all space characters (general category Zs)
|
U+0020 <-- all space characters (general category Zs)
|
||||||
U+0027 ' <-- left/right single quotation mark U+2018..2019,
|
U+0027 ' <-- left/right single quotation mark U+2018..2019,
|
||||||
modifier letter apostrophe U+02BC,
|
modifier letter apostrophe U+02BC,
|
||||||
|
|
@ -23,4 +24,4 @@ U+005F _ <-- all connector characters (general category Pc),
|
||||||
U+0060 ` <-- modifier letter grave accent U+02CB
|
U+0060 ` <-- modifier letter grave accent U+02CB
|
||||||
U+007C | <-- divides U+2223
|
U+007C | <-- divides U+2223
|
||||||
U+007E ~ <-- tilde operator U+223C
|
U+007E ~ <-- tilde operator U+223C
|
||||||
|
```
|
||||||
10
3rdparty/utf8proc/pgsql/Makefile
vendored
10
3rdparty/utf8proc/pgsql/Makefile
vendored
|
|
@ -1,10 +0,0 @@
|
||||||
utf8proc_pgsql.so: utf8proc_pgsql.o
|
|
||||||
ld -shared -o utf8proc_pgsql.so utf8proc_pgsql.o
|
|
||||||
|
|
||||||
utf8proc_pgsql.o: utf8proc_pgsql.c
|
|
||||||
gcc -Wall -fpic -c -I`pg_config --includedir-server` \
|
|
||||||
-o utf8proc_pgsql.o utf8proc_pgsql.c
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm -f *.o *.so
|
|
||||||
|
|
||||||
6
3rdparty/utf8proc/pgsql/utf8proc.sql
vendored
6
3rdparty/utf8proc/pgsql/utf8proc.sql
vendored
|
|
@ -1,6 +0,0 @@
|
||||||
CREATE OR REPLACE FUNCTION unifold (text) RETURNS text
|
|
||||||
LANGUAGE 'C' IMMUTABLE STRICT AS '$libdir/utf8proc_pgsql.so',
|
|
||||||
'utf8proc_pgsql_unifold';
|
|
||||||
CREATE OR REPLACE FUNCTION unistrip (text) RETURNS text
|
|
||||||
LANGUAGE 'C' IMMUTABLE STRICT AS '$libdir/utf8proc_pgsql.so',
|
|
||||||
'utf8proc_pgsql_unistrip';
|
|
||||||
139
3rdparty/utf8proc/pgsql/utf8proc_pgsql.c
vendored
139
3rdparty/utf8proc/pgsql/utf8proc_pgsql.c
vendored
|
|
@ -1,139 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (c) Public Software Group e. V., Berlin, Germany
|
|
||||||
*
|
|
||||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
||||||
* copy of this software and associated documentation files (the "Software"),
|
|
||||||
* to deal in the Software without restriction, including without limitation
|
|
||||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
||||||
* and/or sell copies of the Software, and to permit persons to whom the
|
|
||||||
* Software is furnished to do so, subject to the following conditions:
|
|
||||||
*
|
|
||||||
* The above copyright notice and this permission notice shall be included in
|
|
||||||
* all copies or substantial portions of the Software.
|
|
||||||
*
|
|
||||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
||||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
||||||
* DEALINGS IN THE SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* File name: pgsql/utf8proc_pgsql.c
|
|
||||||
*
|
|
||||||
* Description:
|
|
||||||
* PostgreSQL extension to provide two functions 'unifold' and 'unistrip',
|
|
||||||
* which can be used to case-fold and normalize index fields and
|
|
||||||
* optionally strip marks (e.g. accents) from strings.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
#include "../utf8proc.c"
|
|
||||||
|
|
||||||
#include <postgres.h>
|
|
||||||
#include <utils/elog.h>
|
|
||||||
#include <fmgr.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <utils/builtins.h>
|
|
||||||
|
|
||||||
#ifdef PG_MODULE_MAGIC
|
|
||||||
PG_MODULE_MAGIC;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#define UTF8PROC_PGSQL_FOLD_OPTS ( UTF8PROC_REJECTNA | UTF8PROC_COMPAT | \
|
|
||||||
UTF8PROC_COMPOSE | UTF8PROC_STABLE | UTF8PROC_IGNORE | UTF8PROC_STRIPCC | \
|
|
||||||
UTF8PROC_NLF2LF | UTF8PROC_CASEFOLD | UTF8PROC_LUMP )
|
|
||||||
#define UTF8PROC_PGSQL_STRIP_OPTS ( UTF8PROC_REJECTNA | UTF8PROC_COMPAT | \
|
|
||||||
UTF8PROC_COMPOSE | UTF8PROC_STABLE | UTF8PROC_IGNORE | UTF8PROC_STRIPCC | \
|
|
||||||
UTF8PROC_NLF2LF | UTF8PROC_CASEFOLD | UTF8PROC_LUMP | UTF8PROC_STRIPMARK )
|
|
||||||
|
|
||||||
ssize_t utf8proc_pgsql_utf8map(
|
|
||||||
text *input_string, text **output_string_ptr, int options
|
|
||||||
) {
|
|
||||||
ssize_t result;
|
|
||||||
text *output_string;
|
|
||||||
result = utf8proc_decompose(
|
|
||||||
VARDATA(input_string), VARSIZE(input_string) - VARHDRSZ,
|
|
||||||
NULL, 0, options
|
|
||||||
);
|
|
||||||
if (result < 0) return result;
|
|
||||||
if (result > (SIZE_MAX-1-VARHDRSZ)/sizeof(int32_t))
|
|
||||||
return UTF8PROC_ERROR_OVERFLOW;
|
|
||||||
/* reserve one extra byte for termination */
|
|
||||||
*output_string_ptr = palloc(result * sizeof(int32_t) + 1 + VARHDRSZ);
|
|
||||||
output_string = *output_string_ptr;
|
|
||||||
if (!output_string) return UTF8PROC_ERROR_NOMEM;
|
|
||||||
result = utf8proc_decompose(
|
|
||||||
VARDATA(input_string), VARSIZE(input_string) - VARHDRSZ,
|
|
||||||
(int32_t *)VARDATA(output_string), result, options
|
|
||||||
);
|
|
||||||
if (result < 0) return result;
|
|
||||||
result = utf8proc_reencode(
|
|
||||||
(int32_t *)VARDATA(output_string), result, options
|
|
||||||
);
|
|
||||||
if (result >= 0) SET_VARSIZE(output_string, result + VARHDRSZ);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
void utf8proc_pgsql_utf8map_errchk(ssize_t result, text *output_string) {
|
|
||||||
if (result < 0) {
|
|
||||||
int sqlerrcode;
|
|
||||||
if (output_string) pfree(output_string);
|
|
||||||
switch(result) {
|
|
||||||
case UTF8PROC_ERROR_NOMEM:
|
|
||||||
sqlerrcode = ERRCODE_OUT_OF_MEMORY; break;
|
|
||||||
case UTF8PROC_ERROR_OVERFLOW:
|
|
||||||
sqlerrcode = ERRCODE_PROGRAM_LIMIT_EXCEEDED; break;
|
|
||||||
case UTF8PROC_ERROR_INVALIDUTF8:
|
|
||||||
case UTF8PROC_ERROR_NOTASSIGNED:
|
|
||||||
return;
|
|
||||||
default:
|
|
||||||
sqlerrcode = ERRCODE_INTERNAL_ERROR;
|
|
||||||
}
|
|
||||||
ereport(ERROR, (
|
|
||||||
errcode(sqlerrcode),
|
|
||||||
errmsg("%s", utf8proc_errmsg(result))
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
PG_FUNCTION_INFO_V1(utf8proc_pgsql_unifold);
|
|
||||||
Datum utf8proc_pgsql_unifold(PG_FUNCTION_ARGS) {
|
|
||||||
text *input_string;
|
|
||||||
text *output_string = NULL;
|
|
||||||
ssize_t result;
|
|
||||||
input_string = PG_GETARG_TEXT_P(0);
|
|
||||||
result = utf8proc_pgsql_utf8map(
|
|
||||||
input_string, &output_string, UTF8PROC_PGSQL_FOLD_OPTS
|
|
||||||
);
|
|
||||||
PG_FREE_IF_COPY(input_string, 0);
|
|
||||||
utf8proc_pgsql_utf8map_errchk(result, output_string);
|
|
||||||
if (result >= 0) {
|
|
||||||
PG_RETURN_TEXT_P(output_string);
|
|
||||||
} else {
|
|
||||||
PG_RETURN_NULL();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
PG_FUNCTION_INFO_V1(utf8proc_pgsql_unistrip);
|
|
||||||
Datum utf8proc_pgsql_unistrip(PG_FUNCTION_ARGS) {
|
|
||||||
text *input_string;
|
|
||||||
text *output_string = NULL;
|
|
||||||
ssize_t result;
|
|
||||||
input_string = PG_GETARG_TEXT_P(0);
|
|
||||||
result = utf8proc_pgsql_utf8map(
|
|
||||||
input_string, &output_string, UTF8PROC_PGSQL_STRIP_OPTS
|
|
||||||
);
|
|
||||||
PG_FREE_IF_COPY(input_string, 0);
|
|
||||||
utf8proc_pgsql_utf8map_errchk(result, output_string);
|
|
||||||
if (result >= 0) {
|
|
||||||
PG_RETURN_TEXT_P(output_string);
|
|
||||||
} else {
|
|
||||||
PG_RETURN_NULL();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
2
3rdparty/utf8proc/ruby/extconf.rb
vendored
2
3rdparty/utf8proc/ruby/extconf.rb
vendored
|
|
@ -1,2 +0,0 @@
|
||||||
require 'mkmf'
|
|
||||||
create_makefile("utf8proc_native")
|
|
||||||
64
3rdparty/utf8proc/ruby/gem/LICENSE
vendored
64
3rdparty/utf8proc/ruby/gem/LICENSE
vendored
|
|
@ -1,64 +0,0 @@
|
||||||
|
|
||||||
Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a
|
|
||||||
copy of this software and associated documentation files (the "Software"),
|
|
||||||
to deal in the Software without restriction, including without limitation
|
|
||||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
||||||
and/or sell copies of the Software, and to permit persons to whom the
|
|
||||||
Software is furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in
|
|
||||||
all copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
||||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
||||||
DEALINGS IN THE SOFTWARE.
|
|
||||||
|
|
||||||
|
|
||||||
This software distribution contains derived data from a modified version of
|
|
||||||
the Unicode data files. The following license applies to that data:
|
|
||||||
|
|
||||||
COPYRIGHT AND PERMISSION NOTICE
|
|
||||||
|
|
||||||
Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
|
|
||||||
under the Terms of Use in http://www.unicode.org/copyright.html.
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a
|
|
||||||
copy of the Unicode data files and any associated documentation (the "Data
|
|
||||||
Files") or Unicode software and any associated documentation (the
|
|
||||||
"Software") to deal in the Data Files or Software without restriction,
|
|
||||||
including without limitation the rights to use, copy, modify, merge,
|
|
||||||
publish, distribute, and/or sell copies of the Data Files or Software, and
|
|
||||||
to permit persons to whom the Data Files or Software are furnished to do
|
|
||||||
so, provided that (a) the above copyright notice(s) and this permission
|
|
||||||
notice appear with all copies of the Data Files or Software, (b) both the
|
|
||||||
above copyright notice(s) and this permission notice appear in associated
|
|
||||||
documentation, and (c) there is clear notice in each modified Data File or
|
|
||||||
in the Software as well as in the documentation associated with the Data
|
|
||||||
File(s) or Software that the data or software has been modified.
|
|
||||||
|
|
||||||
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
|
||||||
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
||||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
|
|
||||||
THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
|
|
||||||
INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
|
|
||||||
CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
|
|
||||||
USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
|
|
||||||
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
|
|
||||||
PERFORMANCE OF THE DATA FILES OR SOFTWARE.
|
|
||||||
|
|
||||||
Except as contained in this notice, the name of a copyright holder shall
|
|
||||||
not be used in advertising or otherwise to promote the sale, use or other
|
|
||||||
dealings in these Data Files or Software without prior written
|
|
||||||
authorization of the copyright holder.
|
|
||||||
|
|
||||||
|
|
||||||
Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
|
|
||||||
registered in some jurisdictions. All other trademarks and registered
|
|
||||||
trademarks mentioned herein are the property of their respective owners.
|
|
||||||
|
|
||||||
12
3rdparty/utf8proc/ruby/gem/utf8proc.gemspec
vendored
12
3rdparty/utf8proc/ruby/gem/utf8proc.gemspec
vendored
|
|
@ -1,12 +0,0 @@
|
||||||
require 'rubygems'
|
|
||||||
SPEC = Gem::Specification.new do |s|
|
|
||||||
s.name = 'utf8proc'
|
|
||||||
s.version = '1.1.5'
|
|
||||||
s.author = 'Public Software Group e. V., Berlin, Germany'
|
|
||||||
s.homepage = 'http://www.public-software-group.org/utf8proc'
|
|
||||||
s.summary = 'UTF-8 Unicode string processing'
|
|
||||||
s.files = ['LICENSE', 'lib/utf8proc.rb', 'ext/utf8proc_native.c']
|
|
||||||
s.require_path = 'lib/'
|
|
||||||
s.extensions = ['ext/extconf.rb']
|
|
||||||
s.has_rdoc = false
|
|
||||||
end
|
|
||||||
98
3rdparty/utf8proc/ruby/utf8proc.rb
vendored
98
3rdparty/utf8proc/ruby/utf8proc.rb
vendored
|
|
@ -1,98 +0,0 @@
|
||||||
# Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
|
||||||
#
|
|
||||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
|
||||||
# copy of this software and associated documentation files (the "Software"),
|
|
||||||
# to deal in the Software without restriction, including without limitation
|
|
||||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
||||||
# and/or sell copies of the Software, and to permit persons to whom the
|
|
||||||
# Software is furnished to do so, subject to the following conditions:
|
|
||||||
#
|
|
||||||
# The above copyright notice and this permission notice shall be included in
|
|
||||||
# all copies or substantial portions of the Software.
|
|
||||||
#
|
|
||||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
||||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
||||||
# DEALINGS IN THE SOFTWARE.
|
|
||||||
|
|
||||||
|
|
||||||
#
|
|
||||||
# File name: ruby/utf8proc.rb
|
|
||||||
#
|
|
||||||
# Description:
|
|
||||||
# Part of the ruby wrapper for libutf8proc, which is written in ruby.
|
|
||||||
#
|
|
||||||
|
|
||||||
|
|
||||||
require 'utf8proc_native'
|
|
||||||
|
|
||||||
|
|
||||||
module Utf8Proc
|
|
||||||
|
|
||||||
SpecialChars = {
|
|
||||||
:HT => "\x09",
|
|
||||||
:LF => "\x0A",
|
|
||||||
:VT => "\x0B",
|
|
||||||
:FF => "\x0C",
|
|
||||||
:CR => "\x0D",
|
|
||||||
:FS => "\x1C",
|
|
||||||
:GS => "\x1D",
|
|
||||||
:RS => "\x1E",
|
|
||||||
:US => "\x1F",
|
|
||||||
:LS => "\xE2\x80\xA8",
|
|
||||||
:PS => "\xE2\x80\xA9",
|
|
||||||
}
|
|
||||||
|
|
||||||
module StringExtensions
|
|
||||||
def utf8map(*option_array)
|
|
||||||
options = 0
|
|
||||||
option_array.each do |option|
|
|
||||||
flag = Utf8Proc::Options[option]
|
|
||||||
raise ArgumentError, "Unknown argument given to String#utf8map." unless
|
|
||||||
flag
|
|
||||||
options |= flag
|
|
||||||
end
|
|
||||||
return Utf8Proc::utf8map(self, options)
|
|
||||||
end
|
|
||||||
def utf8map!(*option_array)
|
|
||||||
self.replace(self.utf8map(*option_array))
|
|
||||||
end
|
|
||||||
def utf8nfd; utf8map( :stable, :decompose); end
|
|
||||||
def utf8nfd!; utf8map!(:stable, :decompose); end
|
|
||||||
def utf8nfc; utf8map( :stable, :compose); end
|
|
||||||
def utf8nfc!; utf8map!(:stable, :compose); end
|
|
||||||
def utf8nfkd; utf8map( :stable, :decompose, :compat); end
|
|
||||||
def utf8nfkd!; utf8map!(:stable, :decompose, :compat); end
|
|
||||||
def utf8nfkc; utf8map( :stable, :compose, :compat); end
|
|
||||||
def utf8nfkc!; utf8map!(:stable, :compose, :compat); end
|
|
||||||
def utf8chars
|
|
||||||
result = self.utf8map(:charbound).split("\377")
|
|
||||||
result.shift if result.first == ""
|
|
||||||
result
|
|
||||||
end
|
|
||||||
def char_ary
|
|
||||||
# depecated, use String#utf8chars instead
|
|
||||||
utf8chars
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
module IntegerExtensions
|
|
||||||
def utf8
|
|
||||||
return Utf8Proc::utf8char(self)
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
class String
|
|
||||||
include(Utf8Proc::StringExtensions)
|
|
||||||
end
|
|
||||||
|
|
||||||
class Integer
|
|
||||||
include(Utf8Proc::IntegerExtensions)
|
|
||||||
end
|
|
||||||
|
|
||||||
160
3rdparty/utf8proc/ruby/utf8proc_native.c
vendored
160
3rdparty/utf8proc/ruby/utf8proc_native.c
vendored
|
|
@ -1,160 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
|
||||||
*
|
|
||||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
||||||
* copy of this software and associated documentation files (the "Software"),
|
|
||||||
* to deal in the Software without restriction, including without limitation
|
|
||||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
||||||
* and/or sell copies of the Software, and to permit persons to whom the
|
|
||||||
* Software is furnished to do so, subject to the following conditions:
|
|
||||||
*
|
|
||||||
* The above copyright notice and this permission notice shall be included in
|
|
||||||
* all copies or substantial portions of the Software.
|
|
||||||
*
|
|
||||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
||||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
||||||
* DEALINGS IN THE SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
/*
|
|
||||||
* File name: ruby/utf8proc_native.c
|
|
||||||
*
|
|
||||||
* Description:
|
|
||||||
* Native part of the ruby wrapper for libutf8proc.
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
|
||||||
#include "../utf8proc.c"
|
|
||||||
#include "ruby.h"
|
|
||||||
|
|
||||||
#ifndef RSTRING_PTR
|
|
||||||
#define RSTRING_PTR(s) (RSTRING(s)->ptr)
|
|
||||||
#endif
|
|
||||||
#ifndef RSTRING_LEN
|
|
||||||
#define RSTRING_LEN(s) (RSTRING(s)->len)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
typedef struct utf8proc_ruby_mapenv_struct {
|
|
||||||
int32_t *buffer;
|
|
||||||
} utf8proc_ruby_mapenv_t;
|
|
||||||
|
|
||||||
void utf8proc_ruby_mapenv_free(utf8proc_ruby_mapenv_t *env) {
|
|
||||||
free(env->buffer);
|
|
||||||
free(env);
|
|
||||||
}
|
|
||||||
|
|
||||||
VALUE utf8proc_ruby_module;
|
|
||||||
VALUE utf8proc_ruby_options;
|
|
||||||
VALUE utf8proc_ruby_eUnicodeError;
|
|
||||||
VALUE utf8proc_ruby_eInvalidUtf8Error;
|
|
||||||
VALUE utf8proc_ruby_eCodeNotAssignedError;
|
|
||||||
|
|
||||||
VALUE utf8proc_ruby_map_error(ssize_t result) {
|
|
||||||
VALUE excpt_class;
|
|
||||||
switch (result) {
|
|
||||||
case UTF8PROC_ERROR_NOMEM:
|
|
||||||
excpt_class = rb_eNoMemError; break;
|
|
||||||
case UTF8PROC_ERROR_OVERFLOW:
|
|
||||||
case UTF8PROC_ERROR_INVALIDOPTS:
|
|
||||||
excpt_class = rb_eArgError; break;
|
|
||||||
case UTF8PROC_ERROR_INVALIDUTF8:
|
|
||||||
excpt_class = utf8proc_ruby_eInvalidUtf8Error; break;
|
|
||||||
case UTF8PROC_ERROR_NOTASSIGNED:
|
|
||||||
excpt_class = utf8proc_ruby_eCodeNotAssignedError; break;
|
|
||||||
default:
|
|
||||||
excpt_class = rb_eRuntimeError;
|
|
||||||
}
|
|
||||||
rb_raise(excpt_class, "%s", utf8proc_errmsg(result));
|
|
||||||
return Qnil;
|
|
||||||
}
|
|
||||||
|
|
||||||
VALUE utf8proc_ruby_map(VALUE self, VALUE str_param, VALUE options_param) {
|
|
||||||
VALUE str;
|
|
||||||
int options;
|
|
||||||
VALUE env_obj;
|
|
||||||
utf8proc_ruby_mapenv_t *env;
|
|
||||||
ssize_t result;
|
|
||||||
VALUE retval;
|
|
||||||
str = StringValue(str_param);
|
|
||||||
options = NUM2INT(options_param) & ~UTF8PROC_NULLTERM;
|
|
||||||
env_obj = Data_Make_Struct(rb_cObject, utf8proc_ruby_mapenv_t, NULL,
|
|
||||||
utf8proc_ruby_mapenv_free, env);
|
|
||||||
result = utf8proc_decompose(RSTRING_PTR(str), RSTRING_LEN(str),
|
|
||||||
NULL, 0, options);
|
|
||||||
if (result < 0) {
|
|
||||||
utf8proc_ruby_map_error(result);
|
|
||||||
return Qnil; /* needed to prevent problems with optimization */
|
|
||||||
}
|
|
||||||
env->buffer = ALLOC_N(int32_t, result+1);
|
|
||||||
result = utf8proc_decompose(RSTRING_PTR(str), RSTRING_LEN(str),
|
|
||||||
env->buffer, result, options);
|
|
||||||
if (result < 0) {
|
|
||||||
free(env->buffer);
|
|
||||||
env->buffer = 0;
|
|
||||||
utf8proc_ruby_map_error(result);
|
|
||||||
return Qnil; /* needed to prevent problems with optimization */
|
|
||||||
}
|
|
||||||
result = utf8proc_reencode(env->buffer, result, options);
|
|
||||||
if (result < 0) {
|
|
||||||
free(env->buffer);
|
|
||||||
env->buffer = 0;
|
|
||||||
utf8proc_ruby_map_error(result);
|
|
||||||
return Qnil; /* needed to prevent problems with optimization */
|
|
||||||
}
|
|
||||||
retval = rb_str_new((char *)env->buffer, result);
|
|
||||||
free(env->buffer);
|
|
||||||
env->buffer = 0;
|
|
||||||
return retval;
|
|
||||||
}
|
|
||||||
|
|
||||||
static VALUE utf8proc_ruby_char(VALUE self, VALUE code_param) {
|
|
||||||
char buffer[4];
|
|
||||||
ssize_t result;
|
|
||||||
int uc;
|
|
||||||
uc = NUM2INT(code_param);
|
|
||||||
if (!utf8proc_codepoint_valid(uc))
|
|
||||||
rb_raise(rb_eArgError, "Invalid Unicode code point");
|
|
||||||
result = utf8proc_encode_char(uc, buffer);
|
|
||||||
return rb_str_new(buffer, result);
|
|
||||||
}
|
|
||||||
|
|
||||||
#define register_utf8proc_option(sym, field) \
|
|
||||||
rb_hash_aset(utf8proc_ruby_options, ID2SYM(rb_intern(sym)), INT2FIX(field))
|
|
||||||
|
|
||||||
void Init_utf8proc_native() {
|
|
||||||
utf8proc_ruby_module = rb_define_module("Utf8Proc");
|
|
||||||
rb_define_module_function(utf8proc_ruby_module, "utf8map",
|
|
||||||
utf8proc_ruby_map, 2);
|
|
||||||
rb_define_module_function(utf8proc_ruby_module, "utf8char",
|
|
||||||
utf8proc_ruby_char, 1);
|
|
||||||
utf8proc_ruby_eUnicodeError = rb_define_class_under(utf8proc_ruby_module,
|
|
||||||
"UnicodeError", rb_eStandardError);
|
|
||||||
utf8proc_ruby_eInvalidUtf8Error = rb_define_class_under(
|
|
||||||
utf8proc_ruby_module, "InvalidUtf8Error", utf8proc_ruby_eUnicodeError);
|
|
||||||
utf8proc_ruby_eCodeNotAssignedError = rb_define_class_under(
|
|
||||||
utf8proc_ruby_module, "CodeNotAssignedError",
|
|
||||||
utf8proc_ruby_eUnicodeError);
|
|
||||||
utf8proc_ruby_options = rb_hash_new();
|
|
||||||
register_utf8proc_option("stable", UTF8PROC_STABLE);
|
|
||||||
register_utf8proc_option("compat", UTF8PROC_COMPAT);
|
|
||||||
register_utf8proc_option("compose", UTF8PROC_COMPOSE);
|
|
||||||
register_utf8proc_option("decompose", UTF8PROC_DECOMPOSE);
|
|
||||||
register_utf8proc_option("ignore", UTF8PROC_IGNORE);
|
|
||||||
register_utf8proc_option("rejectna", UTF8PROC_REJECTNA);
|
|
||||||
register_utf8proc_option("nlf2ls", UTF8PROC_NLF2LS);
|
|
||||||
register_utf8proc_option("nlf2ps", UTF8PROC_NLF2PS);
|
|
||||||
register_utf8proc_option("nlf2lf", UTF8PROC_NLF2LF);
|
|
||||||
register_utf8proc_option("stripcc", UTF8PROC_STRIPCC);
|
|
||||||
register_utf8proc_option("casefold", UTF8PROC_CASEFOLD);
|
|
||||||
register_utf8proc_option("charbound", UTF8PROC_CHARBOUND);
|
|
||||||
register_utf8proc_option("lump", UTF8PROC_LUMP);
|
|
||||||
register_utf8proc_option("stripmark", UTF8PROC_STRIPMARK);
|
|
||||||
OBJ_FREEZE(utf8proc_ruby_options);
|
|
||||||
rb_define_const(utf8proc_ruby_module, "Options", utf8proc_ruby_options);
|
|
||||||
}
|
|
||||||
|
|
||||||
50
3rdparty/utf8proc/test/case.c
vendored
Executable file
50
3rdparty/utf8proc/test/case.c
vendored
Executable file
|
|
@ -0,0 +1,50 @@
|
||||||
|
#include "tests.h"
|
||||||
|
#include <wctype.h>
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int error = 0, better = 0;
|
||||||
|
utf8proc_int32_t c;
|
||||||
|
|
||||||
|
(void) argc; /* unused */
|
||||||
|
(void) argv; /* unused */
|
||||||
|
|
||||||
|
/* some simple sanity tests of the character widths */
|
||||||
|
for (c = 0; c <= 0x110000; ++c) {
|
||||||
|
utf8proc_int32_t l = utf8proc_tolower(c);
|
||||||
|
utf8proc_int32_t u = utf8proc_toupper(c);
|
||||||
|
|
||||||
|
check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
|
||||||
|
check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
|
||||||
|
|
||||||
|
if (sizeof(wint_t) > 2 || c < (1<<16)) {
|
||||||
|
wint_t l0 = towlower(c), u0 = towupper(c);
|
||||||
|
|
||||||
|
/* OS unicode tables may be out of date. But if they
|
||||||
|
do have a lower/uppercase mapping, hopefully it
|
||||||
|
is correct? */
|
||||||
|
if (l0 != c && l0 != l) {
|
||||||
|
fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
|
||||||
|
l, c, l0);
|
||||||
|
++error;
|
||||||
|
}
|
||||||
|
else if (l0 != l) { /* often true for out-of-date OS unicode */
|
||||||
|
++better;
|
||||||
|
/* printf("%x != towlower(%x) == %x\n", l, c, l0); */
|
||||||
|
}
|
||||||
|
if (u0 != c && u0 != u) {
|
||||||
|
fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
|
||||||
|
u, c, u0);
|
||||||
|
++error;
|
||||||
|
}
|
||||||
|
else if (u0 != u) { /* often true for out-of-date OS unicode */
|
||||||
|
++better;
|
||||||
|
/* printf("%x != towupper(%x) == %x\n", u, c, u0); */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(!error, "utf8proc case conversion FAILED %d tests.", error);
|
||||||
|
printf("More up-to-date than OS unicode tables for %d tests.\n", better);
|
||||||
|
printf("utf8proc case conversion tests SUCCEEDED.\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
71
3rdparty/utf8proc/test/charwidth.c
vendored
Executable file
71
3rdparty/utf8proc/test/charwidth.c
vendored
Executable file
|
|
@ -0,0 +1,71 @@
|
||||||
|
#include "tests.h"
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <wchar.h>
|
||||||
|
|
||||||
|
static int my_isprint(int c) {
|
||||||
|
int cat = utf8proc_get_property(c)->category;
|
||||||
|
return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
|
||||||
|
(c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int c, error = 0, updates = 0;
|
||||||
|
|
||||||
|
(void) argc; /* unused */
|
||||||
|
(void) argv; /* unused */
|
||||||
|
|
||||||
|
/* some simple sanity tests of the character widths */
|
||||||
|
for (c = 0; c <= 0x110000; ++c) {
|
||||||
|
int cat = utf8proc_get_property(c)->category;
|
||||||
|
int w = utf8proc_charwidth(c);
|
||||||
|
if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
|
||||||
|
w > 0) {
|
||||||
|
fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
|
||||||
|
error = 1;
|
||||||
|
}
|
||||||
|
if (w == 0 &&
|
||||||
|
((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
|
||||||
|
(cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
|
||||||
|
(cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
|
||||||
|
fprintf(stderr, "zero width for symbol-like char %x\n", c);
|
||||||
|
error = 1;
|
||||||
|
}
|
||||||
|
if (c <= 127 && ((!isprint(c) && w > 0) ||
|
||||||
|
(isprint(c) && wcwidth(c) != w))) {
|
||||||
|
fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
|
||||||
|
wcwidth(c), w,
|
||||||
|
isprint(c) ? "printable" : "non-printable", c);
|
||||||
|
error = 1;
|
||||||
|
}
|
||||||
|
if (!my_isprint(c) && w > 0) {
|
||||||
|
fprintf(stderr, "non-printing %x had width %d\n", c, w);
|
||||||
|
error = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(!error, "utf8proc_charwidth FAILED tests.");
|
||||||
|
|
||||||
|
/* print some other information by compariing with system wcwidth */
|
||||||
|
printf("Mismatches with system wcwidth (not necessarily errors):\n");
|
||||||
|
for (c = 0; c <= 0x110000; ++c) {
|
||||||
|
int w = utf8proc_charwidth(c);
|
||||||
|
int wc = wcwidth(c);
|
||||||
|
if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
|
||||||
|
/* lots of these errors for out-of-date system unicode tables */
|
||||||
|
if (wc == -1 && my_isprint(c) && w > 0) {
|
||||||
|
updates += 1;
|
||||||
|
#if 0
|
||||||
|
printf(" wcwidth(%x) = -1 for printable char\n", c);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
if (wc == -1 && !my_isprint(c) && w > 0)
|
||||||
|
printf(" wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
|
||||||
|
if (wc >= 0 && wc != w)
|
||||||
|
printf(" wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
|
||||||
|
}
|
||||||
|
printf(" ... (positive widths for %d chars unknown to wcwidth) ...\n",
|
||||||
|
updates);
|
||||||
|
printf("Character-width tests SUCCEEDED.\n");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
74
3rdparty/utf8proc/test/graphemetest.c
vendored
Executable file
74
3rdparty/utf8proc/test/graphemetest.c
vendored
Executable file
|
|
@ -0,0 +1,74 @@
|
||||||
|
#include "tests.h"
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
char *buf = NULL;
|
||||||
|
size_t bufsize = 0;
|
||||||
|
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
|
||||||
|
utf8proc_uint8_t src[1024];
|
||||||
|
int len;
|
||||||
|
|
||||||
|
check(f != NULL, "error opening GraphemeBreakTest.txt");
|
||||||
|
while (getline(&buf, &bufsize, f) > 0) {
|
||||||
|
size_t bi = 0, si = 0;
|
||||||
|
lineno += 1;
|
||||||
|
|
||||||
|
if (lineno % 100 == 0)
|
||||||
|
printf("checking line %zd...\n", lineno);
|
||||||
|
|
||||||
|
if (buf[0] == '#') continue;
|
||||||
|
|
||||||
|
while (buf[bi]) {
|
||||||
|
bi = skipspaces(buf, bi);
|
||||||
|
if (buf[bi] == '/') { /* grapheme break */
|
||||||
|
src[si++] = '/';
|
||||||
|
bi++;
|
||||||
|
}
|
||||||
|
else if (buf[bi] == '+') { /* no break */
|
||||||
|
bi++;
|
||||||
|
}
|
||||||
|
else if (buf[bi] == '#') { /* start of comments */
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
else { /* hex-encoded codepoint */
|
||||||
|
len = encode((char*) (src + si), buf + bi) - 1;
|
||||||
|
while (src[si]) ++si; /* advance to NUL termination */
|
||||||
|
bi += len;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (si && src[si-1] == '/')
|
||||||
|
--si; /* no break after final grapheme */
|
||||||
|
src[si] = 0; /* NUL-terminate */
|
||||||
|
|
||||||
|
if (si) {
|
||||||
|
utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
|
||||||
|
size_t i = 0, j = 0;
|
||||||
|
utf8proc_ssize_t glen;
|
||||||
|
utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
|
||||||
|
while (i < si) {
|
||||||
|
if (src[i] != '/')
|
||||||
|
utf8[j++] = src[i++];
|
||||||
|
else
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
|
||||||
|
if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
|
||||||
|
/* the test file contains surrogate codepoints, which are only for UTF-16 */
|
||||||
|
printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
check(glen >= 0, "utf8proc_map error = %s",
|
||||||
|
utf8proc_errmsg(glen));
|
||||||
|
for (i = 0; i <= glen; ++i)
|
||||||
|
if (g[i] == 0xff)
|
||||||
|
g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
|
||||||
|
check(!strcmp((char*)g, (char*)src),
|
||||||
|
"grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
|
||||||
|
}
|
||||||
|
free(g);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
printf("Passed tests after %zd lines!\n", lineno);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
166
3rdparty/utf8proc/test/iterate.c
vendored
Executable file
166
3rdparty/utf8proc/test/iterate.c
vendored
Executable file
|
|
@ -0,0 +1,166 @@
|
||||||
|
#include "tests.h"
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <wchar.h>
|
||||||
|
|
||||||
|
static int tests;
|
||||||
|
static int error;
|
||||||
|
|
||||||
|
#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
|
||||||
|
#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
|
||||||
|
|
||||||
|
static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
|
||||||
|
{
|
||||||
|
utf8proc_int32_t out[16];
|
||||||
|
utf8proc_ssize_t ret;
|
||||||
|
|
||||||
|
/* Make a copy to ensure that memory is left uninitialized after "len"
|
||||||
|
* bytes. This way, Valgrind can detect overreads.
|
||||||
|
*/
|
||||||
|
unsigned char tmp[16];
|
||||||
|
memcpy(tmp, buf, len);
|
||||||
|
|
||||||
|
tests++;
|
||||||
|
if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
|
||||||
|
fprintf(stderr, "Failed (%d):", line);
|
||||||
|
for (int i = 0; i < len ; i++) {
|
||||||
|
fprintf(stderr, " 0x%02x", tmp[i]);
|
||||||
|
}
|
||||||
|
fprintf(stderr, " -> %zd\n", ret);
|
||||||
|
error++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
uint32_t byt;
|
||||||
|
unsigned char buf[16];
|
||||||
|
|
||||||
|
tests = error = 0;
|
||||||
|
|
||||||
|
// Check valid sequences that were considered valid erroneously before
|
||||||
|
buf[0] = 0xef;
|
||||||
|
buf[1] = 0xb7;
|
||||||
|
for (byt = 0x90; byt < 0xa0; byt++) {
|
||||||
|
CHECKVALID(2, byt, 3);
|
||||||
|
}
|
||||||
|
// Check 0xfffe and 0xffff
|
||||||
|
buf[1] = 0xbf;
|
||||||
|
CHECKVALID(2, 0xbe, 3);
|
||||||
|
CHECKVALID(2, 0xbf, 3);
|
||||||
|
// Check 0x??fffe & 0x??ffff
|
||||||
|
for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
|
||||||
|
buf[0] = 0xf0 | (byt >> 18);
|
||||||
|
buf[1] = 0x80 | ((byt >> 12) & 0x3f);
|
||||||
|
CHECKVALID(3, 0xbe, 4);
|
||||||
|
CHECKVALID(3, 0xbf, 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Continuation byte not after lead
|
||||||
|
for (byt = 0x80; byt < 0xc0; byt++) {
|
||||||
|
CHECKINVALID(0, byt, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Continuation byte not after lead
|
||||||
|
for (byt = 0x80; byt < 0xc0; byt++) {
|
||||||
|
CHECKINVALID(0, byt, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test lead bytes
|
||||||
|
for (byt = 0xc0; byt <= 0xff; byt++) {
|
||||||
|
// Single lead byte at end of string
|
||||||
|
CHECKINVALID(0, byt, 1);
|
||||||
|
// Lead followed by non-continuation character < 0x80
|
||||||
|
CHECKINVALID(1, 65, 2);
|
||||||
|
// Lead followed by non-continuation character > 0xbf
|
||||||
|
CHECKINVALID(1, 0xc0, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test overlong 2-byte
|
||||||
|
buf[0] = 0xc0;
|
||||||
|
for (byt = 0x81; byt <= 0xbf; byt++) {
|
||||||
|
CHECKINVALID(1, byt, 2);
|
||||||
|
}
|
||||||
|
buf[0] = 0xc1;
|
||||||
|
for (byt = 0x80; byt <= 0xbf; byt++) {
|
||||||
|
CHECKINVALID(1, byt, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test overlong 3-byte
|
||||||
|
buf[0] = 0xe0;
|
||||||
|
buf[2] = 0x80;
|
||||||
|
for (byt = 0x80; byt <= 0x9f; byt++) {
|
||||||
|
CHECKINVALID(1, byt, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test overlong 4-byte
|
||||||
|
buf[0] = 0xf0;
|
||||||
|
buf[2] = 0x80;
|
||||||
|
buf[3] = 0x80;
|
||||||
|
for (byt = 0x80; byt <= 0x8f; byt++) {
|
||||||
|
CHECKINVALID(1, byt, 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 4-byte > 0x10ffff
|
||||||
|
buf[0] = 0xf4;
|
||||||
|
buf[2] = 0x80;
|
||||||
|
buf[3] = 0x80;
|
||||||
|
for (byt = 0x90; byt <= 0xbf; byt++) {
|
||||||
|
CHECKINVALID(1, byt, 4);
|
||||||
|
}
|
||||||
|
buf[1] = 0x80;
|
||||||
|
for (byt = 0xf5; byt <= 0xf7; byt++) {
|
||||||
|
CHECKINVALID(0, byt, 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 5-byte
|
||||||
|
buf[4] = 0x80;
|
||||||
|
for (byt = 0xf8; byt <= 0xfb; byt++) {
|
||||||
|
CHECKINVALID(0, byt, 5);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 6-byte
|
||||||
|
buf[5] = 0x80;
|
||||||
|
for (byt = 0xfc; byt <= 0xfd; byt++) {
|
||||||
|
CHECKINVALID(0, byt, 6);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 7-byte
|
||||||
|
buf[6] = 0x80;
|
||||||
|
CHECKINVALID(0, 0xfe, 7);
|
||||||
|
|
||||||
|
// Three and above byte sequences
|
||||||
|
for (byt = 0xe0; byt < 0xf0; byt++) {
|
||||||
|
// Lead followed by only 1 continuation byte
|
||||||
|
CHECKINVALID(0, byt, 2);
|
||||||
|
// Lead ended by non-continuation character < 0x80
|
||||||
|
CHECKINVALID(2, 65, 3);
|
||||||
|
// Lead ended by non-continuation character > 0xbf
|
||||||
|
CHECKINVALID(2, 0xc0, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3-byte encoded surrogate character(s)
|
||||||
|
buf[0] = 0xed; buf[2] = 0x80;
|
||||||
|
// Single surrogate
|
||||||
|
CHECKINVALID(1, 0xa0, 3);
|
||||||
|
// Trailing surrogate first
|
||||||
|
CHECKINVALID(1, 0xb0, 3);
|
||||||
|
|
||||||
|
// Four byte sequences
|
||||||
|
buf[1] = 0x80;
|
||||||
|
for (byt = 0xf0; byt < 0xf5; byt++) {
|
||||||
|
// Lead followed by only 1 continuation bytes
|
||||||
|
CHECKINVALID(0, byt, 2);
|
||||||
|
// Lead followed by only 2 continuation bytes
|
||||||
|
CHECKINVALID(0, byt, 3);
|
||||||
|
// Lead followed by non-continuation character < 0x80
|
||||||
|
CHECKINVALID(3, 65, 4);
|
||||||
|
// Lead followed by non-continuation character > 0xbf
|
||||||
|
CHECKINVALID(3, 0xc0, 4);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
|
||||||
|
printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
64
3rdparty/utf8proc/test/normtest.c
vendored
Executable file
64
3rdparty/utf8proc/test/normtest.c
vendored
Executable file
|
|
@ -0,0 +1,64 @@
|
||||||
|
#include "tests.h"
|
||||||
|
|
||||||
|
#define CHECK_NORM(NRM, norm, src) { \
|
||||||
|
char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src); \
|
||||||
|
check(!strcmp(norm, src_norm), \
|
||||||
|
"normalization failed for %s -> %s", src, norm); \
|
||||||
|
free(src_norm); \
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
char *buf = NULL;
|
||||||
|
size_t bufsize = 0;
|
||||||
|
FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
|
||||||
|
char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
|
||||||
|
|
||||||
|
check(f != NULL, "error opening NormalizationTest.txt");
|
||||||
|
while (getline(&buf, &bufsize, f) > 0) {
|
||||||
|
size_t offset;
|
||||||
|
lineno += 1;
|
||||||
|
|
||||||
|
if (buf[0] == '@') {
|
||||||
|
printf("line %zd: %s", lineno, buf + 1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if (lineno % 1000 == 0)
|
||||||
|
printf("checking line %zd...\n", lineno);
|
||||||
|
|
||||||
|
if (buf[0] == '#') continue;
|
||||||
|
|
||||||
|
offset = encode(source, buf);
|
||||||
|
offset += encode(NFC, buf + offset);
|
||||||
|
offset += encode(NFD, buf + offset);
|
||||||
|
offset += encode(NFKC, buf + offset);
|
||||||
|
offset += encode(NFKD, buf + offset);
|
||||||
|
|
||||||
|
CHECK_NORM(NFC, NFC, source);
|
||||||
|
CHECK_NORM(NFC, NFC, NFC);
|
||||||
|
CHECK_NORM(NFC, NFC, NFD);
|
||||||
|
CHECK_NORM(NFC, NFKC, NFKC);
|
||||||
|
CHECK_NORM(NFC, NFKC, NFKD);
|
||||||
|
|
||||||
|
CHECK_NORM(NFD, NFD, source);
|
||||||
|
CHECK_NORM(NFD, NFD, NFC);
|
||||||
|
CHECK_NORM(NFD, NFD, NFD);
|
||||||
|
CHECK_NORM(NFD, NFKD, NFKC);
|
||||||
|
CHECK_NORM(NFD, NFKD, NFKD);
|
||||||
|
|
||||||
|
CHECK_NORM(NFKC, NFKC, source);
|
||||||
|
CHECK_NORM(NFKC, NFKC, NFC);
|
||||||
|
CHECK_NORM(NFKC, NFKC, NFD);
|
||||||
|
CHECK_NORM(NFKC, NFKC, NFKC);
|
||||||
|
CHECK_NORM(NFKC, NFKC, NFKD);
|
||||||
|
|
||||||
|
CHECK_NORM(NFKD, NFKD, source);
|
||||||
|
CHECK_NORM(NFKD, NFKD, NFC);
|
||||||
|
CHECK_NORM(NFKD, NFKD, NFD);
|
||||||
|
CHECK_NORM(NFKD, NFKD, NFKC);
|
||||||
|
CHECK_NORM(NFKD, NFKD, NFKD);
|
||||||
|
}
|
||||||
|
fclose(f);
|
||||||
|
printf("Passed tests after %zd lines!\n", lineno);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
49
3rdparty/utf8proc/test/printproperty.c
vendored
Executable file
49
3rdparty/utf8proc/test/printproperty.c
vendored
Executable file
|
|
@ -0,0 +1,49 @@
|
||||||
|
/* simple test program to print out the utf8proc properties for a codepoint */
|
||||||
|
|
||||||
|
#include "tests.h"
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int i;
|
||||||
|
|
||||||
|
for (i = 1; i < argc; ++i) {
|
||||||
|
unsigned int c;
|
||||||
|
if (!strcmp(argv[i], "-V")) {
|
||||||
|
printf("utf8proc version %s\n", utf8proc_version());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
|
||||||
|
const utf8proc_property_t *p = utf8proc_get_property(c);
|
||||||
|
printf("U+%s:\n"
|
||||||
|
" category = %s\n"
|
||||||
|
" combining_class = %d\n"
|
||||||
|
" bidi_class = %d\n"
|
||||||
|
" decomp_type = %d\n"
|
||||||
|
" uppercase_mapping = %x\n"
|
||||||
|
" lowercase_mapping = %x\n"
|
||||||
|
" titlecase_mapping = %x\n"
|
||||||
|
" comb_index = %d\n"
|
||||||
|
" bidi_mirrored = %d\n"
|
||||||
|
" comp_exclusion = %d\n"
|
||||||
|
" ignorable = %d\n"
|
||||||
|
" control_boundary = %d\n"
|
||||||
|
" boundclass = %d\n"
|
||||||
|
" charwidth = %d\n",
|
||||||
|
argv[i],
|
||||||
|
utf8proc_category_string(c),
|
||||||
|
p->combining_class,
|
||||||
|
p->bidi_class,
|
||||||
|
p->decomp_type,
|
||||||
|
utf8proc_toupper(c),
|
||||||
|
utf8proc_tolower(c),
|
||||||
|
utf8proc_totitle(c),
|
||||||
|
p->comb_index,
|
||||||
|
p->bidi_mirrored,
|
||||||
|
p->comp_exclusion,
|
||||||
|
p->ignorable,
|
||||||
|
p->control_boundary,
|
||||||
|
p->boundclass,
|
||||||
|
utf8proc_charwidth(c));
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
46
3rdparty/utf8proc/test/tests.c
vendored
Executable file
46
3rdparty/utf8proc/test/tests.c
vendored
Executable file
|
|
@ -0,0 +1,46 @@
|
||||||
|
/* Common functions for our test programs. */
|
||||||
|
|
||||||
|
#include "tests.h"
|
||||||
|
|
||||||
|
size_t lineno = 0;
|
||||||
|
|
||||||
|
void check(int cond, const char *format, ...)
|
||||||
|
{
|
||||||
|
if (!cond) {
|
||||||
|
va_list args;
|
||||||
|
fprintf(stderr, "line %zd: ", lineno);
|
||||||
|
va_start(args, format);
|
||||||
|
vfprintf(stderr, format, args);
|
||||||
|
va_end(args);
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t skipspaces(const char *buf, size_t i)
|
||||||
|
{
|
||||||
|
while (isspace(buf[i])) ++i;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
|
||||||
|
separated by whitespace, and terminated by any character not in
|
||||||
|
[0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
|
||||||
|
in dest, returning the number of bytes read from buf */
|
||||||
|
size_t encode(char *dest, const char *buf)
|
||||||
|
{
|
||||||
|
size_t i = 0, j, d = 0;
|
||||||
|
for (;;) {
|
||||||
|
int c;
|
||||||
|
i = skipspaces(buf, i);
|
||||||
|
for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
|
||||||
|
; /* find end of hex input */
|
||||||
|
if (j == i) { /* no codepoint found */
|
||||||
|
dest[d] = 0; /* NUL-terminate destination string */
|
||||||
|
return i + 1;
|
||||||
|
}
|
||||||
|
check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
|
||||||
|
i = j; /* skip to char after hex input */
|
||||||
|
d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
|
||||||
|
}
|
||||||
|
}
|
||||||
23
3rdparty/utf8proc/test/tests.h
vendored
Executable file
23
3rdparty/utf8proc/test/tests.h
vendored
Executable file
|
|
@ -0,0 +1,23 @@
|
||||||
|
/* Common functions and includes for our test programs. */
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Set feature macro to enable getline() and wcwidth().
|
||||||
|
*
|
||||||
|
* Please refer to section 2.2.1 of POSIX.1-2008:
|
||||||
|
* http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
|
||||||
|
*/
|
||||||
|
#define _XOPEN_SOURCE 700
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <stdarg.h>
|
||||||
|
|
||||||
|
#include "../utf8proc.h"
|
||||||
|
|
||||||
|
extern size_t lineno;
|
||||||
|
|
||||||
|
void check(int cond, const char *format, ...);
|
||||||
|
size_t skipspaces(const char *buf, size_t i);
|
||||||
|
size_t encode(char *dest, const char *buf);
|
||||||
41
3rdparty/utf8proc/test/valid.c
vendored
Executable file
41
3rdparty/utf8proc/test/valid.c
vendored
Executable file
|
|
@ -0,0 +1,41 @@
|
||||||
|
#include "tests.h"
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <wchar.h>
|
||||||
|
|
||||||
|
int main(int argc, char **argv)
|
||||||
|
{
|
||||||
|
int c, error = 0;
|
||||||
|
|
||||||
|
(void) argc; /* unused */
|
||||||
|
(void) argv; /* unused */
|
||||||
|
|
||||||
|
/* some simple sanity tests of */
|
||||||
|
for (c = 0; c < 0xd800; c++) {
|
||||||
|
if (!utf8proc_codepoint_valid(c)) {
|
||||||
|
fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
|
||||||
|
error++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (;c < 0xe000; c++) {
|
||||||
|
if (utf8proc_codepoint_valid(c)) {
|
||||||
|
fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
|
||||||
|
error++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (;c < 0x110000; c++) {
|
||||||
|
if (!utf8proc_codepoint_valid(c)) {
|
||||||
|
fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
|
||||||
|
error++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (;c < 0x110010; c++) {
|
||||||
|
if (utf8proc_codepoint_valid(c)) {
|
||||||
|
fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
|
||||||
|
error++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
|
||||||
|
printf("Validity tests SUCCEEDED.\n");
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
542
3rdparty/utf8proc/utf8proc.c
vendored
Normal file → Executable file
542
3rdparty/utf8proc/utf8proc.c
vendored
Normal file → Executable file
|
|
@ -1,4 +1,6 @@
|
||||||
|
/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
|
||||||
/*
|
/*
|
||||||
|
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
|
||||||
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||||
*
|
*
|
||||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
|
@ -43,7 +45,7 @@
|
||||||
#include "utf8proc_data.c"
|
#include "utf8proc_data.c"
|
||||||
|
|
||||||
|
|
||||||
const int8_t utf8proc_utf8class[256] = {
|
UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
|
||||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||||||
|
|
@ -81,25 +83,16 @@ const int8_t utf8proc_utf8class[256] = {
|
||||||
#define UTF8PROC_HANGUL_S_START 0xAC00
|
#define UTF8PROC_HANGUL_S_START 0xAC00
|
||||||
#define UTF8PROC_HANGUL_S_END 0xD7A4
|
#define UTF8PROC_HANGUL_S_END 0xD7A4
|
||||||
|
|
||||||
|
/* Should follow semantic-versioning rules (semver.org) based on API
|
||||||
#define UTF8PROC_BOUNDCLASS_START 0
|
compatibility. (Note that the shared-library version number will
|
||||||
#define UTF8PROC_BOUNDCLASS_OTHER 1
|
be different, being based on ABI compatibility.): */
|
||||||
#define UTF8PROC_BOUNDCLASS_CR 2
|
#define STRINGIZEx(x) #x
|
||||||
#define UTF8PROC_BOUNDCLASS_LF 3
|
#define STRINGIZE(x) STRINGIZEx(x)
|
||||||
#define UTF8PROC_BOUNDCLASS_CONTROL 4
|
UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
|
||||||
#define UTF8PROC_BOUNDCLASS_EXTEND 5
|
return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
|
||||||
#define UTF8PROC_BOUNDCLASS_L 6
|
|
||||||
#define UTF8PROC_BOUNDCLASS_V 7
|
|
||||||
#define UTF8PROC_BOUNDCLASS_T 8
|
|
||||||
#define UTF8PROC_BOUNDCLASS_LV 9
|
|
||||||
#define UTF8PROC_BOUNDCLASS_LVT 10
|
|
||||||
|
|
||||||
|
|
||||||
const char *utf8proc_version(void) {
|
|
||||||
return "1.1.5";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const char *utf8proc_errmsg(ssize_t errcode) {
|
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
|
||||||
switch (errcode) {
|
switch (errcode) {
|
||||||
case UTF8PROC_ERROR_NOMEM:
|
case UTF8PROC_ERROR_NOMEM:
|
||||||
return "Memory for processing UTF-8 data could not be allocated.";
|
return "Memory for processing UTF-8 data could not be allocated.";
|
||||||
|
|
@ -112,58 +105,64 @@ const char *utf8proc_errmsg(ssize_t errcode) {
|
||||||
case UTF8PROC_ERROR_INVALIDOPTS:
|
case UTF8PROC_ERROR_INVALIDOPTS:
|
||||||
return "Invalid options for UTF-8 processing chosen.";
|
return "Invalid options for UTF-8 processing chosen.";
|
||||||
default:
|
default:
|
||||||
return "An unknown error occured while processing UTF-8 data.";
|
return "An unknown error occurred while processing UTF-8 data.";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t utf8proc_iterate(
|
#define utf_cont(ch) (((ch) & 0xc0) == 0x80)
|
||||||
const uint8_t *str, ssize_t strlen, int32_t *dst
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
|
||||||
|
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
|
||||||
) {
|
) {
|
||||||
int length;
|
utf8proc_uint32_t uc;
|
||||||
int i;
|
const utf8proc_uint8_t *end;
|
||||||
int32_t uc = -1;
|
|
||||||
*dst = -1;
|
*dst = -1;
|
||||||
if (!strlen) return 0;
|
if (!strlen) return 0;
|
||||||
length = utf8proc_utf8class[str[0]];
|
end = str + ((strlen < 0) ? 4 : strlen);
|
||||||
if (!length) return UTF8PROC_ERROR_INVALIDUTF8;
|
uc = *str++;
|
||||||
if (strlen >= 0 && length > strlen) return UTF8PROC_ERROR_INVALIDUTF8;
|
if (uc < 0x80) {
|
||||||
for (i=1; i<length; i++) {
|
*dst = uc;
|
||||||
if ((str[i] & 0xC0) != 0x80) return UTF8PROC_ERROR_INVALIDUTF8;
|
return 1;
|
||||||
}
|
}
|
||||||
switch (length) {
|
// Must be between 0xc2 and 0xf4 inclusive to be valid
|
||||||
case 1:
|
if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
uc = str[0];
|
if (uc < 0xe0) { // 2-byte sequence
|
||||||
break;
|
// Must have valid continuation character
|
||||||
case 2:
|
if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
uc = ((str[0] & 0x1F) << 6) + (str[1] & 0x3F);
|
*dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
|
||||||
if (uc < 0x80) uc = -1;
|
return 2;
|
||||||
break;
|
|
||||||
case 3:
|
|
||||||
uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) << 6)
|
|
||||||
+ (str[2] & 0x3F);
|
|
||||||
if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) ||
|
|
||||||
(uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1;
|
|
||||||
break;
|
|
||||||
case 4:
|
|
||||||
uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12)
|
|
||||||
+ ((str[2] & 0x3F) << 6) + (str[3] & 0x3F);
|
|
||||||
if (uc < 0x10000 || uc >= 0x110000) uc = -1;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE))
|
if (uc < 0xf0) { // 3-byte sequence
|
||||||
return UTF8PROC_ERROR_INVALIDUTF8;
|
if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
|
||||||
*dst = uc;
|
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
return length;
|
// Check for surrogate chars
|
||||||
|
if (uc == 0xed && *str > 0x9f)
|
||||||
|
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
|
uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
|
||||||
|
if (uc < 0x800)
|
||||||
|
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
|
*dst = uc;
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
// 4-byte sequence
|
||||||
|
// Must have 3 valid continuation characters
|
||||||
|
if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
|
||||||
|
return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
|
// Make sure in correct range (0x10000 - 0x10ffff)
|
||||||
|
if (uc == 0xf0) {
|
||||||
|
if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
|
} else if (uc == 0xf4) {
|
||||||
|
if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
|
}
|
||||||
|
*dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
|
||||||
|
return 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool utf8proc_codepoint_valid(int32_t uc) {
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
|
||||||
if (uc < 0 || uc >= 0x110000 ||
|
return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
|
||||||
((uc & 0xFFFF) >= 0xFFFE) || (uc >= 0xD800 && uc < 0xE000) ||
|
|
||||||
(uc >= 0xFDD0 && uc < 0xFDF0)) return false;
|
|
||||||
else return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
|
||||||
if (uc < 0x00) {
|
if (uc < 0x00) {
|
||||||
return 0;
|
return 0;
|
||||||
} else if (uc < 0x80) {
|
} else if (uc < 0x80) {
|
||||||
|
|
@ -173,12 +172,8 @@ ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
|
||||||
dst[0] = 0xC0 + (uc >> 6);
|
dst[0] = 0xC0 + (uc >> 6);
|
||||||
dst[1] = 0x80 + (uc & 0x3F);
|
dst[1] = 0x80 + (uc & 0x3F);
|
||||||
return 2;
|
return 2;
|
||||||
} else if (uc == 0xFFFF) {
|
// Note: we allow encoding 0xd800-0xdfff here, so as not to change
|
||||||
dst[0] = 0xFF;
|
// the API, however, these are actually invalid in UTF-8
|
||||||
return 1;
|
|
||||||
} else if (uc == 0xFFFE) {
|
|
||||||
dst[0] = 0xFE;
|
|
||||||
return 1;
|
|
||||||
} else if (uc < 0x10000) {
|
} else if (uc < 0x10000) {
|
||||||
dst[0] = 0xE0 + (uc >> 12);
|
dst[0] = 0xE0 + (uc >> 12);
|
||||||
dst[1] = 0x80 + ((uc >> 6) & 0x3F);
|
dst[1] = 0x80 + ((uc >> 6) & 0x3F);
|
||||||
|
|
@ -193,7 +188,39 @@ ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
|
||||||
} else return 0;
|
} else return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const utf8proc_property_t *utf8proc_get_property(int32_t uc) {
|
/* internal "unsafe" version that does not check whether uc is in range */
|
||||||
|
static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
|
||||||
|
if (uc < 0x00) {
|
||||||
|
return 0;
|
||||||
|
} else if (uc < 0x80) {
|
||||||
|
dst[0] = uc;
|
||||||
|
return 1;
|
||||||
|
} else if (uc < 0x800) {
|
||||||
|
dst[0] = 0xC0 + (uc >> 6);
|
||||||
|
dst[1] = 0x80 + (uc & 0x3F);
|
||||||
|
return 2;
|
||||||
|
} else if (uc == 0xFFFF) {
|
||||||
|
dst[0] = 0xFF;
|
||||||
|
return 1;
|
||||||
|
} else if (uc == 0xFFFE) {
|
||||||
|
dst[0] = 0xFE;
|
||||||
|
return 1;
|
||||||
|
} else if (uc < 0x10000) {
|
||||||
|
dst[0] = 0xE0 + (uc >> 12);
|
||||||
|
dst[1] = 0x80 + ((uc >> 6) & 0x3F);
|
||||||
|
dst[2] = 0x80 + (uc & 0x3F);
|
||||||
|
return 3;
|
||||||
|
} else if (uc < 0x110000) {
|
||||||
|
dst[0] = 0xF0 + (uc >> 18);
|
||||||
|
dst[1] = 0x80 + ((uc >> 12) & 0x3F);
|
||||||
|
dst[2] = 0x80 + ((uc >> 6) & 0x3F);
|
||||||
|
dst[3] = 0x80 + (uc & 0x3F);
|
||||||
|
return 4;
|
||||||
|
} else return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* internal "unsafe" version that does not check whether uc is in range */
|
||||||
|
static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
|
||||||
/* ASSERT: uc >= 0 && uc < 0x110000 */
|
/* ASSERT: uc >= 0 && uc < 0x110000 */
|
||||||
return utf8proc_properties + (
|
return utf8proc_properties + (
|
||||||
utf8proc_stage2table[
|
utf8proc_stage2table[
|
||||||
|
|
@ -202,22 +229,185 @@ const utf8proc_property_t *utf8proc_get_property(int32_t uc) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
|
||||||
|
return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* return whether there is a grapheme break between boundclasses lbc and tbc
|
||||||
|
(according to the definition of extended grapheme clusters)
|
||||||
|
|
||||||
|
Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
|
||||||
|
http://www.unicode.org/reports/tr29/tr29-29.html
|
||||||
|
|
||||||
|
CAVEATS:
|
||||||
|
Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
|
||||||
|
and GB 12/13 (regional indicator code points) require knowledge of previous characters
|
||||||
|
and are thus not handled by this function. This may result in an incorrect break before
|
||||||
|
an E_Modifier class codepoint and an incorrectly missing break between two
|
||||||
|
REGIONAL_INDICATOR class code points if such support does not exist in the caller.
|
||||||
|
|
||||||
|
See the special support in grapheme_break_extended, for required bookkeeping by the caller.
|
||||||
|
*/
|
||||||
|
static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
|
||||||
|
return
|
||||||
|
(lbc == UTF8PROC_BOUNDCLASS_START) ? true : // GB1
|
||||||
|
(lbc == UTF8PROC_BOUNDCLASS_CR && // GB3
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_LF) ? false : // ---
|
||||||
|
(lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB4
|
||||||
|
(tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true : // GB5
|
||||||
|
(lbc == UTF8PROC_BOUNDCLASS_L && // GB6
|
||||||
|
(tbc == UTF8PROC_BOUNDCLASS_L || // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_V || // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_LV || // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false : // ---
|
||||||
|
((lbc == UTF8PROC_BOUNDCLASS_LV || // GB7
|
||||||
|
lbc == UTF8PROC_BOUNDCLASS_V) && // ---
|
||||||
|
(tbc == UTF8PROC_BOUNDCLASS_V || // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_T)) ? false : // ---
|
||||||
|
((lbc == UTF8PROC_BOUNDCLASS_LVT || // GB8
|
||||||
|
lbc == UTF8PROC_BOUNDCLASS_T) && // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_T) ? false : // ---
|
||||||
|
(tbc == UTF8PROC_BOUNDCLASS_EXTEND || // GB9
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_ZWJ || // ---
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK || // GB9a
|
||||||
|
lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false : // GB9b
|
||||||
|
((lbc == UTF8PROC_BOUNDCLASS_E_BASE || // GB10 (requires additional handling below)
|
||||||
|
lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) && // ----
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
|
||||||
|
(lbc == UTF8PROC_BOUNDCLASS_ZWJ && // GB11
|
||||||
|
(tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ || // ----
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false : // ----
|
||||||
|
(lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR && // GB12/13 (requires additional handling below)
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false : // ----
|
||||||
|
true; // GB999
|
||||||
|
}
|
||||||
|
|
||||||
|
static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
|
||||||
|
{
|
||||||
|
int lbc_override = lbc;
|
||||||
|
if (state && *state != UTF8PROC_BOUNDCLASS_START)
|
||||||
|
lbc_override = *state;
|
||||||
|
utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
|
||||||
|
if (state) {
|
||||||
|
// Special support for GB 12/13 made possible by GB999. After two RI
|
||||||
|
// class codepoints we want to force a break. Do this by resetting the
|
||||||
|
// second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
|
||||||
|
// after that character according to GB999 (unless of course such a break is
|
||||||
|
// forbidden by a different rule such as GB9).
|
||||||
|
if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
|
||||||
|
*state = UTF8PROC_BOUNDCLASS_OTHER;
|
||||||
|
// Special support for GB10. Fold any EXTEND codepoints into the previous
|
||||||
|
// boundclass if we're dealing with an emoji base boundclass.
|
||||||
|
else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE ||
|
||||||
|
*state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
|
||||||
|
tbc == UTF8PROC_BOUNDCLASS_EXTEND)
|
||||||
|
*state = UTF8PROC_BOUNDCLASS_E_BASE;
|
||||||
|
else
|
||||||
|
*state = tbc;
|
||||||
|
}
|
||||||
|
return break_permitted;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
|
||||||
|
utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
|
||||||
|
|
||||||
|
return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
|
||||||
|
utf8proc_get_property(c2)->boundclass,
|
||||||
|
state);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
|
||||||
|
utf8proc_int32_t c1, utf8proc_int32_t c2) {
|
||||||
|
return utf8proc_grapheme_break_stateful(c1, c2, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
|
||||||
|
{
|
||||||
|
utf8proc_int32_t entry_cp = **entry;
|
||||||
|
if ((entry_cp & 0xF800) == 0xD800) {
|
||||||
|
*entry = *entry + 1;
|
||||||
|
entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
|
||||||
|
entry_cp += 0x10000;
|
||||||
|
}
|
||||||
|
return entry_cp;
|
||||||
|
}
|
||||||
|
|
||||||
|
static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
|
||||||
|
{
|
||||||
|
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
|
||||||
|
return seqindex_decode_entry(&entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
|
||||||
|
utf8proc_ssize_t written = 0;
|
||||||
|
const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
|
||||||
|
int len = seqindex >> 13;
|
||||||
|
if (len >= 7) {
|
||||||
|
len = *entry;
|
||||||
|
entry++;
|
||||||
|
}
|
||||||
|
for (; len >= 0; entry++, len--) {
|
||||||
|
utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
|
||||||
|
|
||||||
|
written += utf8proc_decompose_char(entry_cp, dst+written,
|
||||||
|
(bufsize > written) ? (bufsize - written) : 0, options,
|
||||||
|
last_boundclass);
|
||||||
|
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
|
||||||
|
}
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
|
||||||
|
{
|
||||||
|
utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
|
||||||
|
return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
|
||||||
|
{
|
||||||
|
utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
|
||||||
|
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
|
||||||
|
{
|
||||||
|
utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
|
||||||
|
return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* return a character width analogous to wcwidth (except portable and
|
||||||
|
hopefully less buggy than most system wcwidth functions). */
|
||||||
|
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
|
||||||
|
return utf8proc_get_property(c)->charwidth;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
|
||||||
|
return utf8proc_get_property(c)->category;
|
||||||
|
}
|
||||||
|
|
||||||
|
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
|
||||||
|
static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
|
||||||
|
return s[utf8proc_category(c)];
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
#define utf8proc_decompose_lump(replacement_uc) \
|
#define utf8proc_decompose_lump(replacement_uc) \
|
||||||
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
|
return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
|
||||||
options & ~UTF8PROC_LUMP, last_boundclass)
|
options & ~UTF8PROC_LUMP, last_boundclass)
|
||||||
|
|
||||||
ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
|
||||||
int options, int *last_boundclass) {
|
|
||||||
/* ASSERT: uc >= 0 && uc < 0x110000 */
|
|
||||||
const utf8proc_property_t *property;
|
const utf8proc_property_t *property;
|
||||||
utf8proc_propval_t category;
|
utf8proc_propval_t category;
|
||||||
int32_t hangul_sindex;
|
utf8proc_int32_t hangul_sindex;
|
||||||
property = utf8proc_get_property(uc);
|
if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
|
||||||
|
property = unsafe_get_property(uc);
|
||||||
category = property->category;
|
category = property->category;
|
||||||
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
|
hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
|
||||||
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
||||||
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
|
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
|
||||||
int32_t hangul_tindex;
|
utf8proc_int32_t hangul_tindex;
|
||||||
if (bufsize >= 1) {
|
if (bufsize >= 1) {
|
||||||
dst[0] = UTF8PROC_HANGUL_LBASE +
|
dst[0] = UTF8PROC_HANGUL_LBASE +
|
||||||
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
|
hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
|
||||||
|
|
@ -268,79 +458,20 @@ ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
|
||||||
category == UTF8PROC_CATEGORY_ME) return 0;
|
category == UTF8PROC_CATEGORY_ME) return 0;
|
||||||
}
|
}
|
||||||
if (options & UTF8PROC_CASEFOLD) {
|
if (options & UTF8PROC_CASEFOLD) {
|
||||||
if (property->casefold_mapping) {
|
if (property->casefold_seqindex != UINT16_MAX) {
|
||||||
const int32_t *casefold_entry;
|
return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
|
||||||
ssize_t written = 0;
|
|
||||||
for (casefold_entry = property->casefold_mapping;
|
|
||||||
*casefold_entry >= 0; casefold_entry++) {
|
|
||||||
written += utf8proc_decompose_char(*casefold_entry, dst+written,
|
|
||||||
(bufsize > written) ? (bufsize - written) : 0, options,
|
|
||||||
last_boundclass);
|
|
||||||
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
|
|
||||||
}
|
|
||||||
return written;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
|
||||||
if (property->decomp_mapping &&
|
if (property->decomp_seqindex != UINT16_MAX &&
|
||||||
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
|
(!property->decomp_type || (options & UTF8PROC_COMPAT))) {
|
||||||
const int32_t *decomp_entry;
|
return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
|
||||||
ssize_t written = 0;
|
|
||||||
for (decomp_entry = property->decomp_mapping;
|
|
||||||
*decomp_entry >= 0; decomp_entry++) {
|
|
||||||
written += utf8proc_decompose_char(*decomp_entry, dst+written,
|
|
||||||
(bufsize > written) ? (bufsize - written) : 0, options,
|
|
||||||
last_boundclass);
|
|
||||||
if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
|
|
||||||
}
|
|
||||||
return written;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (options & UTF8PROC_CHARBOUND) {
|
if (options & UTF8PROC_CHARBOUND) {
|
||||||
bool boundary;
|
utf8proc_bool boundary;
|
||||||
int tbc, lbc;
|
int tbc = property->boundclass;
|
||||||
tbc =
|
boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
|
||||||
(uc == 0x000D) ? UTF8PROC_BOUNDCLASS_CR :
|
|
||||||
(uc == 0x000A) ? UTF8PROC_BOUNDCLASS_LF :
|
|
||||||
((category == UTF8PROC_CATEGORY_ZL ||
|
|
||||||
category == UTF8PROC_CATEGORY_ZP ||
|
|
||||||
category == UTF8PROC_CATEGORY_CC ||
|
|
||||||
category == UTF8PROC_CATEGORY_CF) &&
|
|
||||||
!(uc == 0x200C || uc == 0x200D)) ? UTF8PROC_BOUNDCLASS_CONTROL :
|
|
||||||
property->extend ? UTF8PROC_BOUNDCLASS_EXTEND :
|
|
||||||
((uc >= UTF8PROC_HANGUL_L_START && uc < UTF8PROC_HANGUL_L_END) ||
|
|
||||||
uc == UTF8PROC_HANGUL_L_FILLER) ? UTF8PROC_BOUNDCLASS_L :
|
|
||||||
(uc >= UTF8PROC_HANGUL_V_START && uc < UTF8PROC_HANGUL_V_END) ?
|
|
||||||
UTF8PROC_BOUNDCLASS_V :
|
|
||||||
(uc >= UTF8PROC_HANGUL_T_START && uc < UTF8PROC_HANGUL_T_END) ?
|
|
||||||
UTF8PROC_BOUNDCLASS_T :
|
|
||||||
(uc >= UTF8PROC_HANGUL_S_START && uc < UTF8PROC_HANGUL_S_END) ? (
|
|
||||||
((uc-UTF8PROC_HANGUL_SBASE) % UTF8PROC_HANGUL_TCOUNT == 0) ?
|
|
||||||
UTF8PROC_BOUNDCLASS_LV : UTF8PROC_BOUNDCLASS_LVT
|
|
||||||
) :
|
|
||||||
UTF8PROC_BOUNDCLASS_OTHER;
|
|
||||||
lbc = *last_boundclass;
|
|
||||||
boundary =
|
|
||||||
(tbc == UTF8PROC_BOUNDCLASS_EXTEND) ? false :
|
|
||||||
(lbc == UTF8PROC_BOUNDCLASS_START) ? true :
|
|
||||||
(lbc == UTF8PROC_BOUNDCLASS_CR &&
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_LF) ? false :
|
|
||||||
(lbc == UTF8PROC_BOUNDCLASS_CONTROL) ? true :
|
|
||||||
(tbc == UTF8PROC_BOUNDCLASS_CONTROL) ? true :
|
|
||||||
(lbc == UTF8PROC_BOUNDCLASS_L &&
|
|
||||||
(tbc == UTF8PROC_BOUNDCLASS_L ||
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_V ||
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_LV ||
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :
|
|
||||||
((lbc == UTF8PROC_BOUNDCLASS_LV ||
|
|
||||||
lbc == UTF8PROC_BOUNDCLASS_V) &&
|
|
||||||
(tbc == UTF8PROC_BOUNDCLASS_V ||
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_T)) ? false :
|
|
||||||
((lbc == UTF8PROC_BOUNDCLASS_LVT ||
|
|
||||||
lbc == UTF8PROC_BOUNDCLASS_T) &&
|
|
||||||
tbc == UTF8PROC_BOUNDCLASS_T) ? false :
|
|
||||||
true;
|
|
||||||
*last_boundclass = tbc;
|
|
||||||
if (boundary) {
|
if (boundary) {
|
||||||
if (bufsize >= 1) dst[0] = 0xFFFF;
|
if (bufsize >= 1) dst[0] = 0xFFFF;
|
||||||
if (bufsize >= 2) dst[1] = uc;
|
if (bufsize >= 2) dst[1] = uc;
|
||||||
|
|
@ -351,26 +482,26 @@ ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t utf8proc_decompose(
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
|
||||||
const uint8_t *str, ssize_t strlen,
|
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||||
int32_t *buffer, ssize_t bufsize, int options
|
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
|
||||||
) {
|
) {
|
||||||
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
|
/* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
|
||||||
ssize_t wpos = 0;
|
utf8proc_ssize_t wpos = 0;
|
||||||
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
|
if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
|
||||||
return UTF8PROC_ERROR_INVALIDOPTS;
|
return UTF8PROC_ERROR_INVALIDOPTS;
|
||||||
if ((options & UTF8PROC_STRIPMARK) &&
|
if ((options & UTF8PROC_STRIPMARK) &&
|
||||||
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
|
!(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
|
||||||
return UTF8PROC_ERROR_INVALIDOPTS;
|
return UTF8PROC_ERROR_INVALIDOPTS;
|
||||||
{
|
{
|
||||||
int32_t uc;
|
utf8proc_int32_t uc;
|
||||||
ssize_t rpos = 0;
|
utf8proc_ssize_t rpos = 0;
|
||||||
ssize_t decomp_result;
|
utf8proc_ssize_t decomp_result;
|
||||||
int boundclass = UTF8PROC_BOUNDCLASS_START;
|
int boundclass = UTF8PROC_BOUNDCLASS_START;
|
||||||
while (1) {
|
while (1) {
|
||||||
if (options & UTF8PROC_NULLTERM) {
|
if (options & UTF8PROC_NULLTERM) {
|
||||||
rpos += utf8proc_iterate(str + rpos, -1, &uc);
|
rpos += utf8proc_iterate(str + rpos, -1, &uc);
|
||||||
/* checking of return value is not neccessary,
|
/* checking of return value is not necessary,
|
||||||
as 'uc' is < 0 in case of error */
|
as 'uc' is < 0 in case of error */
|
||||||
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
|
if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
|
||||||
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
|
if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
|
||||||
|
|
@ -387,19 +518,20 @@ ssize_t utf8proc_decompose(
|
||||||
if (decomp_result < 0) return decomp_result;
|
if (decomp_result < 0) return decomp_result;
|
||||||
wpos += decomp_result;
|
wpos += decomp_result;
|
||||||
/* prohibiting integer overflows due to too long strings: */
|
/* prohibiting integer overflows due to too long strings: */
|
||||||
if (wpos < 0 || wpos > SSIZE_MAX/sizeof(int32_t)/2)
|
if (wpos < 0 ||
|
||||||
|
wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
|
||||||
return UTF8PROC_ERROR_OVERFLOW;
|
return UTF8PROC_ERROR_OVERFLOW;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
|
if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
|
||||||
ssize_t pos = 0;
|
utf8proc_ssize_t pos = 0;
|
||||||
while (pos < wpos-1) {
|
while (pos < wpos-1) {
|
||||||
int32_t uc1, uc2;
|
utf8proc_int32_t uc1, uc2;
|
||||||
const utf8proc_property_t *property1, *property2;
|
const utf8proc_property_t *property1, *property2;
|
||||||
uc1 = buffer[pos];
|
uc1 = buffer[pos];
|
||||||
uc2 = buffer[pos+1];
|
uc2 = buffer[pos+1];
|
||||||
property1 = utf8proc_get_property(uc1);
|
property1 = unsafe_get_property(uc1);
|
||||||
property2 = utf8proc_get_property(uc2);
|
property2 = unsafe_get_property(uc2);
|
||||||
if (property1->combining_class > property2->combining_class &&
|
if (property1->combining_class > property2->combining_class &&
|
||||||
property2->combining_class > 0) {
|
property2->combining_class > 0) {
|
||||||
buffer[pos] = uc2;
|
buffer[pos] = uc2;
|
||||||
|
|
@ -413,13 +545,13 @@ ssize_t utf8proc_decompose(
|
||||||
return wpos;
|
return wpos;
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
|
||||||
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
|
/* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
|
||||||
ASSERT: 'buffer' has one spare byte of free space at the end! */
|
ASSERT: 'buffer' has one spare byte of free space at the end! */
|
||||||
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
|
if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
|
||||||
ssize_t rpos;
|
utf8proc_ssize_t rpos;
|
||||||
ssize_t wpos = 0;
|
utf8proc_ssize_t wpos = 0;
|
||||||
int32_t uc;
|
utf8proc_int32_t uc;
|
||||||
for (rpos = 0; rpos < length; rpos++) {
|
for (rpos = 0; rpos < length; rpos++) {
|
||||||
uc = buffer[rpos];
|
uc = buffer[rpos];
|
||||||
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
|
if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
|
||||||
|
|
@ -448,23 +580,23 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
|
||||||
length = wpos;
|
length = wpos;
|
||||||
}
|
}
|
||||||
if (options & UTF8PROC_COMPOSE) {
|
if (options & UTF8PROC_COMPOSE) {
|
||||||
int32_t *starter = NULL;
|
utf8proc_int32_t *starter = NULL;
|
||||||
int32_t current_char;
|
utf8proc_int32_t current_char;
|
||||||
const utf8proc_property_t *starter_property = NULL, *current_property;
|
const utf8proc_property_t *starter_property = NULL, *current_property;
|
||||||
utf8proc_propval_t max_combining_class = -1;
|
utf8proc_propval_t max_combining_class = -1;
|
||||||
ssize_t rpos;
|
utf8proc_ssize_t rpos;
|
||||||
ssize_t wpos = 0;
|
utf8proc_ssize_t wpos = 0;
|
||||||
int32_t composition;
|
utf8proc_int32_t composition;
|
||||||
for (rpos = 0; rpos < length; rpos++) {
|
for (rpos = 0; rpos < length; rpos++) {
|
||||||
current_char = buffer[rpos];
|
current_char = buffer[rpos];
|
||||||
current_property = utf8proc_get_property(current_char);
|
current_property = unsafe_get_property(current_char);
|
||||||
if (starter && current_property->combining_class > max_combining_class) {
|
if (starter && current_property->combining_class > max_combining_class) {
|
||||||
/* combination perhaps possible */
|
/* combination perhaps possible */
|
||||||
int32_t hangul_lindex;
|
utf8proc_int32_t hangul_lindex;
|
||||||
int32_t hangul_sindex;
|
utf8proc_int32_t hangul_sindex;
|
||||||
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
|
hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
|
||||||
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
|
if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
|
||||||
int32_t hangul_vindex;
|
utf8proc_int32_t hangul_vindex;
|
||||||
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
|
hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
|
||||||
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
|
if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
|
||||||
*starter = UTF8PROC_HANGUL_SBASE +
|
*starter = UTF8PROC_HANGUL_SBASE +
|
||||||
|
|
@ -477,7 +609,7 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
|
||||||
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
|
hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
|
||||||
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
|
if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
|
||||||
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
|
(hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
|
||||||
int32_t hangul_tindex;
|
utf8proc_int32_t hangul_tindex;
|
||||||
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
|
hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
|
||||||
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
|
if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
|
||||||
*starter += hangul_tindex;
|
*starter += hangul_tindex;
|
||||||
|
|
@ -486,19 +618,26 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!starter_property) {
|
if (!starter_property) {
|
||||||
starter_property = utf8proc_get_property(*starter);
|
starter_property = unsafe_get_property(*starter);
|
||||||
}
|
}
|
||||||
if (starter_property->comb1st_index >= 0 &&
|
if (starter_property->comb_index < 0x8000 &&
|
||||||
current_property->comb2nd_index >= 0) {
|
current_property->comb_index != UINT16_MAX &&
|
||||||
composition = utf8proc_combinations[
|
current_property->comb_index >= 0x8000) {
|
||||||
starter_property->comb1st_index +
|
int sidx = starter_property->comb_index;
|
||||||
current_property->comb2nd_index
|
int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
|
||||||
];
|
if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
|
||||||
if (composition >= 0 && (!(options & UTF8PROC_STABLE) ||
|
idx += sidx + 2;
|
||||||
!(utf8proc_get_property(composition)->comp_exclusion))) {
|
if (current_property->comb_index & 0x4000) {
|
||||||
*starter = composition;
|
composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
|
||||||
starter_property = NULL;
|
} else
|
||||||
continue;
|
composition = utf8proc_combinations[idx];
|
||||||
|
|
||||||
|
if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
|
||||||
|
!(unsafe_get_property(composition)->comp_exclusion))) {
|
||||||
|
*starter = composition;
|
||||||
|
starter_property = NULL;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -517,26 +656,33 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
|
||||||
length = wpos;
|
length = wpos;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
ssize_t rpos, wpos = 0;
|
utf8proc_ssize_t rpos, wpos = 0;
|
||||||
int32_t uc;
|
utf8proc_int32_t uc;
|
||||||
for (rpos = 0; rpos < length; rpos++) {
|
if (options & UTF8PROC_CHARBOUND) {
|
||||||
uc = buffer[rpos];
|
for (rpos = 0; rpos < length; rpos++) {
|
||||||
wpos += utf8proc_encode_char(uc, ((uint8_t *)buffer) + wpos);
|
uc = buffer[rpos];
|
||||||
|
wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (rpos = 0; rpos < length; rpos++) {
|
||||||
|
uc = buffer[rpos];
|
||||||
|
wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
((uint8_t *)buffer)[wpos] = 0;
|
((utf8proc_uint8_t *)buffer)[wpos] = 0;
|
||||||
return wpos;
|
return wpos;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t utf8proc_map(
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
|
||||||
const uint8_t *str, ssize_t strlen, uint8_t **dstptr, int options
|
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
|
||||||
) {
|
) {
|
||||||
int32_t *buffer;
|
utf8proc_int32_t *buffer;
|
||||||
ssize_t result;
|
utf8proc_ssize_t result;
|
||||||
*dstptr = NULL;
|
*dstptr = NULL;
|
||||||
result = utf8proc_decompose(str, strlen, NULL, 0, options);
|
result = utf8proc_decompose(str, strlen, NULL, 0, options);
|
||||||
if (result < 0) return result;
|
if (result < 0) return result;
|
||||||
buffer = malloc(result * sizeof(int32_t) + 1);
|
buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
|
||||||
if (!buffer) return UTF8PROC_ERROR_NOMEM;
|
if (!buffer) return UTF8PROC_ERROR_NOMEM;
|
||||||
result = utf8proc_decompose(str, strlen, buffer, result, options);
|
result = utf8proc_decompose(str, strlen, buffer, result, options);
|
||||||
if (result < 0) {
|
if (result < 0) {
|
||||||
|
|
@ -549,37 +695,37 @@ ssize_t utf8proc_map(
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
int32_t *newptr;
|
utf8proc_int32_t *newptr;
|
||||||
newptr = realloc(buffer, (size_t)result+1);
|
newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
|
||||||
if (newptr) buffer = newptr;
|
if (newptr) buffer = newptr;
|
||||||
}
|
}
|
||||||
*dstptr = (uint8_t *)buffer;
|
*dstptr = (utf8proc_uint8_t *)buffer;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t *utf8proc_NFD(const uint8_t *str) {
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
|
||||||
uint8_t *retval;
|
utf8proc_uint8_t *retval;
|
||||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||||
UTF8PROC_DECOMPOSE);
|
UTF8PROC_DECOMPOSE);
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t *utf8proc_NFC(const uint8_t *str) {
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
|
||||||
uint8_t *retval;
|
utf8proc_uint8_t *retval;
|
||||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||||
UTF8PROC_COMPOSE);
|
UTF8PROC_COMPOSE);
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t *utf8proc_NFKD(const uint8_t *str) {
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
|
||||||
uint8_t *retval;
|
utf8proc_uint8_t *retval;
|
||||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||||
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
|
UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
|
||||||
return retval;
|
return retval;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t *utf8proc_NFKC(const uint8_t *str) {
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
|
||||||
uint8_t *retval;
|
utf8proc_uint8_t *retval;
|
||||||
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
|
||||||
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
|
UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
|
||||||
return retval;
|
return retval;
|
||||||
|
|
|
||||||
862
3rdparty/utf8proc/utf8proc.h
vendored
Normal file → Executable file
862
3rdparty/utf8proc/utf8proc.h
vendored
Normal file → Executable file
|
|
@ -1,78 +1,130 @@
|
||||||
/*
|
/*
|
||||||
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
* Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
|
||||||
|
* Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
|
||||||
*
|
*
|
||||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
* copy of this software and associated documentation files (the "Software"),
|
* copy of this software and associated documentation files (the "Software"),
|
||||||
* to deal in the Software without restriction, including without limitation
|
* to deal in the Software without restriction, including without limitation
|
||||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
* and/or sell copies of the Software, and to permit persons to whom the
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||||||
* Software is furnished to do so, subject to the following conditions:
|
* Software is furnished to do so, subject to the following conditions:
|
||||||
*
|
*
|
||||||
* The above copyright notice and this permission notice shall be included in
|
* The above copyright notice and this permission notice shall be included in
|
||||||
* all copies or substantial portions of the Software.
|
* all copies or substantial portions of the Software.
|
||||||
*
|
*
|
||||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
* DEALINGS IN THE SOFTWARE.
|
* DEALINGS IN THE SOFTWARE.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
|
||||||
/*
|
/**
|
||||||
* File name: utf8proc.h
|
* @mainpage
|
||||||
*
|
*
|
||||||
* Description:
|
* utf8proc is a free/open-source (MIT/expat licensed) C library
|
||||||
* Header files for libutf8proc, which is a mapping tool for UTF-8 strings
|
* providing Unicode normalization, case-folding, and other operations
|
||||||
* with following features:
|
* for strings in the UTF-8 encoding, supporting Unicode version
|
||||||
* - decomposing and composing of strings
|
* 8.0.0. See the utf8proc home page (http://julialang.org/utf8proc/)
|
||||||
* - replacing compatibility characters with their equivalents
|
* for downloads and other information, or the source code on github
|
||||||
* - stripping of "default ignorable characters"
|
* (https://github.com/JuliaLang/utf8proc).
|
||||||
* like SOFT-HYPHEN or ZERO-WIDTH-SPACE
|
*
|
||||||
* - folding of certain characters for string comparison
|
* For the utf8proc API documentation, see: @ref utf8proc.h
|
||||||
* (e.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-")
|
*
|
||||||
* (see "LUMP" option)
|
* The features of utf8proc include:
|
||||||
* - optional rejection of strings containing non-assigned code points
|
*
|
||||||
* - stripping of control characters
|
* - Transformation of strings (@ref utf8proc_map) to:
|
||||||
* - stripping of character marks (accents, etc.)
|
* - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
|
||||||
* - transformation of LF, CRLF, CR and NEL to line-feed (LF)
|
* - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
|
||||||
* or to the unicode chararacters for paragraph separation (PS)
|
* - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
|
||||||
* or line separation (LS).
|
* - case-folding (@ref UTF8PROC_CASEFOLD)
|
||||||
* - unicode case folding (for case insensitive string comparisons)
|
* - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
|
||||||
* - rejection of illegal UTF-8 data
|
* - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
|
||||||
* (i.e. UTF-8 encoded UTF-16 surrogates)
|
* - Character-width computation: @ref utf8proc_charwidth
|
||||||
* - support for korean hangul characters
|
* - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
|
||||||
* Unicode Version 5.0.0 is supported.
|
* - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/** @file */
|
||||||
|
|
||||||
#ifndef UTF8PROC_H
|
#ifndef UTF8PROC_H
|
||||||
#define UTF8PROC_H
|
#define UTF8PROC_H
|
||||||
|
|
||||||
|
/** @name API version
|
||||||
|
*
|
||||||
|
* The utf8proc API version MAJOR.MINOR.PATCH, following
|
||||||
|
* semantic-versioning rules (http://semver.org) based on API
|
||||||
|
* compatibility.
|
||||||
|
*
|
||||||
|
* This is also returned at runtime by @ref utf8proc_version; however, the
|
||||||
|
* runtime version may append a string like "-dev" to the version number
|
||||||
|
* for prerelease versions.
|
||||||
|
*
|
||||||
|
* @note The shared-library version number in the Makefile
|
||||||
|
* (and CMakeLists.txt, and MANIFEST) may be different,
|
||||||
|
* being based on ABI compatibility rather than API compatibility.
|
||||||
|
*/
|
||||||
|
/** @{ */
|
||||||
|
/** The MAJOR version number (increased when backwards API compatibility is broken). */
|
||||||
|
#define UTF8PROC_VERSION_MAJOR 2
|
||||||
|
/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
|
||||||
|
#define UTF8PROC_VERSION_MINOR 0
|
||||||
|
/** The PATCH version (increased for fixes that do not change the API). */
|
||||||
|
#define UTF8PROC_VERSION_PATCH 2
|
||||||
|
/** @} */
|
||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#ifdef _MSC_VER
|
#ifdef _MSC_VER
|
||||||
typedef signed char int8_t;
|
typedef signed char utf8proc_int8_t;
|
||||||
typedef unsigned char uint8_t;
|
typedef unsigned char utf8proc_uint8_t;
|
||||||
typedef short int16_t;
|
typedef short utf8proc_int16_t;
|
||||||
typedef unsigned short uint16_t;
|
typedef unsigned short utf8proc_uint16_t;
|
||||||
typedef int int32_t;
|
typedef int utf8proc_int32_t;
|
||||||
#ifdef _WIN64
|
typedef unsigned int utf8proc_uint32_t;
|
||||||
#define ssize_t __int64
|
# ifdef _WIN64
|
||||||
#else
|
typedef __int64 utf8proc_ssize_t;
|
||||||
#define ssize_t int
|
typedef unsigned __int64 utf8proc_size_t;
|
||||||
#endif
|
# else
|
||||||
typedef unsigned char bool;
|
typedef int utf8proc_ssize_t;
|
||||||
|
typedef unsigned int utf8proc_size_t;
|
||||||
|
# endif
|
||||||
|
# ifndef __cplusplus
|
||||||
|
typedef unsigned char utf8proc_bool;
|
||||||
enum {false, true};
|
enum {false, true};
|
||||||
|
# else
|
||||||
|
typedef bool utf8proc_bool;
|
||||||
|
# endif
|
||||||
#else
|
#else
|
||||||
#include <stdbool.h>
|
# include <stdbool.h>
|
||||||
#include <inttypes.h>
|
# include <inttypes.h>
|
||||||
|
typedef int8_t utf8proc_int8_t;
|
||||||
|
typedef uint8_t utf8proc_uint8_t;
|
||||||
|
typedef int16_t utf8proc_int16_t;
|
||||||
|
typedef uint16_t utf8proc_uint16_t;
|
||||||
|
typedef int32_t utf8proc_int32_t;
|
||||||
|
typedef uint32_t utf8proc_uint32_t;
|
||||||
|
typedef size_t utf8proc_size_t;
|
||||||
|
typedef ssize_t utf8proc_ssize_t;
|
||||||
|
typedef bool utf8proc_bool;
|
||||||
#endif
|
#endif
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
|
||||||
|
#ifdef _WIN32
|
||||||
|
# ifdef UTF8PROC_EXPORTS
|
||||||
|
# define UTF8PROC_DLLEXPORT __declspec(dllexport)
|
||||||
|
# else
|
||||||
|
# define UTF8PROC_DLLEXPORT __declspec(dllimport)
|
||||||
|
# endif
|
||||||
|
#elif __GNUC__ >= 4
|
||||||
|
# define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
|
||||||
|
#else
|
||||||
|
# define UTF8PROC_DLLEXPORT
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -81,301 +133,497 @@ extern "C" {
|
||||||
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
|
#define SSIZE_MAX ((size_t)SIZE_MAX/2)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define UTF8PROC_NULLTERM (1<<0)
|
#ifndef UINT16_MAX
|
||||||
#define UTF8PROC_STABLE (1<<1)
|
# define UINT16_MAX 65535U
|
||||||
#define UTF8PROC_COMPAT (1<<2)
|
#endif
|
||||||
#define UTF8PROC_COMPOSE (1<<3)
|
|
||||||
#define UTF8PROC_DECOMPOSE (1<<4)
|
|
||||||
#define UTF8PROC_IGNORE (1<<5)
|
|
||||||
#define UTF8PROC_REJECTNA (1<<6)
|
|
||||||
#define UTF8PROC_NLF2LS (1<<7)
|
|
||||||
#define UTF8PROC_NLF2PS (1<<8)
|
|
||||||
#define UTF8PROC_NLF2LF (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS)
|
|
||||||
#define UTF8PROC_STRIPCC (1<<9)
|
|
||||||
#define UTF8PROC_CASEFOLD (1<<10)
|
|
||||||
#define UTF8PROC_CHARBOUND (1<<11)
|
|
||||||
#define UTF8PROC_LUMP (1<<12)
|
|
||||||
#define UTF8PROC_STRIPMARK (1<<13)
|
|
||||||
/*
|
|
||||||
* Flags being regarded by several functions in the library:
|
|
||||||
* NULLTERM: The given UTF-8 input is NULL terminated.
|
|
||||||
* STABLE: Unicode Versioning Stability has to be respected.
|
|
||||||
* COMPAT: Compatiblity decomposition
|
|
||||||
* (i.e. formatting information is lost)
|
|
||||||
* COMPOSE: Return a result with composed characters.
|
|
||||||
* DECOMPOSE: Return a result with decomposed characters.
|
|
||||||
* IGNORE: Strip "default ignorable characters"
|
|
||||||
* REJECTNA: Return an error, if the input contains unassigned
|
|
||||||
* code points.
|
|
||||||
* NLF2LS: Indicating that NLF-sequences (LF, CRLF, CR, NEL) are
|
|
||||||
* representing a line break, and should be converted to the
|
|
||||||
* unicode character for line separation (LS).
|
|
||||||
* NLF2PS: Indicating that NLF-sequences are representing a paragraph
|
|
||||||
* break, and should be converted to the unicode character for
|
|
||||||
* paragraph separation (PS).
|
|
||||||
* NLF2LF: Indicating that the meaning of NLF-sequences is unknown.
|
|
||||||
* STRIPCC: Strips and/or convers control characters.
|
|
||||||
* NLF-sequences are transformed into space, except if one of
|
|
||||||
* the NLF2LS/PS/LF options is given.
|
|
||||||
* HorizontalTab (HT) and FormFeed (FF) are treated as a
|
|
||||||
* NLF-sequence in this case.
|
|
||||||
* All other control characters are simply removed.
|
|
||||||
* CASEFOLD: Performs unicode case folding, to be able to do a
|
|
||||||
* case-insensitive string comparison.
|
|
||||||
* CHARBOUND: Inserts 0xFF bytes at the beginning of each sequence which
|
|
||||||
* is representing a single grapheme cluster (see UAX#29).
|
|
||||||
* LUMP: Lumps certain characters together
|
|
||||||
* (e.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-").
|
|
||||||
* (See lump.txt for details.)
|
|
||||||
* If NLF2LF is set, this includes a transformation of
|
|
||||||
* paragraph and line separators to ASCII line-feed (LF).
|
|
||||||
* STRIPMARK: Strips all character markings
|
|
||||||
* (non-spacing, spacing and enclosing) (i.e. accents)
|
|
||||||
* NOTE: this option works only with COMPOSE or DECOMPOSE
|
|
||||||
*/
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Option flags used by several functions in the library.
|
||||||
|
*/
|
||||||
|
typedef enum {
|
||||||
|
/** The given UTF-8 input is NULL terminated. */
|
||||||
|
UTF8PROC_NULLTERM = (1<<0),
|
||||||
|
/** Unicode Versioning Stability has to be respected. */
|
||||||
|
UTF8PROC_STABLE = (1<<1),
|
||||||
|
/** Compatibility decomposition (i.e. formatting information is lost). */
|
||||||
|
UTF8PROC_COMPAT = (1<<2),
|
||||||
|
/** Return a result with decomposed characters. */
|
||||||
|
UTF8PROC_COMPOSE = (1<<3),
|
||||||
|
/** Return a result with decomposed characters. */
|
||||||
|
UTF8PROC_DECOMPOSE = (1<<4),
|
||||||
|
/** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
|
||||||
|
UTF8PROC_IGNORE = (1<<5),
|
||||||
|
/** Return an error, if the input contains unassigned codepoints. */
|
||||||
|
UTF8PROC_REJECTNA = (1<<6),
|
||||||
|
/**
|
||||||
|
* Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
|
||||||
|
* line break, and should be converted to the codepoint for line
|
||||||
|
* separation (LS).
|
||||||
|
*/
|
||||||
|
UTF8PROC_NLF2LS = (1<<7),
|
||||||
|
/**
|
||||||
|
* Indicating that NLF-sequences are representing a paragraph break, and
|
||||||
|
* should be converted to the codepoint for paragraph separation
|
||||||
|
* (PS).
|
||||||
|
*/
|
||||||
|
UTF8PROC_NLF2PS = (1<<8),
|
||||||
|
/** Indicating that the meaning of NLF-sequences is unknown. */
|
||||||
|
UTF8PROC_NLF2LF = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
|
||||||
|
/** Strips and/or convers control characters.
|
||||||
|
*
|
||||||
|
* NLF-sequences are transformed into space, except if one of the
|
||||||
|
* NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
|
||||||
|
* are treated as a NLF-sequence in this case. All other control
|
||||||
|
* characters are simply removed.
|
||||||
|
*/
|
||||||
|
UTF8PROC_STRIPCC = (1<<9),
|
||||||
|
/**
|
||||||
|
* Performs unicode case folding, to be able to do a case-insensitive
|
||||||
|
* string comparison.
|
||||||
|
*/
|
||||||
|
UTF8PROC_CASEFOLD = (1<<10),
|
||||||
|
/**
|
||||||
|
* Inserts 0xFF bytes at the beginning of each sequence which is
|
||||||
|
* representing a single grapheme cluster (see UAX#29).
|
||||||
|
*/
|
||||||
|
UTF8PROC_CHARBOUND = (1<<11),
|
||||||
|
/** Lumps certain characters together.
|
||||||
|
*
|
||||||
|
* E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
|
||||||
|
*
|
||||||
|
* If NLF2LF is set, this includes a transformation of paragraph and
|
||||||
|
* line separators to ASCII line-feed (LF).
|
||||||
|
*/
|
||||||
|
UTF8PROC_LUMP = (1<<12),
|
||||||
|
/** Strips all character markings.
|
||||||
|
*
|
||||||
|
* This includes non-spacing, spacing and enclosing (i.e. accents).
|
||||||
|
* @note This option works only with @ref UTF8PROC_COMPOSE or
|
||||||
|
* @ref UTF8PROC_DECOMPOSE
|
||||||
|
*/
|
||||||
|
UTF8PROC_STRIPMARK = (1<<13),
|
||||||
|
} utf8proc_option_t;
|
||||||
|
|
||||||
|
/** @name Error codes
|
||||||
|
* Error codes being returned by almost all functions.
|
||||||
|
*/
|
||||||
|
/** @{ */
|
||||||
|
/** Memory could not be allocated. */
|
||||||
#define UTF8PROC_ERROR_NOMEM -1
|
#define UTF8PROC_ERROR_NOMEM -1
|
||||||
|
/** The given string is too long to be processed. */
|
||||||
#define UTF8PROC_ERROR_OVERFLOW -2
|
#define UTF8PROC_ERROR_OVERFLOW -2
|
||||||
|
/** The given string is not a legal UTF-8 string. */
|
||||||
#define UTF8PROC_ERROR_INVALIDUTF8 -3
|
#define UTF8PROC_ERROR_INVALIDUTF8 -3
|
||||||
|
/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
|
||||||
#define UTF8PROC_ERROR_NOTASSIGNED -4
|
#define UTF8PROC_ERROR_NOTASSIGNED -4
|
||||||
|
/** Invalid options have been used. */
|
||||||
#define UTF8PROC_ERROR_INVALIDOPTS -5
|
#define UTF8PROC_ERROR_INVALIDOPTS -5
|
||||||
/*
|
/** @} */
|
||||||
* Error codes being returned by almost all functions:
|
|
||||||
* ERROR_NOMEM: Memory could not be allocated.
|
|
||||||
* ERROR_OVERFLOW: The given string is too long to be processed.
|
|
||||||
* ERROR_INVALIDUTF8: The given string is not a legal UTF-8 string.
|
|
||||||
* ERROR_NOTASSIGNED: The REJECTNA flag was set,
|
|
||||||
* and an unassigned code point was found.
|
|
||||||
* ERROR_INVALIDOPTS: Invalid options have been used.
|
|
||||||
*/
|
|
||||||
|
|
||||||
typedef int16_t utf8proc_propval_t;
|
/* @name Types */
|
||||||
|
|
||||||
|
/** Holds the value of a property. */
|
||||||
|
typedef utf8proc_int16_t utf8proc_propval_t;
|
||||||
|
|
||||||
|
/** Struct containing information about a codepoint. */
|
||||||
typedef struct utf8proc_property_struct {
|
typedef struct utf8proc_property_struct {
|
||||||
|
/**
|
||||||
|
* Unicode category.
|
||||||
|
* @see utf8proc_category_t.
|
||||||
|
*/
|
||||||
utf8proc_propval_t category;
|
utf8proc_propval_t category;
|
||||||
utf8proc_propval_t combining_class;
|
utf8proc_propval_t combining_class;
|
||||||
|
/**
|
||||||
|
* Bidirectional class.
|
||||||
|
* @see utf8proc_bidi_class_t.
|
||||||
|
*/
|
||||||
utf8proc_propval_t bidi_class;
|
utf8proc_propval_t bidi_class;
|
||||||
|
/**
|
||||||
|
* @anchor Decomposition type.
|
||||||
|
* @see utf8proc_decomp_type_t.
|
||||||
|
*/
|
||||||
utf8proc_propval_t decomp_type;
|
utf8proc_propval_t decomp_type;
|
||||||
const int32_t *decomp_mapping;
|
utf8proc_uint16_t decomp_seqindex;
|
||||||
|
utf8proc_uint16_t casefold_seqindex;
|
||||||
|
utf8proc_uint16_t uppercase_seqindex;
|
||||||
|
utf8proc_uint16_t lowercase_seqindex;
|
||||||
|
utf8proc_uint16_t titlecase_seqindex;
|
||||||
|
utf8proc_uint16_t comb_index;
|
||||||
unsigned bidi_mirrored:1;
|
unsigned bidi_mirrored:1;
|
||||||
int32_t uppercase_mapping;
|
|
||||||
int32_t lowercase_mapping;
|
|
||||||
int32_t titlecase_mapping;
|
|
||||||
int32_t comb1st_index;
|
|
||||||
int32_t comb2nd_index;
|
|
||||||
unsigned comp_exclusion:1;
|
unsigned comp_exclusion:1;
|
||||||
|
/**
|
||||||
|
* Can this codepoint be ignored?
|
||||||
|
*
|
||||||
|
* Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
|
||||||
|
* passed as an option.
|
||||||
|
*/
|
||||||
unsigned ignorable:1;
|
unsigned ignorable:1;
|
||||||
unsigned control_boundary:1;
|
unsigned control_boundary:1;
|
||||||
unsigned extend:1;
|
/** The width of the codepoint. */
|
||||||
const int32_t *casefold_mapping;
|
unsigned charwidth:2;
|
||||||
|
unsigned pad:2;
|
||||||
|
/**
|
||||||
|
* Boundclass.
|
||||||
|
* @see utf8proc_boundclass_t.
|
||||||
|
*/
|
||||||
|
unsigned boundclass:8;
|
||||||
} utf8proc_property_t;
|
} utf8proc_property_t;
|
||||||
|
|
||||||
#define UTF8PROC_CATEGORY_LU 1
|
/** Unicode categories. */
|
||||||
#define UTF8PROC_CATEGORY_LL 2
|
typedef enum {
|
||||||
#define UTF8PROC_CATEGORY_LT 3
|
UTF8PROC_CATEGORY_CN = 0, /**< Other, not assigned */
|
||||||
#define UTF8PROC_CATEGORY_LM 4
|
UTF8PROC_CATEGORY_LU = 1, /**< Letter, uppercase */
|
||||||
#define UTF8PROC_CATEGORY_LO 5
|
UTF8PROC_CATEGORY_LL = 2, /**< Letter, lowercase */
|
||||||
#define UTF8PROC_CATEGORY_MN 6
|
UTF8PROC_CATEGORY_LT = 3, /**< Letter, titlecase */
|
||||||
#define UTF8PROC_CATEGORY_MC 7
|
UTF8PROC_CATEGORY_LM = 4, /**< Letter, modifier */
|
||||||
#define UTF8PROC_CATEGORY_ME 8
|
UTF8PROC_CATEGORY_LO = 5, /**< Letter, other */
|
||||||
#define UTF8PROC_CATEGORY_ND 9
|
UTF8PROC_CATEGORY_MN = 6, /**< Mark, nonspacing */
|
||||||
#define UTF8PROC_CATEGORY_NL 10
|
UTF8PROC_CATEGORY_MC = 7, /**< Mark, spacing combining */
|
||||||
#define UTF8PROC_CATEGORY_NO 11
|
UTF8PROC_CATEGORY_ME = 8, /**< Mark, enclosing */
|
||||||
#define UTF8PROC_CATEGORY_PC 12
|
UTF8PROC_CATEGORY_ND = 9, /**< Number, decimal digit */
|
||||||
#define UTF8PROC_CATEGORY_PD 13
|
UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
|
||||||
#define UTF8PROC_CATEGORY_PS 14
|
UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
|
||||||
#define UTF8PROC_CATEGORY_PE 15
|
UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
|
||||||
#define UTF8PROC_CATEGORY_PI 16
|
UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
|
||||||
#define UTF8PROC_CATEGORY_PF 17
|
UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
|
||||||
#define UTF8PROC_CATEGORY_PO 18
|
UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
|
||||||
#define UTF8PROC_CATEGORY_SM 19
|
UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
|
||||||
#define UTF8PROC_CATEGORY_SC 20
|
UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
|
||||||
#define UTF8PROC_CATEGORY_SK 21
|
UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
|
||||||
#define UTF8PROC_CATEGORY_SO 22
|
UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
|
||||||
#define UTF8PROC_CATEGORY_ZS 23
|
UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
|
||||||
#define UTF8PROC_CATEGORY_ZL 24
|
UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
|
||||||
#define UTF8PROC_CATEGORY_ZP 25
|
UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
|
||||||
#define UTF8PROC_CATEGORY_CC 26
|
UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
|
||||||
#define UTF8PROC_CATEGORY_CF 27
|
UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
|
||||||
#define UTF8PROC_CATEGORY_CS 28
|
UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
|
||||||
#define UTF8PROC_CATEGORY_CO 29
|
UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
|
||||||
#define UTF8PROC_CATEGORY_CN 30
|
UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
|
||||||
#define UTF8PROC_BIDI_CLASS_L 1
|
UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
|
||||||
#define UTF8PROC_BIDI_CLASS_LRE 2
|
UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
|
||||||
#define UTF8PROC_BIDI_CLASS_LRO 3
|
} utf8proc_category_t;
|
||||||
#define UTF8PROC_BIDI_CLASS_R 4
|
|
||||||
#define UTF8PROC_BIDI_CLASS_AL 5
|
|
||||||
#define UTF8PROC_BIDI_CLASS_RLE 6
|
|
||||||
#define UTF8PROC_BIDI_CLASS_RLO 7
|
|
||||||
#define UTF8PROC_BIDI_CLASS_PDF 8
|
|
||||||
#define UTF8PROC_BIDI_CLASS_EN 9
|
|
||||||
#define UTF8PROC_BIDI_CLASS_ES 10
|
|
||||||
#define UTF8PROC_BIDI_CLASS_ET 11
|
|
||||||
#define UTF8PROC_BIDI_CLASS_AN 12
|
|
||||||
#define UTF8PROC_BIDI_CLASS_CS 13
|
|
||||||
#define UTF8PROC_BIDI_CLASS_NSM 14
|
|
||||||
#define UTF8PROC_BIDI_CLASS_BN 15
|
|
||||||
#define UTF8PROC_BIDI_CLASS_B 16
|
|
||||||
#define UTF8PROC_BIDI_CLASS_S 17
|
|
||||||
#define UTF8PROC_BIDI_CLASS_WS 18
|
|
||||||
#define UTF8PROC_BIDI_CLASS_ON 19
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_FONT 1
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_NOBREAK 2
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_INITIAL 3
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_MEDIAL 4
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_FINAL 5
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_ISOLATED 6
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_CIRCLE 7
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_SUPER 8
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_SUB 9
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_VERTICAL 10
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_WIDE 11
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_NARROW 12
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_SMALL 13
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_SQUARE 14
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_FRACTION 15
|
|
||||||
#define UTF8PROC_DECOMP_TYPE_COMPAT 16
|
|
||||||
|
|
||||||
extern const int8_t utf8proc_utf8class[256];
|
/** Bidirectional character classes. */
|
||||||
|
typedef enum {
|
||||||
|
UTF8PROC_BIDI_CLASS_L = 1, /**< Left-to-Right */
|
||||||
|
UTF8PROC_BIDI_CLASS_LRE = 2, /**< Left-to-Right Embedding */
|
||||||
|
UTF8PROC_BIDI_CLASS_LRO = 3, /**< Left-to-Right Override */
|
||||||
|
UTF8PROC_BIDI_CLASS_R = 4, /**< Right-to-Left */
|
||||||
|
UTF8PROC_BIDI_CLASS_AL = 5, /**< Right-to-Left Arabic */
|
||||||
|
UTF8PROC_BIDI_CLASS_RLE = 6, /**< Right-to-Left Embedding */
|
||||||
|
UTF8PROC_BIDI_CLASS_RLO = 7, /**< Right-to-Left Override */
|
||||||
|
UTF8PROC_BIDI_CLASS_PDF = 8, /**< Pop Directional Format */
|
||||||
|
UTF8PROC_BIDI_CLASS_EN = 9, /**< European Number */
|
||||||
|
UTF8PROC_BIDI_CLASS_ES = 10, /**< European Separator */
|
||||||
|
UTF8PROC_BIDI_CLASS_ET = 11, /**< European Number Terminator */
|
||||||
|
UTF8PROC_BIDI_CLASS_AN = 12, /**< Arabic Number */
|
||||||
|
UTF8PROC_BIDI_CLASS_CS = 13, /**< Common Number Separator */
|
||||||
|
UTF8PROC_BIDI_CLASS_NSM = 14, /**< Nonspacing Mark */
|
||||||
|
UTF8PROC_BIDI_CLASS_BN = 15, /**< Boundary Neutral */
|
||||||
|
UTF8PROC_BIDI_CLASS_B = 16, /**< Paragraph Separator */
|
||||||
|
UTF8PROC_BIDI_CLASS_S = 17, /**< Segment Separator */
|
||||||
|
UTF8PROC_BIDI_CLASS_WS = 18, /**< Whitespace */
|
||||||
|
UTF8PROC_BIDI_CLASS_ON = 19, /**< Other Neutrals */
|
||||||
|
UTF8PROC_BIDI_CLASS_LRI = 20, /**< Left-to-Right Isolate */
|
||||||
|
UTF8PROC_BIDI_CLASS_RLI = 21, /**< Right-to-Left Isolate */
|
||||||
|
UTF8PROC_BIDI_CLASS_FSI = 22, /**< First Strong Isolate */
|
||||||
|
UTF8PROC_BIDI_CLASS_PDI = 23, /**< Pop Directional Isolate */
|
||||||
|
} utf8proc_bidi_class_t;
|
||||||
|
|
||||||
const char *utf8proc_version(void);
|
/** Decomposition type. */
|
||||||
|
typedef enum {
|
||||||
|
UTF8PROC_DECOMP_TYPE_FONT = 1, /**< Font */
|
||||||
|
UTF8PROC_DECOMP_TYPE_NOBREAK = 2, /**< Nobreak */
|
||||||
|
UTF8PROC_DECOMP_TYPE_INITIAL = 3, /**< Initial */
|
||||||
|
UTF8PROC_DECOMP_TYPE_MEDIAL = 4, /**< Medial */
|
||||||
|
UTF8PROC_DECOMP_TYPE_FINAL = 5, /**< Final */
|
||||||
|
UTF8PROC_DECOMP_TYPE_ISOLATED = 6, /**< Isolated */
|
||||||
|
UTF8PROC_DECOMP_TYPE_CIRCLE = 7, /**< Circle */
|
||||||
|
UTF8PROC_DECOMP_TYPE_SUPER = 8, /**< Super */
|
||||||
|
UTF8PROC_DECOMP_TYPE_SUB = 9, /**< Sub */
|
||||||
|
UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
|
||||||
|
UTF8PROC_DECOMP_TYPE_WIDE = 11, /**< Wide */
|
||||||
|
UTF8PROC_DECOMP_TYPE_NARROW = 12, /**< Narrow */
|
||||||
|
UTF8PROC_DECOMP_TYPE_SMALL = 13, /**< Small */
|
||||||
|
UTF8PROC_DECOMP_TYPE_SQUARE = 14, /**< Square */
|
||||||
|
UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
|
||||||
|
UTF8PROC_DECOMP_TYPE_COMPAT = 16, /**< Compat */
|
||||||
|
} utf8proc_decomp_type_t;
|
||||||
|
|
||||||
const char *utf8proc_errmsg(ssize_t errcode);
|
/** Boundclass property. (TR29) */
|
||||||
/*
|
typedef enum {
|
||||||
* Returns a static error string for the given error code.
|
UTF8PROC_BOUNDCLASS_START = 0, /**< Start */
|
||||||
|
UTF8PROC_BOUNDCLASS_OTHER = 1, /**< Other */
|
||||||
|
UTF8PROC_BOUNDCLASS_CR = 2, /**< Cr */
|
||||||
|
UTF8PROC_BOUNDCLASS_LF = 3, /**< Lf */
|
||||||
|
UTF8PROC_BOUNDCLASS_CONTROL = 4, /**< Control */
|
||||||
|
UTF8PROC_BOUNDCLASS_EXTEND = 5, /**< Extend */
|
||||||
|
UTF8PROC_BOUNDCLASS_L = 6, /**< L */
|
||||||
|
UTF8PROC_BOUNDCLASS_V = 7, /**< V */
|
||||||
|
UTF8PROC_BOUNDCLASS_T = 8, /**< T */
|
||||||
|
UTF8PROC_BOUNDCLASS_LV = 9, /**< Lv */
|
||||||
|
UTF8PROC_BOUNDCLASS_LVT = 10, /**< Lvt */
|
||||||
|
UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
|
||||||
|
UTF8PROC_BOUNDCLASS_SPACINGMARK = 12, /**< Spacingmark */
|
||||||
|
UTF8PROC_BOUNDCLASS_PREPEND = 13, /**< Prepend */
|
||||||
|
UTF8PROC_BOUNDCLASS_ZWJ = 14, /**< Zero Width Joiner */
|
||||||
|
UTF8PROC_BOUNDCLASS_E_BASE = 15, /**< Emoji Base */
|
||||||
|
UTF8PROC_BOUNDCLASS_E_MODIFIER = 16, /**< Emoji Modifier */
|
||||||
|
UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ = 17, /**< Glue_After_ZWJ */
|
||||||
|
UTF8PROC_BOUNDCLASS_E_BASE_GAZ = 18, /**< E_BASE + GLUE_AFTER_ZJW */
|
||||||
|
} utf8proc_boundclass_t;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Array containing the byte lengths of a UTF-8 encoded codepoint based
|
||||||
|
* on the first byte.
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];
|
||||||
|
|
||||||
ssize_t utf8proc_iterate(const uint8_t *str, ssize_t strlen, int32_t *dst);
|
/**
|
||||||
/*
|
* Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
|
||||||
* Reads a single char from the UTF-8 sequence being pointed to by 'str'.
|
* (http://semver.org format), possibly with a "-dev" suffix for
|
||||||
* The maximum number of bytes read is 'strlen', unless 'strlen' is
|
* development versions.
|
||||||
* negative.
|
|
||||||
* If a valid unicode char could be read, it is stored in the variable
|
|
||||||
* being pointed to by 'dst', otherwise that variable will be set to -1.
|
|
||||||
* In case of success the number of bytes read is returned, otherwise a
|
|
||||||
* negative error code is returned.
|
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT const char *utf8proc_version(void);
|
||||||
|
|
||||||
bool utf8proc_codepoint_valid(int32_t uc);
|
/**
|
||||||
/*
|
* Returns an informative error string for the given utf8proc error code
|
||||||
* Returns 1, if the given unicode code-point is valid, otherwise 0.
|
* (e.g. the error codes returned by @ref utf8proc_map).
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);
|
||||||
|
|
||||||
ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst);
|
/**
|
||||||
/*
|
* Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
|
||||||
* Encodes the unicode char with the code point 'uc' as an UTF-8 string in
|
* The maximum number of bytes read is `strlen`, unless `strlen` is
|
||||||
* the byte array being pointed to by 'dst'. This array has to be at least
|
* negative (in which case up to 4 bytes are read).
|
||||||
* 4 bytes long.
|
*
|
||||||
* In case of success the number of bytes written is returned,
|
* If a valid codepoint could be read, it is stored in the variable
|
||||||
* otherwise 0.
|
* pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
|
||||||
* This function does not check if 'uc' is a valid unicode code point.
|
* In case of success, the number of bytes read is returned; otherwise, a
|
||||||
|
* negative error code is returned.
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);
|
||||||
|
|
||||||
const utf8proc_property_t *utf8proc_get_property(int32_t uc);
|
/**
|
||||||
/*
|
* Check if a codepoint is valid (regardless of whether it has been
|
||||||
* Returns a pointer to a (constant) struct containing information about
|
* assigned a value by the current Unicode standard).
|
||||||
* the unicode char with the given code point 'uc'.
|
*
|
||||||
* If the character is not existent a pointer to a special struct is
|
* @return 1 if the given `codepoint` is valid and otherwise return 0.
|
||||||
* returned, where 'category' is a NULL pointer.
|
|
||||||
* WARNING: The parameter 'uc' has to be in the range of 0x0000 to
|
|
||||||
* 0x10FFFF, otherwise the program might crash!
|
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);
|
||||||
|
|
||||||
ssize_t utf8proc_decompose_char(
|
/**
|
||||||
int32_t uc, int32_t *dst, ssize_t bufsize,
|
* Encodes the codepoint as an UTF-8 string in the byte array pointed
|
||||||
int options, int *last_boundclass
|
* to by `dst`. This array must be at least 4 bytes long.
|
||||||
|
*
|
||||||
|
* In case of success the number of bytes written is returned, and
|
||||||
|
* otherwise 0 is returned.
|
||||||
|
*
|
||||||
|
* This function does not check whether `codepoint` is valid Unicode.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Look up the properties for a given codepoint.
|
||||||
|
*
|
||||||
|
* @param codepoint The Unicode codepoint.
|
||||||
|
*
|
||||||
|
* @returns
|
||||||
|
* A pointer to a (constant) struct containing information about
|
||||||
|
* the codepoint.
|
||||||
|
* @par
|
||||||
|
* If the codepoint is unassigned or invalid, a pointer to a special struct is
|
||||||
|
* returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
|
||||||
|
|
||||||
|
/** Decompose a codepoint into an array of codepoints.
|
||||||
|
*
|
||||||
|
* @param codepoint the codepoint.
|
||||||
|
* @param dst the destination buffer.
|
||||||
|
* @param bufsize the size of the destination buffer.
|
||||||
|
* @param options one or more of the following flags:
|
||||||
|
* - @ref UTF8PROC_REJECTNA - return an error `codepoint` is unassigned
|
||||||
|
* - @ref UTF8PROC_IGNORE - strip "default ignorable" codepoints
|
||||||
|
* - @ref UTF8PROC_CASEFOLD - apply Unicode casefolding
|
||||||
|
* - @ref UTF8PROC_COMPAT - replace certain codepoints with their
|
||||||
|
* compatibility decomposition
|
||||||
|
* - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
|
||||||
|
* - @ref UTF8PROC_LUMP - lump certain different codepoints together
|
||||||
|
* - @ref UTF8PROC_STRIPMARK - remove all character marks
|
||||||
|
* @param last_boundclass
|
||||||
|
* Pointer to an integer variable containing
|
||||||
|
* the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
|
||||||
|
* option is used. Otherwise, this parameter is ignored.
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
* In case of success, the number of codepoints written is returned; in case
|
||||||
|
* of an error, a negative error code is returned (@ref utf8proc_errmsg).
|
||||||
|
* @par
|
||||||
|
* If the number of written codepoints would be bigger than `bufsize`, the
|
||||||
|
* required buffer size is returned, while the buffer will be overwritten with
|
||||||
|
* undefined data.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
|
||||||
|
utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
|
||||||
|
utf8proc_option_t options, int *last_boundclass
|
||||||
);
|
);
|
||||||
/*
|
|
||||||
* Writes a decomposition of the unicode char 'uc' into the array being
|
|
||||||
* pointed to by 'dst'.
|
|
||||||
* Following flags in the 'options' field are regarded:
|
|
||||||
* REJECTNA: an unassigned unicode code point leads to an error
|
|
||||||
* IGNORE: "default ignorable" chars are stripped
|
|
||||||
* CASEFOLD: unicode casefolding is applied
|
|
||||||
* COMPAT: replace certain characters with their
|
|
||||||
* compatibility decomposition
|
|
||||||
* CHARBOUND: Inserts 0xFF bytes before each grapheme cluster
|
|
||||||
* LUMP: lumps certain different characters together
|
|
||||||
* STRIPMARK: removes all character marks
|
|
||||||
* The pointer 'last_boundclass' has to point to an integer variable which
|
|
||||||
* is storing the last character boundary class, if the CHARBOUND option
|
|
||||||
* is used.
|
|
||||||
* In case of success the number of chars written is returned,
|
|
||||||
* in case of an error, a negative error code is returned.
|
|
||||||
* If the number of written chars would be bigger than 'bufsize',
|
|
||||||
* the buffer (up to 'bufsize') has inpredictable data, and the needed
|
|
||||||
* buffer size is returned.
|
|
||||||
* WARNING: The parameter 'uc' has to be in the range of 0x0000 to
|
|
||||||
* 0x10FFFF, otherwise the program might crash!
|
|
||||||
*/
|
|
||||||
|
|
||||||
ssize_t utf8proc_decompose(
|
/**
|
||||||
const uint8_t *str, ssize_t strlen,
|
* The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
|
||||||
int32_t *buffer, ssize_t bufsize, int options
|
* string and orders the decomposed sequences correctly.
|
||||||
|
*
|
||||||
|
* If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
|
||||||
|
* will be stopped, when a NULL byte is encounted, otherwise `strlen`
|
||||||
|
* bytes are processed. The result (in the form of 32-bit unicode
|
||||||
|
* codepoints) is written into the buffer being pointed to by
|
||||||
|
* `buffer` (which must contain at least `bufsize` entries). In case of
|
||||||
|
* success, the number of codepoints written is returned; in case of an
|
||||||
|
* error, a negative error code is returned (@ref utf8proc_errmsg).
|
||||||
|
*
|
||||||
|
* If the number of written codepoints would be bigger than `bufsize`, the
|
||||||
|
* required buffer size is returned, while the buffer will be overwritten with
|
||||||
|
* undefined data.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
|
||||||
|
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
|
||||||
|
utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
|
||||||
);
|
);
|
||||||
/*
|
|
||||||
* Does the same as 'utf8proc_decompose_char', but acts on a whole UTF-8
|
|
||||||
* string, and orders the decomposed sequences correctly.
|
|
||||||
* If the NULLTERM flag in 'options' is set, processing will be stopped,
|
|
||||||
* when a NULL byte is encounted, otherwise 'strlen' bytes are processed.
|
|
||||||
* The result in form of unicode code points is written into the buffer
|
|
||||||
* being pointed to by 'buffer', having the length of 'bufsize' entries.
|
|
||||||
* In case of success the number of chars written is returned,
|
|
||||||
* in case of an error, a negative error code is returned.
|
|
||||||
* If the number of written chars would be bigger than 'bufsize',
|
|
||||||
* the buffer (up to 'bufsize') has inpredictable data, and the needed
|
|
||||||
* buffer size is returned.
|
|
||||||
*/
|
|
||||||
|
|
||||||
ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options);
|
/**
|
||||||
/*
|
* Reencodes the sequence of `length` codepoints pointed to by `buffer`
|
||||||
* Reencodes the sequence of unicode characters given by the pointer
|
* UTF-8 data in-place (i.e., the result is also stored in `buffer`).
|
||||||
* 'buffer' and 'length' as UTF-8.
|
*
|
||||||
* The result is stored in the same memory area where the data is read.
|
* @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
|
||||||
* Following flags in the 'options' field are regarded:
|
* @param length the length (in codepoints) of the buffer.
|
||||||
* NLF2LS: converts LF, CRLF, CR and NEL into LS
|
* @param options a bitwise or (`|`) of one or more of the following flags:
|
||||||
* NLF2PS: converts LF, CRLF, CR and NEL into PS
|
* - @ref UTF8PROC_NLF2LS - convert LF, CRLF, CR and NEL into LS
|
||||||
* NLF2LF: converts LF, CRLF, CR and NEL into LF
|
* - @ref UTF8PROC_NLF2PS - convert LF, CRLF, CR and NEL into PS
|
||||||
* STRIPCC: strips or converts all non-affected control characters
|
* - @ref UTF8PROC_NLF2LF - convert LF, CRLF, CR and NEL into LF
|
||||||
* COMPOSE: tries to combine decomposed characters into composite
|
* - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
|
||||||
* characters
|
* - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
|
||||||
* STABLE: prohibits combining characters which would violate
|
* codepoints
|
||||||
* the unicode versioning stability
|
* - @ref UTF8PROC_STABLE - prohibit combining characters that would violate
|
||||||
* In case of success the length of the resulting UTF-8 string is
|
* the unicode versioning stability
|
||||||
* returned, otherwise a negative error code is returned.
|
*
|
||||||
* WARNING: The amount of free space being pointed to by 'buffer', has to
|
* @return
|
||||||
* exceed the amount of the input data by one byte, and the
|
* In case of success, the length (in bytes) of the resulting UTF-8 string is
|
||||||
* entries of the array pointed to by 'str' have to be in the
|
* returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
|
||||||
* range of 0x0000 to 0x10FFFF, otherwise the program might
|
*
|
||||||
* crash!
|
* @warning The amount of free space pointed to by `buffer` must
|
||||||
|
* exceed the amount of the input data by one byte, and the
|
||||||
|
* entries of the array pointed to by `str` have to be in the
|
||||||
|
* range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
|
||||||
*/
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);
|
||||||
|
|
||||||
ssize_t utf8proc_map(
|
/**
|
||||||
const uint8_t *str, ssize_t strlen, uint8_t **dstptr, int options
|
* Given a pair of consecutive codepoints, return whether a grapheme break is
|
||||||
|
* permitted between them (as defined by the extended grapheme clusters in UAX#29).
|
||||||
|
*
|
||||||
|
* @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
|
||||||
|
* state to break graphemes. This state can be passed in as a pointer
|
||||||
|
* in the `state` argument and should initially be set to 0. If the
|
||||||
|
* state is not passed in (i.e. a null pointer is passed), UAX#29 rules
|
||||||
|
* GB10/12/13 which require this state will not be applied, essentially
|
||||||
|
* matching the rules in Unicode 8.0.0.
|
||||||
|
*
|
||||||
|
* @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
|
||||||
|
* be called IN ORDER on ALL potential breaks in a string.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
|
||||||
|
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Same as @ref utf8proc_grapheme_break_stateful, except without support for the
|
||||||
|
* Unicode 9 additions to the algorithm. Supported for legacy reasons.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
|
||||||
|
utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||||
|
* lower-case character, if any; otherwise (if there is no lower-case
|
||||||
|
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||||
|
* upper-case character, if any; otherwise (if there is no upper-case
|
||||||
|
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a codepoint `c`, return the codepoint of the corresponding
|
||||||
|
* title-case character, if any; otherwise (if there is no title-case
|
||||||
|
* variant, or if `c` is not a valid codepoint) return `c`.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
|
||||||
|
* except that a width of 0 is returned for non-printable codepoints
|
||||||
|
* instead of -1 as in `wcwidth`.
|
||||||
|
*
|
||||||
|
* @note
|
||||||
|
* If you want to check for particular types of non-printable characters,
|
||||||
|
* (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
|
||||||
|
UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the Unicode category for the codepoint (one of the
|
||||||
|
* @ref utf8proc_category_t constants.)
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the two-letter (nul-terminated) Unicode category string for
|
||||||
|
* the codepoint (e.g. `"Lu"` or `"Co"`).
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps the given UTF-8 string pointed to by `str` to a new UTF-8
|
||||||
|
* string, allocated dynamically by `malloc` and returned via `dstptr`.
|
||||||
|
*
|
||||||
|
* If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
|
||||||
|
* the length is determined by a NULL terminator, otherwise the
|
||||||
|
* parameter `strlen` is evaluated to determine the string length, but
|
||||||
|
* in any case the result will be NULL terminated (though it might
|
||||||
|
* contain NULL characters with the string if `str` contained NULL
|
||||||
|
* characters). Other flags in the `options` field are passed to the
|
||||||
|
* functions defined above, and regarded as described.
|
||||||
|
*
|
||||||
|
* In case of success the length of the new string is returned,
|
||||||
|
* otherwise a negative error code is returned.
|
||||||
|
*
|
||||||
|
* @note The memory of the new UTF-8 string will have been allocated
|
||||||
|
* with `malloc`, and should therefore be deallocated with `free`.
|
||||||
|
*/
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
|
||||||
|
const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
|
||||||
);
|
);
|
||||||
/*
|
|
||||||
* Maps the given UTF-8 string being pointed to by 'str' to a new UTF-8
|
|
||||||
* string, which is allocated dynamically, and afterwards pointed to by
|
|
||||||
* the pointer being pointed to by 'dstptr'.
|
|
||||||
* If the NULLTERM flag in the 'options' field is set, the length is
|
|
||||||
* determined by a NULL terminator, otherwise the parameter 'strlen' is
|
|
||||||
* evaluated to determine the string length, but in any case the result
|
|
||||||
* will be NULL terminated (though it might contain NULL characters
|
|
||||||
* before). Other flags in the 'options' field are passed to the functions
|
|
||||||
* defined above, and regarded as described.
|
|
||||||
* In case of success the length of the new string is returned,
|
|
||||||
* otherwise a negative error code is returned.
|
|
||||||
* NOTICE: The memory of the new UTF-8 string will have been allocated with
|
|
||||||
* 'malloc', and has theirfore to be freed with 'free'.
|
|
||||||
*/
|
|
||||||
|
|
||||||
uint8_t *utf8proc_NFD(const uint8_t *str);
|
/** @name Unicode normalization
|
||||||
uint8_t *utf8proc_NFC(const uint8_t *str);
|
*
|
||||||
uint8_t *utf8proc_NFKD(const uint8_t *str);
|
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
|
||||||
uint8_t *utf8proc_NFKC(const uint8_t *str);
|
* normalized version of the null-terminated string `str`. These
|
||||||
/*
|
* are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
|
||||||
* Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
|
* combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
|
||||||
* normalized version of the null-terminated string 'str'.
|
|
||||||
*/
|
*/
|
||||||
|
/** @{ */
|
||||||
|
/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
|
||||||
|
/** NFC normalization (@ref UTF8PROC_COMPOSE). */
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
|
||||||
|
/** NFD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
|
||||||
|
/** NFD normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
|
||||||
|
UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
|
||||||
|
/** @} */
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
|
|
|
||||||
27521
3rdparty/utf8proc/utf8proc_data.c
vendored
Normal file → Executable file
27521
3rdparty/utf8proc/utf8proc_data.c
vendored
Normal file → Executable file
File diff suppressed because it is too large
Load diff
20
3rdparty/utf8proc/utils.cmake
vendored
Executable file
20
3rdparty/utf8proc/utils.cmake
vendored
Executable file
|
|
@ -0,0 +1,20 @@
|
||||||
|
|
||||||
|
function (disallow_intree_builds)
|
||||||
|
# Adapted from LLVM's toplevel CMakeLists.txt file
|
||||||
|
if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
|
||||||
|
message(FATAL_ERROR "
|
||||||
|
In-source builds are not allowed. CMake would overwrite the
|
||||||
|
makefiles distributed with utf8proc. Please create a directory
|
||||||
|
and run cmake from there. Building in a subdirectory is
|
||||||
|
fine, e.g.:
|
||||||
|
|
||||||
|
mkdir build
|
||||||
|
cd build
|
||||||
|
cmake ..
|
||||||
|
|
||||||
|
This process created the file `CMakeCache.txt' and the
|
||||||
|
directory `CMakeFiles'. Please delete them.
|
||||||
|
|
||||||
|
")
|
||||||
|
endif()
|
||||||
|
endfunction()
|
||||||
4
env/strings.cpp
vendored
4
env/strings.cpp
vendored
|
|
@ -17,10 +17,10 @@ string MakeNormalizeAndLowerUtf8(string const & s)
|
||||||
|
|
||||||
vector<int32_t> buffer(count);
|
vector<int32_t> buffer(count);
|
||||||
int sz = utf8proc_decompose(reinterpret_cast<uint8_t const *>(s.c_str()), count, buffer.data(), count,
|
int sz = utf8proc_decompose(reinterpret_cast<uint8_t const *>(s.c_str()), count, buffer.data(), count,
|
||||||
UTF8PROC_CASEFOLD | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPMARK);
|
utf8proc_option_t(UTF8PROC_CASEFOLD | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPMARK));
|
||||||
CHECK(sz >= 0 && sz <= count, ());
|
CHECK(sz >= 0 && sz <= count, ());
|
||||||
|
|
||||||
sz = utf8proc_reencode(buffer.data(), sz, 0);
|
sz = utf8proc_reencode(buffer.data(), sz, utf8proc_option_t(0));
|
||||||
return string(reinterpret_cast<char *>(buffer.data()), sz);
|
return string(reinterpret_cast<char *>(buffer.data()), sz);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Reference in a new issue