Updated utf8proc to the most recent version.

2016-10-18 13:26:40 +02:00 · 2016-10-18 13:26:40 +02:00 · 7c7825b4b1
commit 7c7825b4b1
parent 71ef49263f
43 changed files with 18825 additions and 16171 deletions
--- a/3rdparty/utf8proc/CMakeLists.txt
+++ b/3rdparty/utf8proc/CMakeLists.txt
@ -0,0 +1,33 @@
+cmake_minimum_required (VERSION 2.8)
+
+include (utils.cmake)
+
+disallow_intree_builds()
+
+project (utf8proc C)
+
+# This is the ABI version number, which may differ from the
+# API version number (defined in utf8proc.h).
+# Be sure to also update these in Makefile and MANIFEST!
+set(SO_MAJOR 2)
+set(SO_MINOR 0)
+set(SO_PATCH 2)
+
+add_definitions (
+  -DUTF8PROC_EXPORTS
+)
+
+if (NOT MSVC)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -std=c99 -pedantic -Wall")
+endif ()
+
+add_library (utf8proc
+  utf8proc.c
+  utf8proc.h
+)
+
+set_target_properties (utf8proc PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  VERSION "${SO_MAJOR}.${SO_MINOR}.${SO_PATCH}"
+  SOVERSION ${SO_MAJOR}
+)
--- a/3rdparty/utf8proc/Changelog
+++ b/3rdparty/utf8proc/Changelog
@ -1,128 +0,0 @@
-Changelog
-
-2006-06-02:
- initial release of version 0.1
-
-2006-06-05:
- changed behaviour of PostgreSQL function to return NULL in case of
-  invalid input, rather than raising an exceptional condition
- improved efficiency of PostgreSQL function (no transformation to C string
-  is done)
-
-2006-06-20:
- added -fpic compiler flag in Makefile
- fixed bug in the C code for the ruby library (usage of non-existent
-  function)
-
-Release of version 0.2
-
-
-2006-07-18:
- changed normalization from NFC to NFKC for postgresql unifold function
-
-2006-08-04:
- added support to mark the beginning of a grapheme cluster with 0xFF
-  (option: CHARBOUND)
- added the ruby method String#chars, which is returning an array of UTF-8
-  encoded grapheme clusters
- added NLF2LF transformation in postgresql unifold function
- added the DECOMPOSE option, if you neither use COMPOSE or DECOMPOSE, no
-  normalization will be performed (different from previous versions)
- using integer constants rather than C-strings for character properties
- fixed (hopefully) a problem with the ruby library on Mac OS X, which
-  occured when compiler optimization was switched on
-
-Release of version 0.3
-
-
-2006-09-17:
- added the LUMP option, which lumps certain characters together
-  (see lump.txt) (also used for the PostgreSQL "unifold" function)
- added the STRIPMARK option, which strips marking characters
-  (or marks of composed characters)
- deprecated ruby method String#char_ary in favour of String#utf8chars
-
-Release of version 1.0
-
-
-2006-09-20:
- included a gem file for the ruby version of the library
-
-Release of version 1.0.1
-
-
-2006-09-21:
- included a check in Integer#utf8, which raises an exception, if the given
-  code-point is invalid because of being too high (this was missing yet)
-
-2006-12-26:
- added support for PostgreSQL version 8.2
-
-Release of version 1.0.2
-
-
-2007-03-16:
- Fixed a bug in the ruby library, which caused an error, when splitting an
-  empty string at grapheme cluster boundaries (method String#utf8chars).
-
-Release of version 1.0.3
-
-
-2007-06-25:
- Added a new PostgreSQL function 'unistrip', which behaves like 'unifold',
-  but also removes all character marks (e.g. accents).
-
-2007-07-22:
- Changed license from BSD to MIT style.
- Added a new function 'utf8proc_codepoint_valid' to the C library.
- Changed compiler flags in Makefile from -g -O0 to -O2
- The ruby script, which was used to build the utf8proc_data.c file, is now
-  included in the distribution.
-
-Release of version 1.1.1
-
-
-2007-07-25:
- Fixed a serious bug in the data file generator, which caused characters
-  being treated incorrectly, when stripping default ignorable characters or
-  calculating grapheme cluster boundaries.
-
-Release of version 1.1.2
-
-
-2008-10-04:
- Added a function utf8proc_version returning a string containing the version
-  number of the library.
- Included a target libutf8proc.dylib for MacOSX.
-
-2009-05-01:
- PostgreSQL 8.3 compatibility (use of SET_VARSIZE macro)
-
-Release of version 1.1.3
-
-
-2009-06-14:
- replaced C++ style comments for compatibility reasons
- added typecasts to suppress compiler warnings
- removed redundant source files for ruby-gemfile generation
-
-2009-08-19:
- Changed copyright notice for Public Software Group e. V.
- Minor changes in the README file
- Release of version 1.1.4
-
-2009-08-20:
- Use RSTRING_PTR() and RSTRING_LEN() instead of RSTRING()->ptr and
-  RSTRING()->len for ruby1.9 compatibility (and #define them, if not
-  existent)
-
-2009-10-02:
- Patches for compatibility with Microsoft Visual Studio
-
-2009-10-08:
- Fixes to make utf8proc usable in C++ programs
-
-2009-10-16:
- Release of version 1.1.5
-
-2009-10-08:
--- a/3rdparty/utf8proc/Doxyfile
+++ b/3rdparty/utf8proc/Doxyfile
--- a/3rdparty/utf8proc/LICENSE.md
+++ b/3rdparty/utf8proc/LICENSE.md
@ -1,5 +1,13 @@
+## utf8proc license ##

-Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+**utf8proc** is a software package originally developed
+by Jan Behrens and the rest of the Public Software Group, who
+deserve nearly all of the credit for this library, that is now maintained by the Julia-language developers.  Like the original utf8proc,
+whose copyright and license statements are reproduced below, all new
+work on the utf8proc library is licensed under the [MIT "expat"
+license](http://opensource.org/licenses/MIT):
+
+*Copyright &copy; 2014-2015 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas Fonseca, and other contributors listed in the git history.*

 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the "Software"),
@ -19,14 +27,37 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.

+## Original utf8proc license ##
+
+*Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany*
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+## Unicode data license ##

 This software distribution contains derived data from a modified version of
 the Unicode data files. The following license applies to that data:

-COPYRIGHT AND PERMISSION NOTICE
+**COPYRIGHT AND PERMISSION NOTICE**

-Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
-under the Terms of Use in http://www.unicode.org/copyright.html.
+*Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+under the Terms of Use in http://www.unicode.org/copyright.html.*

 Permission is hereby granted, free of charge, to any person obtaining a
 copy of the Unicode data files and any associated documentation (the "Data
@ -57,8 +88,6 @@ not be used in advertising or otherwise to promote the sale, use or other
 dealings in these Data Files or Software without prior written
 authorization of the copyright holder.

-
 Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
 registered in some jurisdictions. All other trademarks and registered
 trademarks mentioned herein are the property of their respective owners.
-
--- a/3rdparty/utf8proc/MANIFEST
+++ b/3rdparty/utf8proc/MANIFEST
@ -0,0 +1,7 @@
+include/
+include/utf8proc.h
+lib/
+lib/libutf8proc.a
+lib/libutf8proc.so -> libutf8proc.so.2.0.2
+lib/libutf8proc.so.2 -> libutf8proc.so.2.0.2
+lib/libutf8proc.so.2.0.2
--- a/3rdparty/utf8proc/Makefile
+++ b/3rdparty/utf8proc/Makefile
@ -1,68 +1,146 @@
 # libutf8proc Makefile

+# programs
+MAKE=make
+AR?=ar
+CC?=gcc
+INSTALL=install
+FIND=find

-# settings
+# compiler settings
+CFLAGS ?= -O2
+PICFLAG = -fPIC
+C99FLAG = -std=c99
+WCFLAGS = -Wall -pedantic
+UCFLAGS = $(CFLAGS) $(PICFLAG) $(C99FLAG) $(WCFLAGS) -DUTF8PROC_EXPORTS

-cflags = -O2 -std=c99 -pedantic -Wall -fpic $(CFLAGS)
-cc = $(CC) $(cflags)
+# shared-library version MAJOR.MINOR.PATCH ... this may be *different*
+# from the utf8proc version number because it indicates ABI compatibility,
+# not API compatibility: MAJOR should be incremented whenever *binary*
+# compatibility is broken, even if the API is backward-compatible.
+# The API version number is defined in utf8proc.h.
+# Be sure to also update these ABI versions in MANIFEST and CMakeLists.txt!
+MAJOR=2
+MINOR=0
+PATCH=2

+OS := $(shell uname)
+ifeq ($(OS),Darwin) # MacOS X
+  SHLIB_EXT = dylib
+  SHLIB_VERS_EXT = $(MAJOR).dylib
+else # GNU/Linux, at least (Windows should probably use cmake)
+  SHLIB_EXT = so
+  SHLIB_VERS_EXT = so.$(MAJOR).$(MINOR).$(PATCH)
+endif
+
+# installation directories (for 'make install')
+prefix=/usr/local
+libdir=$(prefix)/lib
+includedir=$(prefix)/include

 # meta targets

-c-library: libutf8proc.a libutf8proc.so
+.PHONY: all clean data update manifest install

-ruby-library: ruby/utf8proc_native.so
+all: libutf8proc.a libutf8proc.$(SHLIB_EXT)

-pgsql-library: pgsql/utf8proc_pgsql.so
+clean:
+	rm -f utf8proc.o libutf8proc.a libutf8proc.$(SHLIB_VERS_EXT) libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	rm -f libutf8proc.so.$(MAJOR)
+endif
+	rm -f test/tests.o test/normtest test/graphemetest test/printproperty test/charwidth test/valid test/iterate test/case
+	rm -rf MANIFEST.new tmp
+	$(MAKE) -C bench clean
+	$(MAKE) -C data clean

-all: c-library ruby-library ruby-gem pgsql-library
+data: data/utf8proc_data.c.new

-clean::
-	rm -f utf8proc.o libutf8proc.a libutf8proc.so
-	cd ruby/ && test -e Makefile && (make clean && rm -f Makefile) || true
-	rm -Rf ruby/gem/lib ruby/gem/ext
-	rm -f ruby/gem/utf8proc-*.gem
-	cd pgsql/ && make clean
+update: data/utf8proc_data.c.new
+	cp -f data/utf8proc_data.c.new utf8proc_data.c
+
+manifest: MANIFEST.new

 # real targets

+data/utf8proc_data.c.new: libutf8proc.$(SHLIB_EXT) data/data_generator.rb data/charwidths.jl
+	$(MAKE) -C data utf8proc_data.c.new
+
 utf8proc.o: utf8proc.h utf8proc.c utf8proc_data.c
-	$(cc) -c -o utf8proc.o utf8proc.c
+	$(CC) $(UCFLAGS) -c -o utf8proc.o utf8proc.c

 libutf8proc.a: utf8proc.o
 	rm -f libutf8proc.a
-	ar rs libutf8proc.a utf8proc.o
+	$(AR) rs libutf8proc.a utf8proc.o

-libutf8proc.so: utf8proc.o
-	$(cc) -shared -o libutf8proc.so utf8proc.o
-	chmod a-x libutf8proc.so
+libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH): utf8proc.o
+	$(CC) $(LDFLAGS) -shared -o $@ -Wl,-soname -Wl,libutf8proc.so.$(MAJOR) utf8proc.o
+	chmod a-x $@

-libutf8proc.dylib: utf8proc.o
-	$(cc) -dynamiclib -o $@ $^ -install_name $(libdir)/$@
+libutf8proc.so: libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH)
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@
+	ln -f -s libutf8proc.so.$(MAJOR).$(MINOR).$(PATCH) $@.$(MAJOR)

-ruby/Makefile: ruby/extconf.rb
-	cd ruby && ruby extconf.rb
+libutf8proc.$(MAJOR).dylib: utf8proc.o
+	$(CC) -dynamiclib -o $@ $^ -install_name $(libdir)/$@ -Wl,-compatibility_version -Wl,$(MAJOR) -Wl,-current_version -Wl,$(MAJOR).$(MINOR).$(PATCH)

-ruby/utf8proc_native.so: utf8proc.h utf8proc.c utf8proc_data.c \
-		ruby/utf8proc_native.c ruby/Makefile
-	cd ruby && make
+libutf8proc.dylib: libutf8proc.$(MAJOR).dylib
+	ln -f -s libutf8proc.$(MAJOR).dylib $@

-ruby/gem/lib/utf8proc.rb: ruby/utf8proc.rb
-	test -e ruby/gem/lib || mkdir ruby/gem/lib
-	cp ruby/utf8proc.rb ruby/gem/lib/
+install: libutf8proc.a libutf8proc.$(SHLIB_EXT) libutf8proc.$(SHLIB_VERS_EXT)
+	mkdir -m 755 -p $(DESTDIR)$(includedir)
+	$(INSTALL) -m 644 utf8proc.h $(DESTDIR)$(includedir)
+	mkdir -m 755 -p $(DESTDIR)$(libdir)
+	$(INSTALL) -m 644 libutf8proc.a $(DESTDIR)$(libdir)
+	$(INSTALL) -m 755 libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.$(SHLIB_EXT)
+ifneq ($(OS),Darwin)
+	ln -f -s libutf8proc.$(SHLIB_VERS_EXT) $(DESTDIR)$(libdir)/libutf8proc.so.$(MAJOR)
+endif

-ruby/gem/ext/extconf.rb: ruby/extconf.rb
-	test -e ruby/gem/ext || mkdir ruby/gem/ext
-	cp ruby/extconf.rb ruby/gem/ext/
+MANIFEST.new:
+	rm -rf tmp
+	$(MAKE) install prefix=/usr DESTDIR=$(PWD)/tmp
+	$(FIND) tmp/usr -mindepth 1 -type l -printf "%P -> %l\n" -or -type f -printf "%P\n" -or -type d -printf "%P/\n" | LC_ALL=C sort > $@
+	rm -rf tmp

-ruby/gem/ext/utf8proc_native.c: utf8proc.h utf8proc_data.c utf8proc.c ruby/utf8proc_native.c
-	test -e ruby/gem/ext || mkdir ruby/gem/ext
-	cat utf8proc.h utf8proc_data.c utf8proc.c ruby/utf8proc_native.c | grep -v '#include "utf8proc.h"' | grep -v '#include "utf8proc_data.c"' | grep -v '#include "../utf8proc.c"' > ruby/gem/ext/utf8proc_native.c
+# Test programs

-ruby-gem:: ruby/gem/lib/utf8proc.rb ruby/gem/ext/extconf.rb ruby/gem/ext/utf8proc_native.c
-	cd ruby/gem && gem build utf8proc.gemspec
+data/NormalizationTest.txt:
+	$(MAKE) -C data NormalizationTest.txt

-pgsql/utf8proc_pgsql.so: utf8proc.h utf8proc.c utf8proc_data.c \
-		pgsql/utf8proc_pgsql.c
-	cd pgsql && make
+data/GraphemeBreakTest.txt:
+	$(MAKE) -C data GraphemeBreakTest.txt

+test/tests.o: test/tests.c test/tests.h utf8proc.h
+	$(CC) $(UCFLAGS) -c -o test/tests.o test/tests.c
+
+test/normtest: test/normtest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/normtest.c test/tests.o utf8proc.o -o $@
+
+test/graphemetest: test/graphemetest.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/graphemetest.c test/tests.o utf8proc.o -o $@
+
+test/printproperty: test/printproperty.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/printproperty.c test/tests.o utf8proc.o -o $@
+
+test/charwidth: test/charwidth.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/charwidth.c test/tests.o utf8proc.o -o $@
+
+test/valid: test/valid.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/valid.c test/tests.o utf8proc.o -o $@
+
+test/iterate: test/iterate.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/iterate.c test/tests.o utf8proc.o -o $@
+
+test/case: test/case.c test/tests.o utf8proc.o utf8proc.h test/tests.h
+	$(CC) $(UCFLAGS) test/case.c test/tests.o utf8proc.o -o $@
+
+check: test/normtest data/NormalizationTest.txt test/graphemetest data/GraphemeBreakTest.txt test/printproperty test/case test/charwidth test/valid test/iterate bench/bench.c bench/util.c bench/util.h utf8proc.o
+	$(MAKE) -C bench
+	test/normtest data/NormalizationTest.txt
+	test/graphemetest data/GraphemeBreakTest.txt
+	test/charwidth
+	test/valid
+	test/iterate
+	test/case
--- a/3rdparty/utf8proc/NEWS.md
+++ b/3rdparty/utf8proc/NEWS.md
@ -0,0 +1,281 @@
+# utf8proc release history #
+
+## Version 2.0.2 ##
+
+2016-07-27:
+
+- Move `-Wmissing-prototypes` warning flag from `Makefile` to `.travis.yml`
+  since MSVC does not understand this flag and it is occasionally useful to
+  build using MSVC through the `Makefile` ([#79]).
+
+- Use a different variable name for a nested loop in `bench/bench.c`, and
+  declare it in a C89 way rather than inside the `for` to avoid "error:
+  'for' loop initial declarations are only allowed in C99 mode" ([#80]).
+
+## Version 2.0.1 ##
+
+2016-07-13:
+
+- Bug fix in `utf8proc_grapheme_break_stateful` ([#77]).
+
+- Tests now use versioned Unicode files, so they will no longer
+  break when a new version of Unicode is released ([#78]).
+
+## Version 2.0 ##
+
+2016-07-13:
+
+- Updated for Unicode 9.0 ([#70]).
+
+- New `utf8proc_grapheme_break_stateful` to handle the complicated
+  grapheme-breaking rules in Unicode 9.  The old `utf8proc_grapheme_break`
+  is still provided, but may incorrectly identify grapheme breaks
+  in some Unicode-9 sequences.
+
+- Smaller Unicode tables ([#62], [#68]).  This required changes
+  in the `utf8proc_property_t` structure, which breaks backward
+  compatibility if you access this `struct` directly.  The
+  functions in the API remain backward-compatible, however.
+
+- Buffer overrun fix ([#66]).
+
+## Version 1.3.1 ##
+
+2015-11-02:
+
+- Do not export symbol for internal function `unsafe_encode_char()` ([#55]).
+
+- Install relative symbolic links for shared libraries ([#58]).
+
+- Enable and fix compiler warnings ([#55], [#58]).
+
+- Add missing files to `make clean` ([#58]).
+
+## Version 1.3 ##
+
+2015-07-06:
+
+- Updated for Unicode 8.0 ([#45]).
+
+- New `utf8proc_tolower` and `utf8proc_toupper` functions, portable
+  replacements for `towlower` and `towupper` in the C library ([#40]).
+
+- Don't treat Unicode "non-characters" as invalid, and improved
+  validity checking in general ([#35]).
+
+- Prefix all typedefs with `utf8proc_`, e.g. `utf8proc_int32_t`,
+  to avoid collisions with other libraries ([#32]).
+
+- Rename `DLLEXPORT` to `UTF8PROC_DLLEXPORT` to prevent collisions.
+
+- Fix build breakage in the benchmark routines.
+
+- More fine-grained Makefile variables (`PICFLAG` etcetera), so that
+  compilation flags can be selectively overridden, and in particular
+  so that `CFLAGS` can be changed without accidentally eliminating
+  necessary flags like `-fPIC` and `-std=c99` ([#43]).
+
+- Updated character-width tables based on Unifont 8.0.01 ([#51]) and
+  the Unicode 8 character categories ([#47]).
+
+## Version 1.2 ##
+
+2015-03-28:
+
+- Updated for Unicode 7.0 ([#6]).
+
+- New function `utf8proc_grapheme_break(c1,c2)` that returns whether
+  there is a grapheme break between `c1` and `c2` ([#20]).
+
+- New function `utf8proc_charwidth(c)` that returns the number of
+  column-positions that should be required for `c`; essentially a
+  portable replacment for `wcwidth(c)` ([#27]).
+
+- New function `utf8proc_category(c)` that returns the Unicode
+  category of `c` (as one of the constants `UTF8PROC_CATEGORY_xx`).
+  Also, a function `utf8proc_category_string(c)` that returns the Unicode
+  category of `c` as a two-character string.
+
+- `cmake` script `CMakeLists.txt`, in addition to `Makefile`, for
+  easier compilation on Windows ([#28]).
+
+- Various `Makefile` improvements: a `make check` target to perform
+  tests ([#13]), `make install`, a rule to automate updating the Unicode
+  tables, etcetera.
+
+- The shared library is now versioned (e.g. has a soname on GNU/Linux) ([#24]).
+
+- C++/MSVC compatibility ([#17]).
+
+- Most `#defined` constants are now `enums` ([#29]).
+
+- New preprocessor constants `UTF8PROC_VERSION_MAJOR`,
+  `UTF8PROC_VERSION_MINOR`, and `UTF8PROC_VERSION_PATCH` for compile-time
+  detection of the API version.
+
+- Doxygen-formatted documentation ([#29]).
+
+- The Ruby and PostgreSQL plugins have been removed due to lack of testing ([#22]).
+
+## Version 1.1.6 ##
+
+2013-11-27:
+
+- PostgreSQL 9.2 and 9.3 compatibility (lowercase `c` language name)
+
+## Version 1.1.5 ##
+
+2009-08-20:
+
+- Use `RSTRING_PTR()` and `RSTRING_LEN()` instead of `RSTRING()->ptr` and
+  `RSTRING()->len` for ruby1.9 compatibility (and `#define` them, if not
+  existent)
+
+2009-10-02:
+
+- Patches for compatibility with Microsoft Visual Studio
+
+2009-10-08:
+
+- Fixes to make utf8proc usable in C++ programs
+
+2009-10-16:
+
+## Version 1.1.4 ##
+
+2009-06-14:
+
+- replaced C++ style comments for compatibility reasons
+- added typecasts to suppress compiler warnings
+- removed redundant source files for ruby-gemfile generation
+
+2009-08-19:
+
+- Changed copyright notice for Public Software Group e. V.
+- Minor changes in the `README` file
+
+## Version 1.1.3 ##
+
+2008-10-04:
+
+- Added a function `utf8proc_version` returning a string containing the version
+  number of the library.
+- Included a target `libutf8proc.dylib` for MacOSX.
+
+2009-05-01:
+- PostgreSQL 8.3 compatibility (use of `SET_VARSIZE` macro)
+
+## Version 1.1.2 ##
+
+2007-07-25:
+
+- Fixed a serious bug in the data file generator, which caused characters
+  being treated incorrectly, when stripping default ignorable characters or
+  calculating grapheme cluster boundaries.
+
+## Version 1.1.1 ##
+
+2007-06-25:
+
+- Added a new PostgreSQL function `unistrip`, which behaves like `unifold`,
+  but also removes all character marks (e.g. accents).
+
+2007-07-22:
+
+- Changed license from BSD to MIT style.
+- Added a new function `utf8proc_codepoint_valid` to the C library.
+- Changed compiler flags in `Makefile` from `-g -O0` to `-O2`
+- The ruby script, which was used to build the `utf8proc_data.c` file, is now
+  included in the distribution.
+
+## Version 1.0.3 ##
+
+2007-03-16:
+
+- Fixed a bug in the ruby library, which caused an error, when splitting an
+  empty string at grapheme cluster boundaries (method `String#utf8chars`).
+
+## Version 1.0.2 ##
+
+2006-09-21:
+
+- included a check in `Integer#utf8`, which raises an exception, if the given
+  code-point is invalid because of being too high (this was missing yet)
+
+2006-12-26:
+
+- added support for PostgreSQL version 8.2
+
+## Version 1.0.1 ##
+
+2006-09-20:
+
+- included a gem file for the ruby version of the library
+
+Release of version 1.0.1
+
+## Version 1.0 ##
+
+2006-09-17:
+
+- added the `LUMP` option, which lumps certain characters together (see `lump.md`) (also used for the PostgreSQL `unifold` function)
+- added the `STRIPMARK` option, which strips marking characters (or marks of composed characters)
+- deprecated ruby method `String#char_ary` in favour of `String#utf8chars`
+
+## Version 0.3 ##
+
+2006-07-18:
+
+- changed normalization from NFC to NFKC for postgresql unifold function
+
+2006-08-04:
+
+- added support to mark the beginning of a grapheme cluster with 0xFF (option: `CHARBOUND`)
+- added the ruby method `String#chars`, which is returning an array of UTF-8 encoded grapheme clusters
+- added `NLF2LF` transformation in postgresql `unifold` function
+- added the `DECOMPOSE` option, if you neither use `COMPOSE` or `DECOMPOSE`, no normalization will be performed (different from previous versions)
+- using integer constants rather than C-strings for character properties
+- fixed (hopefully) a problem with the ruby library on Mac OS X, which occurred when compiler optimization was switched on
+
+## Version 0.2 ##
+
+2006-06-05:
+
+- changed behaviour of PostgreSQL function to return NULL in case of invalid input, rather than raising an exceptional condition
+- improved efficiency of PostgreSQL function (no transformation to C string is done)
+
+2006-06-20:
+
+- added -fpic compiler flag in Makefile
+- fixed bug in the C code for the ruby library (usage of non-existent function)
+
+## Version 0.1 ##
+
+2006-06-02: initial release of version 0.1
+
+[#6]: https://github.com/JuliaLang/utf8proc/issues/6
+[#13]: https://github.com/JuliaLang/utf8proc/issues/13
+[#17]: https://github.com/JuliaLang/utf8proc/issues/17
+[#20]: https://github.com/JuliaLang/utf8proc/issues/20
+[#22]: https://github.com/JuliaLang/utf8proc/issues/22
+[#24]: https://github.com/JuliaLang/utf8proc/issues/24
+[#27]: https://github.com/JuliaLang/utf8proc/issues/27
+[#28]: https://github.com/JuliaLang/utf8proc/issues/28
+[#29]: https://github.com/JuliaLang/utf8proc/issues/29
+[#32]: https://github.com/JuliaLang/utf8proc/issues/32
+[#35]: https://github.com/JuliaLang/utf8proc/issues/35
+[#40]: https://github.com/JuliaLang/utf8proc/issues/40
+[#43]: https://github.com/JuliaLang/utf8proc/issues/43
+[#45]: https://github.com/JuliaLang/utf8proc/issues/45
+[#47]: https://github.com/JuliaLang/utf8proc/issues/47
+[#51]: https://github.com/JuliaLang/utf8proc/issues/51
+[#55]: https://github.com/JuliaLang/utf8proc/issues/55
+[#58]: https://github.com/JuliaLang/utf8proc/issues/58
+[#62]: https://github.com/JuliaLang/utf8proc/issues/62
+[#66]: https://github.com/JuliaLang/utf8proc/issues/66
+[#68]: https://github.com/JuliaLang/utf8proc/issues/68
+[#70]: https://github.com/JuliaLang/utf8proc/issues/70
+[#77]: https://github.com/JuliaLang/utf8proc/issues/77
+[#78]: https://github.com/JuliaLang/utf8proc/issues/78
+[#79]: https://github.com/JuliaLang/utf8proc/issues/79
+[#80]: https://github.com/JuliaLang/utf8proc/issues/80
--- a/3rdparty/utf8proc/README
+++ b/3rdparty/utf8proc/README
@ -1,116 +0,0 @@
-
-Please read the LICENSE file, which is shipping with this software.
-
-
-*** QUICK START ***
-
-For compilation of the C library call "make c-library", for compilation of
-the ruby library call "make ruby-library" and for compilation of the
-PostgreSQL extension call "make pgsql-library".
-
-For ruby you can also create a gem-file by calling "make ruby-gem".
-
-"make all" can be used to build everything, but both ruby and PostgreSQL
-installations are required in this case.
-
-
-*** GENERAL INFORMATION ***
-
-The C library is found in this directory after successful compilation and
-is named "libutf8proc.a" and "libutf8proc.so". The ruby library consists of
-the files "utf8proc.rb" and "utf8proc_native.so", which are found in the
-subdirectory "ruby/". If you chose to create a gem-file it is placed in the
-"ruby/gem" directory. The PostgreSQL extension is named "utf8proc_pgsql.so"
-and resides in the "pgsql/" directory.
-
-Both the ruby library and the PostgreSQL extension are built as stand-alone
-libraries and are therefore not dependent the dynamic version of the
-C library files, but this behaviour might change in future releases.
-
-The Unicode version being supported is 5.0.0.
-Note: Version 4.1.0 of Unicode Standard Annex #29 was used, as
-      version 5.0.0 had not been available at the time of implementation.
-
-For Unicode normalizations, the following options have to be used:
-Normalization Form C:  STABLE, COMPOSE
-Normalization Form D:  STABLE, DECOMPOSE
-Normalization Form KC: STABLE, COMPOSE, COMPAT
-Normalization Form KD: STABLE, DECOMPOSE, COMPAT
-
-
-*** C LIBRARY ***
-
-The documentation for the C library is found in the utf8proc.h header file.
-"utf8proc_map" is most likely function you will be using for mapping UTF-8
-strings, unless you want to allocate memory yourself.
-
-
-*** RUBY API ***
-
-The ruby library adds the methods "utf8map" and "utf8map!" to the String
-class, and the method "utf8" to the Integer class.
-
-The String#utf8map method does the same as the "utf8proc_map" C function.
-Options for the mapping procedure are passed as symbols, i.e:
-"Hello".utf8map(:casefold) => "hello"
-
-The descriptions of all options are found in the C header file
-"utf8proc.h". Please notice that the according symbols in ruby are all
-lowercase.
-
-String#utf8map! is the destructive function in the meaning that the string
-is replaced by the result.
-
-There are shortcuts for the 4 normalization forms specified by Unicode:
-String#utf8nfd,  String#utf8nfd!,
-String#utf8nfc,  String#utf8nfc!,
-String#utf8nfkd, String#utf8nfkd!,
-String#utf8nfkc, String#utf8nfkc!
-
-The method Integer#utf8 returns a UTF-8 string, which is containing the
-unicode char given by the code point.
-0x000A.utf8 => "\n"
-0x2028.utf8 => "\342\200\250"
-
-
-*** POSTGRESQL API ***
-
-For PostgreSQL there are two SQL functions supplied named "unifold" and
-"unistrip". These functions function can be used to prepare index fields in
-order to be folded in a way where string-comparisons make more sense, e.g.
-where "bathtub" == "bath<soft hyphen>tub"
-or "Hello World" == "hello world".
-
-CREATE TABLE people (
-  id    serial8 primary key,
-  name  text,
-  CHECK (unifold(name) NOTNULL)
-);
-CREATE INDEX name_idx ON people (unifold(name));
-SELECT * FROM people WHERE unifold(name) = unifold('John Doe');
-
-The function "unistrip" removes character marks like accents or diaeresis,
-while "unifold" keeps then.
-
-NOTICE: The outputs of the function can change between releases, as
-        utf8proc does not follow a versioning stability policy. You have to
-        rebuild your database indicies, if you upgrade to a newer version
-        of utf8proc.
-
-
-*** TODO ***
-
- detect stable code points and process segments independently in order to
-  save memory
- do a quick check before normalizing strings to optimize speed
- support stream processing
-
-
-*** CONTACT ***
-
-If you find any bugs or experience difficulties in compiling this software,
-please contact us:
-
-Project page: http://www.public-software-group.org/utf8proc
-
-
--- a/3rdparty/utf8proc/README.md
+++ b/3rdparty/utf8proc/README.md
@ -0,0 +1,69 @@
+# utf8proc
+[![Travis CI Status](https://travis-ci.org/JuliaLang/utf8proc.png)](https://travis-ci.org/JuliaLang/utf8proc)
+[![AppVeyor Status](https://ci.appveyor.com/api/projects/status/aou20lfkyhj8xbwq/branch/master?svg=true)](https://ci.appveyor.com/project/tkelman/utf8proc/branch/master)
+
+
+[utf8proc](http://julialang.org/utf8proc/) is a small, clean C
+library that provides Unicode normalization, case-folding, and other
+operations for data in the [UTF-8
+encoding](http://en.wikipedia.org/wiki/UTF-8).  It was [initially
+developed](http://www.public-software-group.org/utf8proc) by Jan
+Behrens and the rest of the [Public Software
+Group](http://www.public-software-group.org/), who deserve *nearly all
+of the credit* for this package.  With the blessing of the Public
+Software Group, the [Julia developers](http://julialang.org/) have
+taken over development of utf8proc, since the original developers have
+moved to other projects.
+
+(utf8proc is used for basic Unicode
+support in the [Julia language](http://julialang.org/), and the Julia
+developers became involved because they wanted to add Unicode 7 support and other features.)
+
+(The original utf8proc package also includes Ruby and PostgreSQL plug-ins.
+We removed those from utf8proc in order to focus exclusively on the C
+library for the time being, but plan to add them back in or release them as separate packages.)
+
+The utf8proc package is licensed under the
+free/open-source [MIT "expat"
+license](http://opensource.org/licenses/MIT) (plus certain Unicode
+data governed by the similarly permissive [Unicode data
+license](http://www.unicode.org/copyright.html#Exhibit1)); please see
+the included `LICENSE.md` file for more detailed information.
+
+## Quick Start
+
+For compilation of the C library run `make`.
+
+## General Information
+
+The C library is found in this directory after successful compilation
+and is named `libutf8proc.a` (for the static library) and
+`libutf8proc.so` (for the dynamic library).
+
+The Unicode version supported is 9.0.0.
+
+For Unicode normalizations, the following options are used:
+
+* Normalization Form C:  `STABLE`, `COMPOSE`
+* Normalization Form D:  `STABLE`, `DECOMPOSE`
+* Normalization Form KC: `STABLE`, `COMPOSE`, `COMPAT`
+* Normalization Form KD: `STABLE`, `DECOMPOSE`, `COMPAT`
+
+## C Library
+
+The documentation for the C library is found in the `utf8proc.h` header file.
+`utf8proc_map` is function you will most likely be using for mapping UTF-8
+strings, unless you want to allocate memory yourself.
+
+## To Do
+
+See the Github [issues list](https://github.com/JuliaLang/utf8proc/issues).
+
+## Contact
+
+Bug reports, feature requests, and other queries can be filed at
+the [utf8proc issues page on Github](https://github.com/JuliaLang/utf8proc/issues).
+
+## See also
+
+An independent Lua translation of this library, [lua-mojibake](https://github.com/differentprogramming/lua-mojibake), is also available.
--- a/3rdparty/utf8proc/appveyor.yml
+++ b/3rdparty/utf8proc/appveyor.yml
@ -0,0 +1,42 @@
+branches:
+  only:
+    - master
+    - /release-.*/
+
+notifications:
+  - provider: Email
+    on_build_success: false
+    on_build_failure: false
+    on_build_status_changed: false
+
+build_script:
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+      https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+      Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+      throw "There are newer queued builds for this pull request, failing early." }
+  - mkdir msvc_static
+  - cd msvc_static
+  - cmake ..
+  - cmake --build .
+  - mkdir ..\msvc_shared
+  - cd ..\msvc_shared
+  - cmake .. -DBUILD_SHARED_LIBS=ON
+  - cmake --build .
+  - set PATH=C:\MinGW\bin;%PATH%
+  - C:\MinGW\msys\1.0\bin\sh --login -c "
+      cd /c/projects/utf8proc &&
+      mkdir mingw_static &&
+      cd mingw_static &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -G'MSYS Makefiles' &&
+      make &&
+      mkdir ../mingw_shared &&
+      cd ../mingw_shared &&
+      cmake .. -DCMAKE_VERBOSE_MAKEFILE=ON -DBUILD_SHARED_LIBS=ON -G'MSYS Makefiles' &&
+      make
+      "
+
+on_finish:
+  # Uncomment the following line for interactive debugging, which
+  # will print login data for a temporary remote session after the
+  # build. This requires an RDP version 6 client, e.g., FreeRDP.
+  #- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
--- a/3rdparty/utf8proc/bench/Makefile
+++ b/3rdparty/utf8proc/bench/Makefile
@ -0,0 +1,39 @@
+CURL=curl
+
+CC = cc
+CFLAGS = -O2 -std=c99 -pedantic -Wall
+
+all: bench
+
+LIBUTF8PROC = ../utf8proc.o
+
+bench: bench.o util.o $(LIBUTF8PROC)
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ bench.o util.o $(LIBUTF8PROC)
+
+DATAURL = https://raw.githubusercontent.com/duerst/eprun/master/benchmark
+DATAFILES = Deutsch_.txt Japanese_.txt Korean_.txt Vietnamese_.txt
+
+$(DATAFILES):
+	$(CURL) -O $(DATAURL)/$@
+
+bench.out: $(DATAFILES) bench
+	./bench -nfkc $(DATAFILES) > $@
+
+# you may need make CPPFLAGS=... LDFLAGS=... to help it find ICU
+icu: icu.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ icu.o util.o -licuuc
+
+icu.out: $(DATAFILES) icu
+	./icu $(DATAFILES) > $@
+
+unistring: unistring.o util.o
+	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ unistring.o util.o -lunistring
+
+unistring.out: $(DATAFILES) unistring
+	./unistring $(DATAFILES) > $@
+
+.c.o:
+	$(CC) $(CPPFLAGS) -I.. $(CFLAGS) -c -o $@ $<
+
+clean:
+	rm -rf *.o *.txt bench *.out icu unistring
--- a/3rdparty/utf8proc/bench/bench.c
+++ b/3rdparty/utf8proc/bench/bench.c
@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "utf8proc.h"
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i, j;
+	 int options = 0;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE|UTF8PROC_COMPAT;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_COMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   options |= UTF8PROC_STABLE|UTF8PROC_DECOMPOSE;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-casefold")) {
+			   options |= UTF8PROC_CASEFOLD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (j = 0; j < 100; ++j) {
+			   utf8proc_map(src, len, &dest, options);
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/icu.c
+++ b/3rdparty/utf8proc/bench/icu.c
@ -0,0 +1,61 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+/* ICU4C */
+#include <unicode/utypes.h>
+#include <unicode/ustring.h>
+#include <unicode/ucnv.h>
+#include <unicode/unorm2.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+
+	 UErrorCode err;
+	 UConverter *uc = ucnv_open("UTF8", &err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+
+	 const UNormalizer2 *NFKC = unorm2_getNFKCInstance(&err);
+	 if (U_FAILURE(err)) return EXIT_FAILURE;
+	 
+	 for (i = 1; i < argc; ++i) {
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  /* convert UTF8 data to ICU's UTF16 */
+		  UChar *usrc = (UChar*) malloc(2*len * sizeof(UChar));
+		  ucnv_toUChars(uc, usrc, 2*len, (char*) src, len, &err);
+		  if (U_FAILURE(err)) return EXIT_FAILURE;
+		  size_t ulen = u_strlen(usrc);
+
+		  /* ICU's insane normalization API requires you to
+			 know the size of the destination buffer in advance,
+			 or alternatively to repeatly try normalizing and
+			 double the buffer size until it succeeds.  Here, I just
+			 allocate a huge destination buffer to avoid the issue. */
+		  UChar *udest = (UChar*) malloc(10*ulen * sizeof(UChar));
+
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   unorm2_normalize(NFKC, usrc, ulen, udest, 10*ulen, &err);
+			   if (U_FAILURE(err)) return EXIT_FAILURE;
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(udest);
+		  free(usrc);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/unistring.c
+++ b/3rdparty/utf8proc/bench/unistring.c
@ -0,0 +1,60 @@
+/* comparitive benchmark of GNU libunistring */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* libunistring */
+#include <unistr.h>
+#include <uninorm.h>
+
+#include "util.h"
+
+int main(int argc, char **argv)
+{
+	 int i;
+	 uninorm_t nf = UNINORM_NFKC;
+
+	 for (i = 1; i < argc; ++i) {
+		  if (!strcmp(argv[i], "-nfkc")) {
+			   nf = UNINORM_NFKC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfkd")) {
+			   nf = UNINORM_NFKD;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfc")) {
+			   nf = UNINORM_NFC;
+			   continue;
+		  }
+		  if (!strcmp(argv[i], "-nfd")) {
+			   nf = UNINORM_NFD;
+			   continue;
+		  }
+		  if (argv[i][0] == '-') {
+			   fprintf(stderr, "unrecognized option: %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t len;
+		  uint8_t *src = readfile(argv[i], &len);
+		  if (!src) {
+			   fprintf(stderr, "error reading %s\n", argv[i]);
+			   return EXIT_FAILURE;
+		  }
+
+		  size_t destlen;
+		  uint8_t *dest;
+		  mytime start = gettime();
+		  for (int i = 0; i < 100; ++i) {
+			   dest = u8_normalize(nf, src, len, NULL, &destlen);
+			   if (!dest) return EXIT_FAILURE;
+			   free(dest);
+		  }
+		  printf("%s: %g\n", argv[i], elapsed(gettime(), start) / 100);
+		  free(src);
+	 }
+
+	 return EXIT_SUCCESS;
+}
--- a/3rdparty/utf8proc/bench/util.c
+++ b/3rdparty/utf8proc/bench/util.c
@ -0,0 +1,39 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#include "util.h"
+
+/* read file named FILENAME into an array of *len bytes,
+   returning NULL on error */
+uint8_t *readfile(const char *filename, size_t *len)
+{
+	 *len = 0;
+	 struct stat st;
+	 if (0 != stat(filename, &st)) return NULL;
+	 *len = st.st_size;
+	 FILE *f = fopen(filename, "r");
+	 if (!f) return NULL;
+	 uint8_t *s = (uint8_t *) malloc(sizeof(uint8_t) * *len);
+	 if (!s) return NULL;
+	 if (fread(s, 1, *len, f) != *len) {
+		  free(s);
+		  s = NULL;
+	 }
+	 fclose(f);
+	 return s;
+}
+
+mytime gettime(void) {
+	 mytime t;
+	 gettimeofday(&t, NULL);
+	 return t;
+}
+
+/* time difference in seconds */
+double elapsed(mytime t1, mytime t0)
+{
+     return (double)(t1.tv_sec - t0.tv_sec) +
+          (double)(t1.tv_usec - t0.tv_usec) * 1.0E-6;
+}
+
--- a/3rdparty/utf8proc/bench/util.h
+++ b/3rdparty/utf8proc/bench/util.h
@ -0,0 +1,22 @@
+#ifndef UTIL_H
+#define UTIL_H 1
+
+#include <inttypes.h>
+#include <sys/time.h>
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+uint8_t *readfile(const char *filename, size_t *len);
+
+typedef struct timeval mytime;
+mytime gettime(void);
+double elapsed(mytime t1, mytime t0);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* UTIL_H */
--- a/3rdparty/utf8proc/data/Makefile
+++ b/3rdparty/utf8proc/data/Makefile
@ -0,0 +1,66 @@
+# Unicode data generation rules.  Except for the test data files, most
+# users will not use these Makefile rules, which are primarily to re-generate
+# unicode_data.c when we get a new Unicode version or charwidth data; they
+# require ruby, fontforge, and julia to be installed.
+
+# programs
+CURL=curl
+RUBY=ruby
+PERL=perl
+MAKE=make
+JULIA=julia
+FONTFORGE=fontforge
+CURLFLAGS = --retry 5 --location
+
+.PHONY: clean
+
+.DELETE_ON_ERROR:
+
+utf8proc_data.c.new: data_generator.rb UnicodeData.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt CharWidths.txt
+	$(RUBY) data_generator.rb < UnicodeData.txt > $@
+
+# GNU Unifont version for font metric calculations:
+UNIFONT_VERSION=9.0.01
+
+unifont.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont-$(UNIFONT_VERSION).ttf
+
+unifont_upper.ttf:
+	$(CURL) $(CURLFLAGS) -o $@ $(URLCACHE)https://mirrors.kernel.org/gnu/unifont/unifont-$(UNIFONT_VERSION)/unifont_upper-$(UNIFONT_VERSION).ttf
+
+%.sfd: %.ttf
+	$(FONTFORGE) -lang=ff -c "Open(\"$<\");Save(\"$@\");Quit(0);"
+
+CharWidths.txt: charwidths.jl unifont.sfd unifont_upper.sfd EastAsianWidth.txt
+	$(JULIA) charwidths.jl > $@
+
+# Unicode data version
+UNICODE_VERSION=9.0.0
+
+UnicodeData.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/UnicodeData.txt
+
+EastAsianWidth.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/EastAsianWidth.txt
+
+GraphemeBreakProperty.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakProperty.txt
+
+DerivedCoreProperties.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/DerivedCoreProperties.txt
+
+CompositionExclusions.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CompositionExclusions.txt
+
+CaseFolding.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/CaseFolding.txt
+
+NormalizationTest.txt:
+	$(CURL) $(CURLFLAGS) -o $@ -O $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/NormalizationTest.txt
+
+GraphemeBreakTest.txt:
+	$(CURL) $(CURLFLAGS) $(URLCACHE)http://www.unicode.org/Public/$(UNICODE_VERSION)/ucd/auxiliary/GraphemeBreakTest.txt | $(PERL) -pe 's,÷,/,g;s,×,+,g' > $@
+
+clean:
+	rm -f UnicodeData.txt EastAsianWidth.txt GraphemeBreakProperty.txt DerivedCoreProperties.txt CompositionExclusions.txt CaseFolding.txt NormalizationTest.txt GraphemeBreakTest.txt CharWidths.txt unifont*.ttf unifont*.sfd
+	rm -f utf8proc_data.c.new
--- a/3rdparty/utf8proc/data/charwidths.jl
+++ b/3rdparty/utf8proc/data/charwidths.jl
@ -0,0 +1,190 @@
+# Following work by @jiahao, we compute character widths using a combination of
+#   * advance widths from GNU Unifont (advance width 512 = 1 en)
+#   * UAX 11: East Asian Width
+#   * a few exceptions as needed
+# Adapted from http://nbviewer.ipython.org/gist/jiahao/07e8b08bf6d8671e9734
+#
+# Requires Julia (obviously) and FontForge.
+
+#############################################################################
+# Julia 0.3/0.4 compatibility (taken from Compat package)
+if VERSION < v"0.4.0-dev+1387"
+    typealias AbstractString String
+end
+if VERSION < v"0.4.0-dev+1419"
+    const UInt32 = Uint32
+end
+if VERSION < v"0.4.0-dev+3874"
+    Base.parse{T<:Integer}(::Type{T}, s::AbstractString) = parseint(T, s)
+end
+
+CharWidths = Dict{Int,Int}()
+
+#############################################################################
+# Use ../libutf8proc for category codes, rather than the one in Julia,
+# to minimize bootstrapping complexity when a new version of Unicode comes out.
+catcode(c) = ccall((:utf8proc_category,"../libutf8proc"), Cint, (Int32,), c)
+
+# use Base.UTF8proc module to get category codes constants, since
+# we won't change these in utf8proc.
+import Base.UTF8proc
+
+#############################################################################
+# Use a default width of 1 for all character categories that are
+# letter/symbol/number-like.  This can be overriden by Unifont or UAX 11
+# below, but provides a useful nonzero fallback for new codepoints when
+# a new Unicode version has been released but Unifont hasn't been updated yet.
+
+zerowidth = Set{Int}() # categories that may contain zero-width chars
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MN)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_MC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ME)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_SK)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZL)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_ZP)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CC)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CF)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CS)
+push!(zerowidth, UTF8proc.UTF8PROC_CATEGORY_CO)
+for c in 0x0000:0x110000
+    if catcode(c) ∉ zerowidth
+        CharWidths[c] = 1
+    end
+end
+
+#############################################################################
+# Widths from GNU Unifont
+
+#Read sfdfile for character widths
+function parsesfd(filename::AbstractString, CharWidths::Dict{Int,Int}=Dict{Int,Int}())
+    state=:seekchar
+    lineno = 0
+    codepoint = width = nothing
+    for line in readlines(open(filename))
+        lineno += 1
+        if state==:seekchar         #StartChar: nonmarkingreturn
+            if contains(line, "StartChar: ")
+                codepoint = nothing
+                width = nothing
+                state = :readdata
+            end
+        elseif state==:readdata #Encoding: 65538 -1 2, Width: 1024
+            contains(line, "Encoding:") && (codepoint = parse(Int, split(line)[3]))
+            contains(line, "Width:") && (width = parse(Int, split(line)[2]))
+            if codepoint!=nothing && width!=nothing && codepoint >= 0
+                w=div(width, 512) # 512 units to the en
+                if w > 0
+                    # only add nonzero widths, since (1) the default is zero
+                    # and (2) this circumvents some apparent bugs in Unifont
+                    # (https://savannah.gnu.org/bugs/index.php?45395)
+                    CharWidths[codepoint] = w
+                end
+                state = :seekchar
+            end
+        end
+    end
+    CharWidths
+end
+CharWidths=parsesfd("unifont.sfd", CharWidths)
+CharWidths=parsesfd("unifont_upper.sfd", CharWidths)
+
+#############################################################################
+# Widths from UAX #11: East Asian Width
+#   .. these take precedence over the Unifont width for all codepoints
+#      listed explicitly as wide/full/narrow/half-width
+
+for line in readlines(open("EastAsianWidth.txt"))
+    #Strip comments
+    line[1] == '#' && continue
+    precomment = split(line, '#')[1]
+    #Parse code point range and width code
+    tokens = split(precomment, ';')
+    length(tokens) >= 2 || continue
+    charrange = tokens[1]
+    width = strip(tokens[2])
+    #Parse code point range into Julia UnitRange
+    rangetokens = split(charrange, "..")
+    charstart = parse(UInt32, "0x"*rangetokens[1])
+    charend = parse(UInt32, "0x"*rangetokens[length(rangetokens)>1 ? 2 : 1])
+
+    #Assign widths
+    for c in charstart:charend
+        if width=="W" || width=="F" # wide or full
+            CharWidths[c]=2
+        elseif width=="Na"|| width=="H" # narrow or half
+            CharWidths[c]=1
+        end
+    end
+end
+
+#############################################################################
+# A few exceptions to the above cases, found by manual comparison
+# to other wcwidth functions and similar checks.
+
+for c in keys(CharWidths)
+    cat = catcode(c)
+
+    # make sure format control character (category Cf) have width 0,
+    # except for the Arabic characters 0x06xx (see unicode std 6.2, sec. 8.2)
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CF && c ∉ [0x0601,0x0602,0x0603,0x06dd]
+        CharWidths[c]=0
+    end
+
+    # Unifont has nonzero width for a number of non-spacing combining
+    # characters, e.g. (in 7.0.06): f84,17b4,17b5,180b,180d,2d7f, and
+    # the variation selectors
+    if cat==UTF8proc.UTF8PROC_CATEGORY_MN
+        CharWidths[c]=0
+    end
+
+    # We also assign width of zero to unassigned and private-use
+    # codepoints (Unifont includes ConScript Unicode Registry PUA fonts,
+    # but since these are nonstandard it seems questionable to recognize them).
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CO || cat==UTF8proc.UTF8PROC_CATEGORY_CN
+        CharWidths[c]=0
+    end
+
+    # for some reason, Unifont has width-2 glyphs for ASCII control chars
+    if cat==UTF8proc.UTF8PROC_CATEGORY_CC
+        CharWidths[c]=0
+    end
+end
+
+#By definition, should have zero width (on the same line)
+#0x002028 ' ' category: Zl name: LINE SEPARATOR/
+#0x002029 ' ' category: Zp name: PARAGRAPH SEPARATOR/
+CharWidths[0x2028]=0
+CharWidths[0x2029]=0
+
+#By definition, should be narrow = width of 1 en space
+#0x00202f ' ' category: Zs name: NARROW NO-BREAK SPACE/
+CharWidths[0x202f]=1
+
+#By definition, should be wide = width of 1 em space
+#0x002001 ' ' category: Zs name: EM QUAD/
+#0x002003 ' ' category: Zs name: EM SPACE/
+CharWidths[0x2001]=2
+CharWidths[0x2003]=2
+
+#############################################################################
+# Output (to a file or pipe) for processing by data_generator.rb
+# ... don't bother to output zero widths since that will be the default.
+
+firstc = 0x000000
+lastv = 0
+uhex(c) = uppercase(hex(c,4))
+for c in 0x0000:0x110000
+    v = get(CharWidths, c, 0)
+    if v != lastv || c == 0x110000
+        v < 4 || error("invalid charwidth $v for $c")
+        if firstc+1 < c
+            println(uhex(firstc), "..", uhex(c-1), "; ", lastv)
+        else
+            println(uhex(firstc), "; ", lastv)
+        end
+        firstc = c
+        lastv = v
+    end
+end
--- a/3rdparty/utf8proc/data/data_generator.rb
+++ b/3rdparty/utf8proc/data/data_generator.rb
@ -0,0 +1,411 @@
+#!/usr/bin/env ruby
+
+#  This file was used to generate the 'unicode_data.c' file by parsing the
+#  Unicode data file 'UnicodeData.txt' of the Unicode Character Database.
+#  It is included for informational purposes only and not intended for
+#  production use.
+
+
+#  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of this software and associated documentation files (the "Software"),
+#  to deal in the Software without restriction, including without limitation
+#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
+#  and/or sell copies of the Software, and to permit persons to whom the
+#  Software is furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+#  DEALINGS IN THE SOFTWARE.
+
+
+#  This file contains derived data from a modified version of the
+#  Unicode data files. The following license applies to that data:
+#
+#  COPYRIGHT AND PERMISSION NOTICE
+#
+#  Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
+#  under the Terms of Use in http://www.unicode.org/copyright.html.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a
+#  copy of the Unicode data files and any associated documentation (the "Data
+#  Files") or Unicode software and any associated documentation (the
+#  "Software") to deal in the Data Files or Software without restriction,
+#  including without limitation the rights to use, copy, modify, merge,
+#  publish, distribute, and/or sell copies of the Data Files or Software, and
+#  to permit persons to whom the Data Files or Software are furnished to do
+#  so, provided that (a) the above copyright notice(s) and this permission
+#  notice appear with all copies of the Data Files or Software, (b) both the
+#  above copyright notice(s) and this permission notice appear in associated
+#  documentation, and (c) there is clear notice in each modified Data File or
+#  in the Software as well as in the documentation associated with the Data
+#  File(s) or Software that the data or software has been modified.
+#
+#  THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+#  KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
+#  THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
+#  INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
+#  CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
+#  USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+#  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+#  PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+#
+#  Except as contained in this notice, the name of a copyright holder shall
+#  not be used in advertising or otherwise to promote the sale, use or other
+#  dealings in these Data Files or Software without prior written
+#  authorization of the copyright holder.
+
+
+$ignorable_list = File.read("DerivedCoreProperties.txt")[/# Derived Property: Default_Ignorable_Code_Point.*?# Total code points:/m]
+$ignorable = []
+$ignorable_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)/
+    $1.hex.upto($2.hex) { |e2| $ignorable << e2 }
+  elsif entry =~ /^[0-9A-F]+/
+    $ignorable << $&.hex
+  end
+end
+
+$grapheme_boundclass_list = File.read("GraphemeBreakProperty.txt")
+$grapheme_boundclass = Hash.new("UTF8PROC_BOUNDCLASS_OTHER")
+$grapheme_boundclass_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $1.hex.upto($2.hex) { |e2| $grapheme_boundclass[e2] = "UTF8PROC_BOUNDCLASS_" + $3.upcase }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([A-Za-z_]+)/
+    $grapheme_boundclass[$1.hex] = "UTF8PROC_BOUNDCLASS_" + $2.upcase
+  end
+end
+
+$charwidth_list = File.read("CharWidths.txt")
+$charwidth = Hash.new(0)
+$charwidth_list.each_line do |entry|
+  if entry =~ /^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*([0-9]+)/
+    $1.hex.upto($2.hex) { |e2| $charwidth[e2] = $3.to_i }
+  elsif entry =~ /^([0-9A-F]+)\s*;\s*([0-9]+)/
+    $charwidth[$1.hex] = $2.to_i
+  end
+end
+
+$exclusions = File.read("CompositionExclusions.txt")[/# \(1\) Script Specifics.*?# Total code points:/m]
+$exclusions = $exclusions.chomp.split("\n").collect { |e| e.hex }
+
+$excl_version = File.read("CompositionExclusions.txt")[/# \(2\) Post Composition Version precomposed characters.*?# Total code points:/m]
+$excl_version = $excl_version.chomp.split("\n").collect { |e| e.hex }
+
+$case_folding_string = File.open("CaseFolding.txt", :encoding => 'utf-8').read
+$case_folding = {}
+$case_folding_string.chomp.split("\n").each do |line|
+  next unless line =~ /([0-9A-F]+); [CFS]; ([0-9A-F ]+);/i
+  $case_folding[$1.hex] = $2.split(" ").collect { |e| e.hex }
+end
+
+$int_array = []
+$int_array_indicies = {}
+
+def str2c(string, prefix)
+  return "0" if string.nil?
+  return "UTF8PROC_#{prefix}_#{string.upcase}"
+end
+def pushary(array)
+  idx = $int_array_indicies[array]
+  unless idx
+    $int_array_indicies[array] = $int_array.length
+    idx = $int_array.length
+    array.each { |entry| $int_array << entry }
+  end
+  return idx
+end
+def cpary2utf16encoded(array)
+  return array.flat_map { |cp|
+      if (cp <= 0xFFFF)
+        raise "utf-16 code: #{cp}" if cp & 0b1111100000000000 == 0b1101100000000000
+        cp
+      else
+        temp = cp - 0x10000
+        [(temp >> 10) | 0b1101100000000000, (temp & 0b0000001111111111) | 0b1101110000000000]
+      end
+    }
+end
+def cpary2c(array)
+  return "UINT16_MAX" if array.nil? || array.length == 0
+  lencode = array.length - 1 #no sequence has len 0, so we encode len 1 as 0, len 2 as 1, ... 
+  array = cpary2utf16encoded(array)
+  if lencode >= 7 #we have only 3 bits for the length (which is already cutting it close. might need to change it to 2 bits in future Unicode versions)
+    array = [lencode] + array 
+    lencode = 7
+  end  
+  idx = pushary(array) 
+  raise "Array index out of bound" if idx > 0x1FFF
+  return "#{idx | (lencode << 13)}"
+end
+def singlecpmap(cp)
+  return "UINT16_MAX" if cp == nil
+  idx = pushary(cpary2utf16encoded([cp]))
+  raise "Array index out of bound" if idx > 0xFFFF
+  return "#{idx}"
+end
+
+class UnicodeChar
+  attr_accessor :code, :name, :category, :combining_class, :bidi_class,
+                :decomp_type, :decomp_mapping,
+                :bidi_mirrored,
+                :uppercase_mapping, :lowercase_mapping, :titlecase_mapping,
+                #caches:
+                :c_entry_index, :c_decomp_mapping, :c_case_folding
+  def initialize(line)
+    raise "Could not parse input." unless line =~ /^
+      ([0-9A-F]+);        # code
+      ([^;]+);            # name
+      ([A-Z]+);           # general category
+      ([0-9]+);           # canonical combining class
+      ([A-Z]+);           # bidi class
+      (<([A-Z]*)>)?       # decomposition type
+      ((\ ?[0-9A-F]+)*);  # decompomposition mapping
+      ([0-9]*);           # decimal digit
+      ([0-9]*);           # digit
+      ([^;]*);            # numeric
+      ([YN]*);            # bidi mirrored
+      ([^;]*);            # unicode 1.0 name
+      ([^;]*);            # iso comment
+      ([0-9A-F]*);        # simple uppercase mapping
+      ([0-9A-F]*);        # simple lowercase mapping
+      ([0-9A-F]*)$/ix     # simple titlecase mapping
+    @code              = $1.hex
+    @name              = $2
+    @category          = $3
+    @combining_class   = Integer($4)
+    @bidi_class        = $5
+    @decomp_type       = $7
+    @decomp_mapping    = ($8=='') ? nil :
+                         $8.split.collect { |element| element.hex }
+    @bidi_mirrored     = ($13=='Y') ? true : false
+    @uppercase_mapping = ($16=='') ? nil : $16.hex
+    @lowercase_mapping = ($17=='') ? nil : $17.hex
+    @titlecase_mapping = ($18=='') ? nil : $18.hex
+  end
+  def case_folding
+    $case_folding[code]
+  end
+  def c_entry(comb_indicies)
+    "  " <<
+    "{#{str2c category, 'CATEGORY'}, #{combining_class}, " <<
+    "#{str2c bidi_class, 'BIDI_CLASS'}, " <<
+    "#{str2c decomp_type, 'DECOMP_TYPE'}, " <<
+    "#{c_decomp_mapping}, " <<
+    "#{c_case_folding}, " <<
+    "#{singlecpmap uppercase_mapping }, " <<
+    "#{singlecpmap lowercase_mapping }, " <<
+    "#{singlecpmap titlecase_mapping }, " <<
+    "#{comb_indicies[code] ? comb_indicies[code]: 'UINT16_MAX'}, " <<
+    "#{bidi_mirrored}, " <<
+    "#{$exclusions.include?(code) or $excl_version.include?(code)}, " <<
+    "#{$ignorable.include?(code)}, " <<
+    "#{%W[Zl Zp Cc Cf].include?(category) and not [0x200C, 0x200D].include?(category)}, " <<
+    "#{$charwidth[code]}, 0, " <<
+    "#{$grapheme_boundclass[code]}},\n"
+  end
+end
+
+chars = []
+char_hash = {}
+
+while gets
+  if $_ =~ /^([0-9A-F]+);<[^;>,]+, First>;/i
+    first = $1.hex
+    gets
+    char = UnicodeChar.new($_)
+    raise "No last character of sequence found." unless
+      $_ =~ /^([0-9A-F]+);<([^;>,]+), Last>;/i
+    last = $1.hex
+    name = "<#{$2}>"
+    for i in first..last
+      char_clone = char.clone
+      char_clone.code = i
+      char_clone.name = name
+      char_hash[char_clone.code] = char_clone
+      chars << char_clone
+    end
+  else
+    char = UnicodeChar.new($_)
+    char_hash[char.code] = char
+    chars << char
+  end
+end
+
+comb1st_indicies = {}
+comb2nd_indicies = {}
+comb2nd_indicies_sorted_keys = []
+comb2nd_indicies_nonbasic = {}
+comb_array = []
+
+chars.each do |char|
+  if !char.nil? and char.decomp_type.nil? and char.decomp_mapping and
+      char.decomp_mapping.length == 2 and !char_hash[char.decomp_mapping[0]].nil? and
+      char_hash[char.decomp_mapping[0]].combining_class == 0 and
+      not $exclusions.include?(char.code)
+
+    dm0 = char.decomp_mapping[0]
+    dm1 = char.decomp_mapping[1]
+    unless comb1st_indicies[dm0]
+      comb1st_indicies[dm0] = comb1st_indicies.keys.length
+    end
+    unless comb2nd_indicies[dm1]
+      comb2nd_indicies_sorted_keys << dm1
+      comb2nd_indicies[dm1] = comb2nd_indicies.keys.length 
+    end
+    comb_array[comb1st_indicies[dm0]] ||= []
+    raise "Duplicate canonical mapping: #{char.code} #{dm0} #{dm1}" if comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]]
+    comb_array[comb1st_indicies[dm0]][comb2nd_indicies[dm1]] = char.code
+    
+    comb2nd_indicies_nonbasic[dm1] = true if char.code > 0xFFFF
+  end
+  char.c_decomp_mapping = cpary2c(char.decomp_mapping)
+  char.c_case_folding = cpary2c(char.case_folding)
+end 
+
+comb_indicies = {}
+cumoffset = 0
+comb1st_indicies_lastoffsets = []
+comb1st_indicies_firstoffsets = []
+comb1st_indicies.each do |dm0, index|
+  first = nil
+  last = nil
+  offset = 0
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    if comb_array[index][b] 
+      first = offset unless first
+      last = offset
+      last += 1 if comb2nd_indicies_nonbasic[dm1]
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]
+  end
+  comb1st_indicies_firstoffsets[index] = first
+  comb1st_indicies_lastoffsets[index] = last
+  raise "double index" if comb_indicies[dm0]
+  comb_indicies[dm0] = cumoffset
+  cumoffset += last - first + 1 + 2
+end
+
+offset = 0
+comb2nd_indicies_sorted_keys.each do |dm1|
+  raise "double index" if comb_indicies[dm1]
+  comb_indicies[dm1] = 0x8000 | (comb2nd_indicies[dm1] + offset)
+  raise "too large comb index" if  comb2nd_indicies[dm1] + offset > 0x4000
+  if comb2nd_indicies_nonbasic[dm1]
+    comb_indicies[dm1] = comb_indicies[dm1] | 0x4000
+    offset += 1
+  end
+end
+
+properties_indicies = {}
+properties = []
+chars.each do |char|
+  c_entry = char.c_entry(comb_indicies)
+  char.c_entry_index = properties_indicies[c_entry]
+  unless char.c_entry_index
+    properties_indicies[c_entry] = properties.length
+    char.c_entry_index = properties.length
+    properties << c_entry
+  end
+end
+
+stage1 = []
+stage2 = []
+for code in 0...0x110000
+  next unless code % 0x100 == 0
+  stage2_entry = []
+  for code2 in code...(code+0x100)
+    if char_hash[code2]
+      stage2_entry << (char_hash[code2].c_entry_index + 1)
+    else
+      stage2_entry << 0
+    end
+  end
+  old_index = stage2.index(stage2_entry)
+  if old_index
+    stage1 << (old_index * 0x100)
+  else
+    stage1 << (stage2.length * 0x100)
+    stage2 << stage2_entry
+  end
+end
+
+$stdout << "const utf8proc_uint16_t utf8proc_sequences[] = {\n  "
+i = 0
+$int_array.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_uint16_t utf8proc_stage1table[] = {\n  "
+i = 0
+stage1.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_uint16_t utf8proc_stage2table[] = {\n  "
+i = 0
+stage2.flatten.each do |entry|
+  i += 1
+  if i == 8
+    i = 0
+    $stdout << "\n  "
+  end
+  $stdout << entry << ", "
+end
+$stdout << "};\n\n"
+
+$stdout << "const utf8proc_property_t utf8proc_properties[] = {\n"
+$stdout << "  {0, 0, 0, 0, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX,  false,false,false,false, 0, 0, UTF8PROC_BOUNDCLASS_OTHER},\n"
+properties.each { |line|
+  $stdout << line
+}
+$stdout << "};\n\n"
+
+
+
+$stdout << "const utf8proc_uint16_t utf8proc_combinations[] = {\n  "
+i = 0
+comb1st_indicies.keys.each_index do |a|
+  offset = 0
+  $stdout << comb1st_indicies_firstoffsets[a] << ", " << comb1st_indicies_lastoffsets[a] << ", "
+  comb2nd_indicies_sorted_keys.each_with_index do |dm1, b|
+    break if offset > comb1st_indicies_lastoffsets[a] 
+    if offset >= comb1st_indicies_firstoffsets[a]
+      i += 1
+      if i == 8
+        i = 0
+        $stdout << "\n  "
+      end
+      v = comb_array[a][b] ? comb_array[a][b] : 0
+      $stdout << (( v & 0xFFFF0000 ) >> 16) << ", " if comb2nd_indicies_nonbasic[dm1]
+      $stdout << (v & 0xFFFF) << ", "
+    end
+    offset += 1
+    offset += 1 if comb2nd_indicies_nonbasic[dm1]    
+  end
+  $stdout  << "\n"
+end
+$stdout << "};\n\n"
+
--- a/3rdparty/utf8proc/data_generator.rb
+++ b/3rdparty/utf8proc/data_generator.rb
--- a/3rdparty/utf8proc/lump.txt
+++ b/3rdparty/utf8proc/lump.txt
@ -1,3 +1,4 @@
+```
 U+0020      <-- all space characters (general category Zs)
 U+0027  '   <-- left/right single quotation mark U+2018..2019,
                modifier letter apostrophe U+02BC,
@ -23,4 +24,4 @@ U+005F  _   <-- all connector characters (general category Pc),
 U+0060  `   <-- modifier letter grave accent U+02CB
 U+007C  |   <-- divides U+2223
 U+007E  ~   <-- tilde operator U+223C
-
+```
--- a/3rdparty/utf8proc/pgsql/Makefile
+++ b/3rdparty/utf8proc/pgsql/Makefile
@ -1,10 +0,0 @@
-utf8proc_pgsql.so: utf8proc_pgsql.o
-	ld -shared -o utf8proc_pgsql.so utf8proc_pgsql.o
-
-utf8proc_pgsql.o: utf8proc_pgsql.c
-	gcc -Wall -fpic -c -I`pg_config --includedir-server` \
-		-o utf8proc_pgsql.o utf8proc_pgsql.c
-
-clean:
-	rm -f *.o *.so
-
--- a/3rdparty/utf8proc/pgsql/utf8proc.sql
+++ b/3rdparty/utf8proc/pgsql/utf8proc.sql
@ -1,6 +0,0 @@
-CREATE OR REPLACE FUNCTION unifold (text) RETURNS text
-  LANGUAGE 'C' IMMUTABLE STRICT AS '$libdir/utf8proc_pgsql.so',
-  'utf8proc_pgsql_unifold';
-CREATE OR REPLACE FUNCTION unistrip (text) RETURNS text
-  LANGUAGE 'C' IMMUTABLE STRICT AS '$libdir/utf8proc_pgsql.so',
-  'utf8proc_pgsql_unistrip';
--- a/3rdparty/utf8proc/pgsql/utf8proc_pgsql.c
+++ b/3rdparty/utf8proc/pgsql/utf8proc_pgsql.c
@ -1,139 +0,0 @@
-/*
- *  Copyright (c) Public Software Group e. V., Berlin, Germany
- *
- *  Permission is hereby granted, free of charge, to any person obtaining a
- *  copy of this software and associated documentation files (the "Software"),
- *  to deal in the Software without restriction, including without limitation
- *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
- *  and/or sell copies of the Software, and to permit persons to whom the
- *  Software is furnished to do so, subject to the following conditions:
- *
- *  The above copyright notice and this permission notice shall be included in
- *  all copies or substantial portions of the Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- *  DEALINGS IN THE SOFTWARE.
- */
-
-
-/*
- *  File name:    pgsql/utf8proc_pgsql.c
- *
- *  Description:
- *  PostgreSQL extension to provide two functions 'unifold' and 'unistrip',
- *  which can be used to case-fold and normalize index fields and
- *  optionally strip marks (e.g. accents) from strings.
- */
-
-
-#include "../utf8proc.c"
-
-#include <postgres.h>
-#include <utils/elog.h>
-#include <fmgr.h>
-#include <string.h>
-#include <unistd.h>
-#include <utils/builtins.h>
-
-#ifdef PG_MODULE_MAGIC
-PG_MODULE_MAGIC;
-#endif
-
-#define UTF8PROC_PGSQL_FOLD_OPTS ( UTF8PROC_REJECTNA | UTF8PROC_COMPAT | \
-  UTF8PROC_COMPOSE | UTF8PROC_STABLE | UTF8PROC_IGNORE | UTF8PROC_STRIPCC | \
-  UTF8PROC_NLF2LF | UTF8PROC_CASEFOLD | UTF8PROC_LUMP )
-#define UTF8PROC_PGSQL_STRIP_OPTS ( UTF8PROC_REJECTNA | UTF8PROC_COMPAT | \
-  UTF8PROC_COMPOSE | UTF8PROC_STABLE | UTF8PROC_IGNORE | UTF8PROC_STRIPCC | \
-  UTF8PROC_NLF2LF | UTF8PROC_CASEFOLD | UTF8PROC_LUMP | UTF8PROC_STRIPMARK )
-
-ssize_t utf8proc_pgsql_utf8map(
-  text *input_string, text **output_string_ptr, int options
-) {
-  ssize_t result;
-  text *output_string;
-  result = utf8proc_decompose(
-    VARDATA(input_string), VARSIZE(input_string) - VARHDRSZ,
-    NULL, 0, options
-  );
-  if (result < 0) return result;
-  if (result > (SIZE_MAX-1-VARHDRSZ)/sizeof(int32_t))
-    return UTF8PROC_ERROR_OVERFLOW;
-  /* reserve one extra byte for termination */
-  *output_string_ptr = palloc(result * sizeof(int32_t) + 1 + VARHDRSZ);
-  output_string = *output_string_ptr;
-  if (!output_string) return UTF8PROC_ERROR_NOMEM;
-  result = utf8proc_decompose(
-    VARDATA(input_string), VARSIZE(input_string) - VARHDRSZ,
-    (int32_t *)VARDATA(output_string), result, options
-  );
-  if (result < 0) return result;
-  result = utf8proc_reencode(
-    (int32_t *)VARDATA(output_string), result, options
-  );
-  if (result >= 0) SET_VARSIZE(output_string, result + VARHDRSZ);
-  return result;
-}
-
-void utf8proc_pgsql_utf8map_errchk(ssize_t result, text *output_string) {
-  if (result < 0) {
-    int sqlerrcode;
-    if (output_string) pfree(output_string);
-    switch(result) {
-      case UTF8PROC_ERROR_NOMEM:
-      sqlerrcode = ERRCODE_OUT_OF_MEMORY; break;
-      case UTF8PROC_ERROR_OVERFLOW:
-      sqlerrcode = ERRCODE_PROGRAM_LIMIT_EXCEEDED; break;
-      case UTF8PROC_ERROR_INVALIDUTF8:
-      case UTF8PROC_ERROR_NOTASSIGNED:
-      return;
-      default:
-      sqlerrcode = ERRCODE_INTERNAL_ERROR;
-    }
-    ereport(ERROR, (
-      errcode(sqlerrcode),
-      errmsg("%s", utf8proc_errmsg(result))
-    ));
-  }
-}
-
-PG_FUNCTION_INFO_V1(utf8proc_pgsql_unifold);
-Datum utf8proc_pgsql_unifold(PG_FUNCTION_ARGS) {
-  text *input_string;
-  text *output_string = NULL;
-  ssize_t result;
-  input_string = PG_GETARG_TEXT_P(0);
-  result = utf8proc_pgsql_utf8map(
-    input_string, &output_string, UTF8PROC_PGSQL_FOLD_OPTS
-  );
-  PG_FREE_IF_COPY(input_string, 0);
-  utf8proc_pgsql_utf8map_errchk(result, output_string);
-  if (result >= 0) {
-    PG_RETURN_TEXT_P(output_string);
-  } else {
-    PG_RETURN_NULL();
-  }
-}
-
-PG_FUNCTION_INFO_V1(utf8proc_pgsql_unistrip);
-Datum utf8proc_pgsql_unistrip(PG_FUNCTION_ARGS) {
-  text *input_string;
-  text *output_string = NULL;
-  ssize_t result;
-  input_string = PG_GETARG_TEXT_P(0);
-  result = utf8proc_pgsql_utf8map(
-    input_string, &output_string, UTF8PROC_PGSQL_STRIP_OPTS
-  );
-  PG_FREE_IF_COPY(input_string, 0);
-  utf8proc_pgsql_utf8map_errchk(result, output_string);
-  if (result >= 0) {
-    PG_RETURN_TEXT_P(output_string);
-  } else {
-    PG_RETURN_NULL();
-  }
-}
-
--- a/3rdparty/utf8proc/ruby/extconf.rb
+++ b/3rdparty/utf8proc/ruby/extconf.rb
@ -1,2 +0,0 @@
-require 'mkmf'
-create_makefile("utf8proc_native")
--- a/3rdparty/utf8proc/ruby/gem/LICENSE
+++ b/3rdparty/utf8proc/ruby/gem/LICENSE
@ -1,64 +0,0 @@
-
-Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-
-This software distribution contains derived data from a modified version of
-the Unicode data files. The following license applies to that data:
-
-COPYRIGHT AND PERMISSION NOTICE
-
-Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed
-under the Terms of Use in http://www.unicode.org/copyright.html.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of the Unicode data files and any associated documentation (the "Data
-Files") or Unicode software and any associated documentation (the
-"Software") to deal in the Data Files or Software without restriction,
-including without limitation the rights to use, copy, modify, merge,
-publish, distribute, and/or sell copies of the Data Files or Software, and
-to permit persons to whom the Data Files or Software are furnished to do
-so, provided that (a) the above copyright notice(s) and this permission
-notice appear with all copies of the Data Files or Software, (b) both the
-above copyright notice(s) and this permission notice appear in associated
-documentation, and (c) there is clear notice in each modified Data File or
-in the Software as well as in the documentation associated with the Data
-File(s) or Software that the data or software has been modified.
-
-THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
-THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
-INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR
-CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
-USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-PERFORMANCE OF THE DATA FILES OR SOFTWARE.
-
-Except as contained in this notice, the name of a copyright holder shall
-not be used in advertising or otherwise to promote the sale, use or other
-dealings in these Data Files or Software without prior written
-authorization of the copyright holder.
-
-
-Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
-registered in some jurisdictions. All other trademarks and registered
-trademarks mentioned herein are the property of their respective owners.
-
--- a/3rdparty/utf8proc/ruby/gem/utf8proc.gemspec
+++ b/3rdparty/utf8proc/ruby/gem/utf8proc.gemspec
@ -1,12 +0,0 @@
-require 'rubygems'
-SPEC = Gem::Specification.new do |s|
-  s.name = 'utf8proc'
-  s.version = '1.1.5'
-  s.author = 'Public Software Group e. V., Berlin, Germany'
-  s.homepage = 'http://www.public-software-group.org/utf8proc'
-  s.summary = 'UTF-8 Unicode string processing'
-  s.files = ['LICENSE', 'lib/utf8proc.rb', 'ext/utf8proc_native.c']
-  s.require_path = 'lib/'
-  s.extensions = ['ext/extconf.rb']
-  s.has_rdoc = false
-end
--- a/3rdparty/utf8proc/ruby/utf8proc.rb
+++ b/3rdparty/utf8proc/ruby/utf8proc.rb
@ -1,98 +0,0 @@
-#  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
-#
-#  Permission is hereby granted, free of charge, to any person obtaining a
-#  copy of this software and associated documentation files (the "Software"),
-#  to deal in the Software without restriction, including without limitation
-#  the rights to use, copy, modify, merge, publish, distribute, sublicense,
-#  and/or sell copies of the Software, and to permit persons to whom the
-#  Software is furnished to do so, subject to the following conditions:
-#
-#  The above copyright notice and this permission notice shall be included in
-#  all copies or substantial portions of the Software.
-#
-#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-#  DEALINGS IN THE SOFTWARE.
-
-
-#
-#  File name:    ruby/utf8proc.rb
-#
-#  Description:
-#  Part of the ruby wrapper for libutf8proc, which is written in ruby.
-#
-
-
-require 'utf8proc_native'
-
-
-module Utf8Proc
-
-  SpecialChars = {
-    :HT => "\x09",
-    :LF => "\x0A",
-    :VT => "\x0B",
-    :FF => "\x0C",
-    :CR => "\x0D",
-    :FS => "\x1C",
-    :GS => "\x1D",
-    :RS => "\x1E",
-    :US => "\x1F",
-    :LS => "\xE2\x80\xA8",
-    :PS => "\xE2\x80\xA9",
-  }
-
-  module StringExtensions
-    def utf8map(*option_array)
-      options = 0
-      option_array.each do |option|
-        flag = Utf8Proc::Options[option]
-        raise ArgumentError, "Unknown argument given to String#utf8map." unless
-          flag
-        options |= flag
-      end
-      return Utf8Proc::utf8map(self, options)
-    end
-    def utf8map!(*option_array)
-      self.replace(self.utf8map(*option_array))
-    end
-    def utf8nfd;   utf8map( :stable, :decompose); end
-    def utf8nfd!;  utf8map!(:stable, :decompose); end
-    def utf8nfc;   utf8map( :stable, :compose); end
-    def utf8nfc!;  utf8map!(:stable, :compose); end
-    def utf8nfkd;  utf8map( :stable, :decompose, :compat); end
-    def utf8nfkd!; utf8map!(:stable, :decompose, :compat); end
-    def utf8nfkc;  utf8map( :stable, :compose, :compat); end
-    def utf8nfkc!; utf8map!(:stable, :compose, :compat); end
-    def utf8chars
-      result = self.utf8map(:charbound).split("\377")
-      result.shift if result.first == ""
-      result
-    end
-    def char_ary
-      # depecated, use String#utf8chars instead
-      utf8chars
-    end
-  end
-
-  module IntegerExtensions
-    def utf8
-      return Utf8Proc::utf8char(self)
-    end
-  end
-
-end
-
-
-class String
-  include(Utf8Proc::StringExtensions)
-end
-
-class Integer
-  include(Utf8Proc::IntegerExtensions)
-end
-
--- a/3rdparty/utf8proc/ruby/utf8proc_native.c
+++ b/3rdparty/utf8proc/ruby/utf8proc_native.c
@ -1,160 +0,0 @@
-/*
- *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
- *
- *  Permission is hereby granted, free of charge, to any person obtaining a
- *  copy of this software and associated documentation files (the "Software"),
- *  to deal in the Software without restriction, including without limitation
- *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
- *  and/or sell copies of the Software, and to permit persons to whom the
- *  Software is furnished to do so, subject to the following conditions:
- *
- *  The above copyright notice and this permission notice shall be included in
- *  all copies or substantial portions of the Software.
- *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- *  DEALINGS IN THE SOFTWARE.
- */
-
-
-/*
- *  File name:    ruby/utf8proc_native.c
- *
- *  Description:
- *  Native part of the ruby wrapper for libutf8proc.
- */
-
-
-#include "../utf8proc.c"
-#include "ruby.h"
-
-#ifndef RSTRING_PTR
-#define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif
-#ifndef RSTRING_LEN
-#define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif
-
-typedef struct utf8proc_ruby_mapenv_struct {
-  int32_t *buffer;
-} utf8proc_ruby_mapenv_t;
-
-void utf8proc_ruby_mapenv_free(utf8proc_ruby_mapenv_t *env) {
-  free(env->buffer);
-  free(env);
-}
-
-VALUE utf8proc_ruby_module;
-VALUE utf8proc_ruby_options;
-VALUE utf8proc_ruby_eUnicodeError;
-VALUE utf8proc_ruby_eInvalidUtf8Error;
-VALUE utf8proc_ruby_eCodeNotAssignedError;
-
-VALUE utf8proc_ruby_map_error(ssize_t result) {
-  VALUE excpt_class;
-  switch (result) {
-    case UTF8PROC_ERROR_NOMEM:
-    excpt_class = rb_eNoMemError; break;
-    case UTF8PROC_ERROR_OVERFLOW:
-    case UTF8PROC_ERROR_INVALIDOPTS:
-    excpt_class = rb_eArgError; break;
-    case UTF8PROC_ERROR_INVALIDUTF8:
-    excpt_class = utf8proc_ruby_eInvalidUtf8Error; break;
-    case UTF8PROC_ERROR_NOTASSIGNED:
-    excpt_class = utf8proc_ruby_eCodeNotAssignedError; break;
-    default:
-    excpt_class = rb_eRuntimeError;
-  }
-  rb_raise(excpt_class, "%s", utf8proc_errmsg(result));
-  return Qnil;
-}
-
-VALUE utf8proc_ruby_map(VALUE self, VALUE str_param, VALUE options_param) {
-  VALUE str;
-  int options;
-  VALUE env_obj;
-  utf8proc_ruby_mapenv_t *env;
-  ssize_t result;
-  VALUE retval;
-  str = StringValue(str_param);
-  options = NUM2INT(options_param) & ~UTF8PROC_NULLTERM;
-  env_obj = Data_Make_Struct(rb_cObject, utf8proc_ruby_mapenv_t, NULL,
-    utf8proc_ruby_mapenv_free, env);
-  result = utf8proc_decompose(RSTRING_PTR(str), RSTRING_LEN(str),
-    NULL, 0, options);
-  if (result < 0) {
-    utf8proc_ruby_map_error(result);
-    return Qnil;  /* needed to prevent problems with optimization */
-  }
-  env->buffer = ALLOC_N(int32_t, result+1);
-  result = utf8proc_decompose(RSTRING_PTR(str), RSTRING_LEN(str),
-    env->buffer, result, options);
-  if (result < 0) {
-    free(env->buffer);
-    env->buffer = 0;
-    utf8proc_ruby_map_error(result);
-    return Qnil;  /* needed to prevent problems with optimization */
-  }
-  result = utf8proc_reencode(env->buffer, result, options);
-  if (result < 0) {
-    free(env->buffer);
-    env->buffer = 0;
-    utf8proc_ruby_map_error(result);
-    return Qnil;  /* needed to prevent problems with optimization */
-  }
-  retval = rb_str_new((char *)env->buffer, result);
-  free(env->buffer);
-  env->buffer = 0;
-  return retval;
-}
-
-static VALUE utf8proc_ruby_char(VALUE self, VALUE code_param) {
-  char buffer[4];
-  ssize_t result;
-  int uc;
-  uc = NUM2INT(code_param);
-  if (!utf8proc_codepoint_valid(uc))
-    rb_raise(rb_eArgError, "Invalid Unicode code point");
-  result = utf8proc_encode_char(uc, buffer);
-  return rb_str_new(buffer, result);
-}
-
-#define register_utf8proc_option(sym, field) \
-  rb_hash_aset(utf8proc_ruby_options, ID2SYM(rb_intern(sym)), INT2FIX(field))
-
-void Init_utf8proc_native() {
-  utf8proc_ruby_module = rb_define_module("Utf8Proc");
-  rb_define_module_function(utf8proc_ruby_module, "utf8map",
-    utf8proc_ruby_map, 2);
-  rb_define_module_function(utf8proc_ruby_module, "utf8char",
-    utf8proc_ruby_char, 1);
-  utf8proc_ruby_eUnicodeError = rb_define_class_under(utf8proc_ruby_module,
-    "UnicodeError", rb_eStandardError);
-  utf8proc_ruby_eInvalidUtf8Error = rb_define_class_under(
-    utf8proc_ruby_module, "InvalidUtf8Error", utf8proc_ruby_eUnicodeError);
-  utf8proc_ruby_eCodeNotAssignedError = rb_define_class_under(
-    utf8proc_ruby_module, "CodeNotAssignedError",
-    utf8proc_ruby_eUnicodeError);
-  utf8proc_ruby_options = rb_hash_new();
-  register_utf8proc_option("stable",    UTF8PROC_STABLE);
-  register_utf8proc_option("compat",    UTF8PROC_COMPAT);
-  register_utf8proc_option("compose",   UTF8PROC_COMPOSE);
-  register_utf8proc_option("decompose", UTF8PROC_DECOMPOSE);
-  register_utf8proc_option("ignore",    UTF8PROC_IGNORE);
-  register_utf8proc_option("rejectna",  UTF8PROC_REJECTNA);
-  register_utf8proc_option("nlf2ls",    UTF8PROC_NLF2LS);
-  register_utf8proc_option("nlf2ps",    UTF8PROC_NLF2PS);
-  register_utf8proc_option("nlf2lf",    UTF8PROC_NLF2LF);
-  register_utf8proc_option("stripcc",   UTF8PROC_STRIPCC);
-  register_utf8proc_option("casefold",  UTF8PROC_CASEFOLD);
-  register_utf8proc_option("charbound", UTF8PROC_CHARBOUND);
-  register_utf8proc_option("lump",      UTF8PROC_LUMP);
-  register_utf8proc_option("stripmark", UTF8PROC_STRIPMARK);
-  OBJ_FREEZE(utf8proc_ruby_options);
-  rb_define_const(utf8proc_ruby_module, "Options", utf8proc_ruby_options);
-}
-
--- a/3rdparty/utf8proc/test/case.c
+++ b/3rdparty/utf8proc/test/case.c
@ -0,0 +1,50 @@
+#include "tests.h"
+#include <wctype.h>
+
+int main(int argc, char **argv)
+{
+     int error = 0, better = 0;
+     utf8proc_int32_t c;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          utf8proc_int32_t l = utf8proc_tolower(c);
+          utf8proc_int32_t u = utf8proc_toupper(c);
+
+          check(l == c || utf8proc_codepoint_valid(l), "invalid tolower");
+          check(u == c || utf8proc_codepoint_valid(u), "invalid toupper");
+
+          if (sizeof(wint_t) > 2 || c < (1<<16)) {
+               wint_t l0 = towlower(c), u0 = towupper(c);
+               
+               /* OS unicode tables may be out of date.  But if they
+                  do have a lower/uppercase mapping, hopefully it
+                  is correct? */
+               if (l0 != c && l0 != l) {
+                    fprintf(stderr, "MISMATCH %x != towlower(%x) == %x\n",
+                            l, c, l0);
+                    ++error;
+               }
+               else if (l0 != l) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towlower(%x) == %x\n", l, c, l0); */
+               }
+               if (u0 != c && u0 != u) {
+                    fprintf(stderr, "MISMATCH %x != towupper(%x) == %x\n",
+                            u, c, u0);
+                    ++error;
+               }
+               else if (u0 != u) { /* often true for out-of-date OS unicode */
+                    ++better;
+                    /* printf("%x != towupper(%x) == %x\n", u, c, u0); */
+               }
+          }
+     }
+     check(!error, "utf8proc case conversion FAILED %d tests.", error);
+     printf("More up-to-date than OS unicode tables for %d tests.\n", better);
+     printf("utf8proc case conversion tests SUCCEEDED.\n");
+     return 0;
+}
--- a/3rdparty/utf8proc/test/charwidth.c
+++ b/3rdparty/utf8proc/test/charwidth.c
@ -0,0 +1,71 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static int my_isprint(int c) {
+     int cat = utf8proc_get_property(c)->category;
+     return (UTF8PROC_CATEGORY_LU <= cat && cat <= UTF8PROC_CATEGORY_ZS) ||
+          (c == 0x0601 || c == 0x0602 || c == 0x0603 || c == 0x06dd);
+}
+
+int main(int argc, char **argv)
+{
+     int c, error = 0, updates = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of the character widths */
+     for (c = 0; c <= 0x110000; ++c) {
+          int cat = utf8proc_get_property(c)->category;
+          int w = utf8proc_charwidth(c);
+          if ((cat == UTF8PROC_CATEGORY_MN || cat == UTF8PROC_CATEGORY_ME) &&
+              w > 0) {
+               fprintf(stderr, "nonzero width %d for combining char %x\n", w, c);
+               error = 1;
+          }
+          if (w == 0 &&
+			  ((cat >= UTF8PROC_CATEGORY_LU && cat <= UTF8PROC_CATEGORY_LO) ||
+			   (cat >= UTF8PROC_CATEGORY_ND && cat <= UTF8PROC_CATEGORY_SC) ||
+			   (cat >= UTF8PROC_CATEGORY_SO && cat <= UTF8PROC_CATEGORY_ZS))) {
+               fprintf(stderr, "zero width for symbol-like char %x\n", c);
+               error = 1;
+          }
+          if (c <= 127 && ((!isprint(c) && w > 0) ||
+                           (isprint(c) && wcwidth(c) != w))) {
+               fprintf(stderr, "wcwidth %d mismatch %d for %s ASCII %x\n",
+                       wcwidth(c), w, 
+                       isprint(c) ? "printable" : "non-printable", c);
+               error = 1;
+          }
+          if (!my_isprint(c) && w > 0) {
+               fprintf(stderr, "non-printing %x had width %d\n", c, w);
+               error = 1;
+          }
+     }
+     check(!error, "utf8proc_charwidth FAILED tests.");
+
+     /* print some other information by compariing with system wcwidth */
+     printf("Mismatches with system wcwidth (not necessarily errors):\n");
+     for (c = 0; c <= 0x110000; ++c) {
+          int w = utf8proc_charwidth(c);
+          int wc = wcwidth(c);
+          if (sizeof(wchar_t) == 2 && c >= (1<<16)) continue;
+          /* lots of these errors for out-of-date system unicode tables */
+          if (wc == -1 && my_isprint(c) && w > 0) {
+			   updates += 1;
+#if 0
+               printf("  wcwidth(%x) = -1 for printable char\n", c);
+#endif
+		  }
+          if (wc == -1 && !my_isprint(c) && w > 0)
+               printf("  wcwidth(%x) = -1 for non-printable width-%d char\n", c, w);
+          if (wc >= 0 && wc != w)
+               printf("  wcwidth(%x) = %d != charwidth %d\n", c, wc, w);
+     }
+	 printf("   ... (positive widths for %d chars unknown to wcwidth) ...\n",
+			updates);
+     printf("Character-width tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/3rdparty/utf8proc/test/graphemetest.c
+++ b/3rdparty/utf8proc/test/graphemetest.c
@ -0,0 +1,74 @@
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+    char *buf = NULL;
+    size_t bufsize = 0;
+    FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+    utf8proc_uint8_t src[1024];
+    int len;
+    
+    check(f != NULL, "error opening GraphemeBreakTest.txt");
+    while (getline(&buf, &bufsize, f) > 0) {
+        size_t bi = 0, si = 0;
+        lineno += 1;
+        
+        if (lineno % 100 == 0)
+            printf("checking line %zd...\n", lineno);
+        
+        if (buf[0] == '#') continue;
+        
+        while (buf[bi]) {
+            bi = skipspaces(buf, bi);
+            if (buf[bi] == '/') { /* grapheme break */
+                src[si++] = '/';
+                bi++;
+            }
+            else if (buf[bi] == '+') { /* no break */
+                bi++;
+            }
+            else if (buf[bi] == '#') { /* start of comments */
+                break;
+            }
+	    else { /* hex-encoded codepoint */
+                len = encode((char*) (src + si), buf + bi) - 1;
+                while (src[si]) ++si; /* advance to NUL termination */
+                bi += len;
+            }
+        }
+        if (si && src[si-1] == '/')
+            --si; /* no break after final grapheme */
+        src[si] = 0; /* NUL-terminate */
+        
+        if (si) {
+            utf8proc_uint8_t utf8[1024]; /* copy src without 0xff grapheme separators */
+            size_t i = 0, j = 0;
+            utf8proc_ssize_t glen;
+            utf8proc_uint8_t *g; /* utf8proc_map grapheme results */
+            while (i < si) {
+                if (src[i] != '/')
+                    utf8[j++] = src[i++];
+                else
+                    i++;
+            }
+            glen = utf8proc_map(utf8, j, &g, UTF8PROC_CHARBOUND);
+            if (glen == UTF8PROC_ERROR_INVALIDUTF8) {
+                 /* the test file contains surrogate codepoints, which are only for UTF-16 */
+                 printf("line %zd: ignoring invalid UTF-8 codepoints\n", lineno);
+            }
+            else {
+                 check(glen >= 0, "utf8proc_map error = %s",
+                       utf8proc_errmsg(glen));
+                 for (i = 0; i <= glen; ++i)
+                      if (g[i] == 0xff)
+                          g[i] = '/'; /* easier-to-read output (/ is not in test strings) */
+                 check(!strcmp((char*)g, (char*)src),
+                       "grapheme mismatch: \"%s\" instead of \"%s\"", (char*)g, (char*)src);
+            }
+            free(g);
+        }
+    }
+    fclose(f);
+    printf("Passed tests after %zd lines!\n", lineno);
+    return 0;
+}
--- a/3rdparty/utf8proc/test/iterate.c
+++ b/3rdparty/utf8proc/test/iterate.c
@ -0,0 +1,166 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+static  int     tests;
+static  int     error;
+
+#define CHECKVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,len,__LINE__)
+#define CHECKINVALID(pos, val, len) buf[pos] = val; testbytes(buf,len,UTF8PROC_ERROR_INVALIDUTF8,__LINE__)
+
+static void testbytes(unsigned char *buf, int len, utf8proc_ssize_t retval, int line)
+{
+    utf8proc_int32_t out[16];
+    utf8proc_ssize_t ret;
+
+    /* Make a copy to ensure that memory is left uninitialized after "len"
+     * bytes. This way, Valgrind can detect overreads.
+     */
+    unsigned char tmp[16];
+    memcpy(tmp, buf, len);
+
+    tests++;
+    if ((ret = utf8proc_iterate(tmp, len, out)) != retval) {
+        fprintf(stderr, "Failed (%d):", line);
+        for (int i = 0; i < len ; i++) {
+            fprintf(stderr, " 0x%02x", tmp[i]);
+        }
+        fprintf(stderr, " -> %zd\n", ret);
+        error++;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    uint32_t byt;
+    unsigned char buf[16];
+
+    tests = error = 0;
+
+    // Check valid sequences that were considered valid erroneously before
+    buf[0] = 0xef;
+    buf[1] = 0xb7;
+    for (byt = 0x90; byt < 0xa0; byt++) {
+        CHECKVALID(2, byt, 3);
+    }
+    // Check 0xfffe and 0xffff
+    buf[1] = 0xbf;
+    CHECKVALID(2, 0xbe, 3);
+    CHECKVALID(2, 0xbf, 3);
+    // Check 0x??fffe & 0x??ffff
+    for (byt = 0x1fffe; byt < 0x110000; byt += 0x10000) {
+        buf[0] = 0xf0 | (byt >> 18);
+        buf[1] = 0x80 | ((byt >> 12) & 0x3f);
+        CHECKVALID(3, 0xbe, 4);
+        CHECKVALID(3, 0xbf, 4);
+    }
+    
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Continuation byte not after lead
+    for (byt = 0x80; byt < 0xc0; byt++) {
+        CHECKINVALID(0, byt, 1);
+    }
+
+    // Test lead bytes
+    for (byt = 0xc0; byt <= 0xff; byt++) {
+	// Single lead byte at end of string
+        CHECKINVALID(0, byt, 1);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(1, 65, 2);
+	// Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(1, 0xc0, 2);
+    }
+
+    // Test overlong 2-byte
+    buf[0] = 0xc0;
+    for (byt = 0x81; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+    buf[0] = 0xc1;
+    for (byt = 0x80; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 2);
+    }
+
+    // Test overlong 3-byte
+    buf[0] = 0xe0;
+    buf[2] = 0x80;
+    for (byt = 0x80; byt <= 0x9f; byt++) {
+        CHECKINVALID(1, byt, 3);
+    }
+
+    // Test overlong 4-byte
+    buf[0] = 0xf0;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x80; byt <= 0x8f; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+
+    // Test 4-byte > 0x10ffff
+    buf[0] = 0xf4;
+    buf[2] = 0x80;
+    buf[3] = 0x80;
+    for (byt = 0x90; byt <= 0xbf; byt++) {
+        CHECKINVALID(1, byt, 4);
+    }
+    buf[1] = 0x80;
+    for (byt = 0xf5; byt <= 0xf7; byt++) {
+        CHECKINVALID(0, byt, 4);
+    }
+
+    // Test 5-byte
+    buf[4] = 0x80;
+    for (byt = 0xf8; byt <= 0xfb; byt++) {
+        CHECKINVALID(0, byt, 5);
+    }
+
+    // Test 6-byte
+    buf[5] = 0x80;
+    for (byt = 0xfc; byt <= 0xfd; byt++) {
+        CHECKINVALID(0, byt, 6);
+    }
+
+    // Test 7-byte
+    buf[6] = 0x80;
+    CHECKINVALID(0, 0xfe, 7);
+
+    // Three and above byte sequences
+    for (byt = 0xe0; byt < 0xf0; byt++) {
+        // Lead followed by only 1 continuation byte
+        CHECKINVALID(0, byt, 2);
+        // Lead ended by non-continuation character < 0x80
+        CHECKINVALID(2, 65, 3);
+        // Lead ended by non-continuation character > 0xbf
+        CHECKINVALID(2, 0xc0, 3);
+    }
+
+    // 3-byte encoded surrogate character(s)
+    buf[0] = 0xed; buf[2] = 0x80;
+    // Single surrogate
+    CHECKINVALID(1, 0xa0, 3);
+    // Trailing surrogate first
+    CHECKINVALID(1, 0xb0, 3);
+
+    // Four byte sequences
+    buf[1] = 0x80;
+    for (byt = 0xf0; byt < 0xf5; byt++) {
+        // Lead followed by only 1 continuation bytes
+        CHECKINVALID(0, byt, 2);
+        // Lead followed by only 2 continuation bytes
+        CHECKINVALID(0, byt, 3);
+        // Lead followed by non-continuation character < 0x80
+        CHECKINVALID(3, 65, 4);
+        // Lead followed by non-continuation character > 0xbf
+        CHECKINVALID(3, 0xc0, 4);
+
+    }
+
+     check(!error, "utf8proc_iterate FAILED %d tests out of %d", error, tests);
+     printf("utf8proc_iterate tests SUCCEEDED, (%d) tests passed.\n", tests);
+
+     return 0;
+}
--- a/3rdparty/utf8proc/test/normtest.c
+++ b/3rdparty/utf8proc/test/normtest.c
@ -0,0 +1,64 @@
+#include "tests.h"
+
+#define CHECK_NORM(NRM, norm, src) {                                 \
+    char *src_norm = (char*) utf8proc_ ## NRM((utf8proc_uint8_t*) src);      \
+    check(!strcmp(norm, src_norm),                                  \
+          "normalization failed for %s -> %s", src, norm);          \
+    free(src_norm);                                                 \
+}
+
+int main(int argc, char **argv)
+{
+     char *buf = NULL;
+     size_t bufsize = 0;
+     FILE *f = argc > 1 ? fopen(argv[1], "r") : NULL;
+     char source[1024], NFC[1024], NFD[1024], NFKC[1024], NFKD[1024];
+
+     check(f != NULL, "error opening NormalizationTest.txt");
+     while (getline(&buf, &bufsize, f) > 0) {
+          size_t offset;
+          lineno += 1;
+
+          if (buf[0] == '@') {
+               printf("line %zd: %s", lineno, buf + 1);
+               continue;
+          }
+          else if (lineno % 1000 == 0)
+               printf("checking line %zd...\n", lineno);
+
+          if (buf[0] == '#') continue;
+
+          offset = encode(source, buf);
+          offset += encode(NFC, buf + offset);
+          offset += encode(NFD, buf + offset);
+          offset += encode(NFKC, buf + offset);
+          offset += encode(NFKD, buf + offset);
+
+          CHECK_NORM(NFC, NFC, source);
+          CHECK_NORM(NFC, NFC, NFC);
+          CHECK_NORM(NFC, NFC, NFD);
+          CHECK_NORM(NFC, NFKC, NFKC);
+          CHECK_NORM(NFC, NFKC, NFKD);
+
+          CHECK_NORM(NFD, NFD, source);
+          CHECK_NORM(NFD, NFD, NFC);
+          CHECK_NORM(NFD, NFD, NFD);
+          CHECK_NORM(NFD, NFKD, NFKC);
+          CHECK_NORM(NFD, NFKD, NFKD);
+
+          CHECK_NORM(NFKC, NFKC, source);
+          CHECK_NORM(NFKC, NFKC, NFC);
+          CHECK_NORM(NFKC, NFKC, NFD);
+          CHECK_NORM(NFKC, NFKC, NFKC);
+          CHECK_NORM(NFKC, NFKC, NFKD);
+
+          CHECK_NORM(NFKD, NFKD, source);
+          CHECK_NORM(NFKD, NFKD, NFC);
+          CHECK_NORM(NFKD, NFKD, NFD);
+          CHECK_NORM(NFKD, NFKD, NFKC);
+          CHECK_NORM(NFKD, NFKD, NFKD);
+     }
+     fclose(f);
+     printf("Passed tests after %zd lines!\n", lineno);
+     return 0;
+}
--- a/3rdparty/utf8proc/test/printproperty.c
+++ b/3rdparty/utf8proc/test/printproperty.c
@ -0,0 +1,49 @@
+/* simple test program to print out the utf8proc properties for a codepoint */
+
+#include "tests.h"
+
+int main(int argc, char **argv)
+{
+     int i;
+
+     for (i = 1; i < argc; ++i) {
+          unsigned int c;
+          if (!strcmp(argv[i], "-V")) {
+               printf("utf8proc version %s\n", utf8proc_version());
+               continue;
+          }
+          check(sscanf(argv[i],"%x",&c) == 1, "invalid hex input %s", argv[i]);
+          const utf8proc_property_t *p = utf8proc_get_property(c);
+          printf("U+%s:\n"
+                 "  category = %s\n"
+                 "  combining_class = %d\n"
+                 "  bidi_class = %d\n"
+                 "  decomp_type = %d\n"
+                 "  uppercase_mapping = %x\n"
+                 "  lowercase_mapping = %x\n"
+                 "  titlecase_mapping = %x\n"
+                 "  comb_index = %d\n"
+                 "  bidi_mirrored = %d\n"
+                 "  comp_exclusion = %d\n"
+                 "  ignorable = %d\n"
+                 "  control_boundary = %d\n"
+                 "  boundclass = %d\n"
+                 "  charwidth = %d\n",
+                 argv[i],
+                 utf8proc_category_string(c),
+                 p->combining_class,
+                 p->bidi_class,
+                 p->decomp_type,
+                 utf8proc_toupper(c),
+                 utf8proc_tolower(c),
+                 utf8proc_totitle(c),
+                 p->comb_index,
+                 p->bidi_mirrored,
+                 p->comp_exclusion,
+                 p->ignorable,
+                 p->control_boundary,
+                 p->boundclass,
+                 utf8proc_charwidth(c));
+     }
+     return 0;
+}
--- a/3rdparty/utf8proc/test/tests.c
+++ b/3rdparty/utf8proc/test/tests.c
@ -0,0 +1,46 @@
+/* Common functions for our test programs. */
+
+#include "tests.h"
+
+size_t lineno = 0;
+
+void check(int cond, const char *format, ...)
+{
+     if (!cond) {
+          va_list args;
+          fprintf(stderr, "line %zd: ", lineno);
+          va_start(args, format);
+          vfprintf(stderr, format, args);
+          va_end(args);
+          fprintf(stderr, "\n");
+          exit(1);
+     }
+}
+
+size_t skipspaces(const char *buf, size_t i)
+{
+    while (isspace(buf[i])) ++i;
+    return i;
+}
+
+/* if buf points to a sequence of codepoints encoded as hexadecimal strings,
+   separated by whitespace, and terminated by any character not in
+   [0-9a-fA-F] or whitespace, then stores the corresponding utf8 string
+   in dest, returning the number of bytes read from buf */
+size_t encode(char *dest, const char *buf)
+{
+     size_t i = 0, j, d = 0;
+     for (;;) {
+          int c;
+          i = skipspaces(buf, i);
+          for (j=i; buf[j] && strchr("0123456789abcdef", tolower(buf[j])); ++j)
+               ; /* find end of hex input */
+          if (j == i) { /* no codepoint found */
+               dest[d] = 0; /* NUL-terminate destination string */
+               return i + 1;
+          }
+          check(sscanf(buf + i, "%x", (unsigned int *)&c) == 1, "invalid hex input %s", buf+i);
+          i = j; /* skip to char after hex input */
+          d += utf8proc_encode_char(c, (utf8proc_uint8_t *) (dest + d));
+     }
+}
--- a/3rdparty/utf8proc/test/tests.h
+++ b/3rdparty/utf8proc/test/tests.h
@ -0,0 +1,23 @@
+/* Common functions and includes for our test programs. */
+
+/*
+ * Set feature macro to enable getline() and wcwidth().
+ *
+ * Please refer to section 2.2.1 of POSIX.1-2008:
+ * http://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_02_01_02
+ */
+#define _XOPEN_SOURCE 700
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "../utf8proc.h"
+
+extern size_t lineno;
+
+void check(int cond, const char *format, ...);
+size_t skipspaces(const char *buf, size_t i);
+size_t encode(char *dest, const char *buf);
--- a/3rdparty/utf8proc/test/valid.c
+++ b/3rdparty/utf8proc/test/valid.c
@ -0,0 +1,41 @@
+#include "tests.h"
+#include <ctype.h>
+#include <wchar.h>
+
+int main(int argc, char **argv)
+{
+     int c, error = 0;
+
+     (void) argc; /* unused */
+     (void) argv; /* unused */
+
+     /* some simple sanity tests of  */
+     for (c = 0; c < 0xd800; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0xe000; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%04x) -> true\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110000; c++) {
+        if (!utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> false\n", c);
+           error++;
+        }
+     }
+     for (;c < 0x110010; c++) {
+        if (utf8proc_codepoint_valid(c)) {
+           fprintf(stderr, "Failed: codepoint_valid(%06x) -> true\n", c);
+           error++;
+        }
+     }
+     check(!error, "utf8proc_codepoint_valid FAILED %d tests.", error);
+     printf("Validity tests SUCCEEDED.\n");
+
+     return 0;
+}
--- a/3rdparty/utf8proc/utf8proc.c
+++ b/3rdparty/utf8proc/utf8proc.c
@ -1,4 +1,6 @@
+/* -*- mode: c; c-basic-offset: 2; tab-width: 2; indent-tabs-mode: nil -*- */
 /*
+ *  Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
 *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
 *
 *  Permission is hereby granted, free of charge, to any person obtaining a
@ -43,7 +45,7 @@
 #include "utf8proc_data.c"


-const int8_t utf8proc_utf8class[256] = {
+UTF8PROC_DLLEXPORT const utf8proc_int8_t utf8proc_utf8class[256] = {
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@ -81,25 +83,16 @@ const int8_t utf8proc_utf8class[256] = {
 #define UTF8PROC_HANGUL_S_START  0xAC00
 #define UTF8PROC_HANGUL_S_END    0xD7A4

-
-#define UTF8PROC_BOUNDCLASS_START    0
-#define UTF8PROC_BOUNDCLASS_OTHER    1
-#define UTF8PROC_BOUNDCLASS_CR       2
-#define UTF8PROC_BOUNDCLASS_LF       3
-#define UTF8PROC_BOUNDCLASS_CONTROL  4
-#define UTF8PROC_BOUNDCLASS_EXTEND   5
-#define UTF8PROC_BOUNDCLASS_L        6
-#define UTF8PROC_BOUNDCLASS_V        7
-#define UTF8PROC_BOUNDCLASS_T        8
-#define UTF8PROC_BOUNDCLASS_LV       9
-#define UTF8PROC_BOUNDCLASS_LVT     10
-
-
-const char *utf8proc_version(void) {
-  return "1.1.5";
+/* Should follow semantic-versioning rules (semver.org) based on API
+   compatibility.  (Note that the shared-library version number will
+   be different, being based on ABI compatibility.): */
+#define STRINGIZEx(x) #x
+#define STRINGIZE(x) STRINGIZEx(x)
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void) {
+  return STRINGIZE(UTF8PROC_VERSION_MAJOR) "." STRINGIZE(UTF8PROC_VERSION_MINOR) "." STRINGIZE(UTF8PROC_VERSION_PATCH) "";
 }

-const char *utf8proc_errmsg(ssize_t errcode) {
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode) {
  switch (errcode) {
    case UTF8PROC_ERROR_NOMEM:
    return "Memory for processing UTF-8 data could not be allocated.";
@ -112,58 +105,64 @@ const char *utf8proc_errmsg(ssize_t errcode) {
    case UTF8PROC_ERROR_INVALIDOPTS:
    return "Invalid options for UTF-8 processing chosen.";
    default:
-    return "An unknown error occured while processing UTF-8 data.";
+    return "An unknown error occurred while processing UTF-8 data.";
  }
 }

-ssize_t utf8proc_iterate(
-  const uint8_t *str, ssize_t strlen, int32_t *dst
+#define utf_cont(ch)  (((ch) & 0xc0) == 0x80)
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *dst
 ) {
-  int length;
-  int i;
-  int32_t uc = -1;
+  utf8proc_uint32_t uc;
+  const utf8proc_uint8_t *end;
+
  *dst = -1;
  if (!strlen) return 0;
-  length = utf8proc_utf8class[str[0]];
-  if (!length) return UTF8PROC_ERROR_INVALIDUTF8;
-  if (strlen >= 0 && length > strlen) return UTF8PROC_ERROR_INVALIDUTF8;
-  for (i=1; i<length; i++) {
-    if ((str[i] & 0xC0) != 0x80) return UTF8PROC_ERROR_INVALIDUTF8;
+  end = str + ((strlen < 0) ? 4 : strlen);
+  uc = *str++;
+  if (uc < 0x80) {
+    *dst = uc;
+    return 1;
  }
-  switch (length) {
-    case 1:
-    uc = str[0];
-    break;
-    case 2:
-    uc = ((str[0] & 0x1F) <<  6) + (str[1] & 0x3F);
-    if (uc < 0x80) uc = -1;
-    break;
-    case 3:
-    uc = ((str[0] & 0x0F) << 12) + ((str[1] & 0x3F) <<  6)
-      + (str[2] & 0x3F);
-    if (uc < 0x800 || (uc >= 0xD800 && uc < 0xE000) ||
-      (uc >= 0xFDD0 && uc < 0xFDF0)) uc = -1;
-    break;
-    case 4:
-    uc = ((str[0] & 0x07) << 18) + ((str[1] & 0x3F) << 12)
-      + ((str[2] & 0x3F) <<  6) + (str[3] & 0x3F);
-    if (uc < 0x10000 || uc >= 0x110000) uc = -1;
-    break;
+  // Must be between 0xc2 and 0xf4 inclusive to be valid
+  if ((uc - 0xc2) > (0xf4-0xc2)) return UTF8PROC_ERROR_INVALIDUTF8;
+  if (uc < 0xe0) {         // 2-byte sequence
+     // Must have valid continuation character
+     if (str >= end || !utf_cont(*str)) return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = ((uc & 0x1f)<<6) | (*str & 0x3f);
+     return 2;
  }
-  if (uc < 0 || ((uc & 0xFFFF) >= 0xFFFE))
-    return UTF8PROC_ERROR_INVALIDUTF8;
-  *dst = uc;
-  return length;
+  if (uc < 0xf0) {        // 3-byte sequence
+     if ((str + 1 >= end) || !utf_cont(*str) || !utf_cont(str[1]))
+        return UTF8PROC_ERROR_INVALIDUTF8;
+     // Check for surrogate chars
+     if (uc == 0xed && *str > 0x9f)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     uc = ((uc & 0xf)<<12) | ((*str & 0x3f)<<6) | (str[1] & 0x3f);
+     if (uc < 0x800)
+         return UTF8PROC_ERROR_INVALIDUTF8;
+     *dst = uc;
+     return 3;
+  }
+  // 4-byte sequence
+  // Must have 3 valid continuation characters
+  if ((str + 2 >= end) || !utf_cont(*str) || !utf_cont(str[1]) || !utf_cont(str[2]))
+     return UTF8PROC_ERROR_INVALIDUTF8;
+  // Make sure in correct range (0x10000 - 0x10ffff)
+  if (uc == 0xf0) {
+    if (*str < 0x90) return UTF8PROC_ERROR_INVALIDUTF8;
+  } else if (uc == 0xf4) {
+    if (*str > 0x8f) return UTF8PROC_ERROR_INVALIDUTF8;
+  }
+  *dst = ((uc & 7)<<18) | ((*str & 0x3f)<<12) | ((str[1] & 0x3f)<<6) | (str[2] & 0x3f);
+  return 4;
 }

-bool utf8proc_codepoint_valid(int32_t uc) {
-  if (uc < 0 || uc >= 0x110000 ||
-    ((uc & 0xFFFF) >= 0xFFFE) || (uc >= 0xD800 && uc < 0xE000) ||
-    (uc >= 0xFDD0 && uc < 0xFDF0)) return false;
-  else return true;
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t uc) {
+    return (((utf8proc_uint32_t)uc)-0xd800 > 0x07ff) && ((utf8proc_uint32_t)uc < 0x110000);
 }

-ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
  if (uc < 0x00) {
    return 0;
  } else if (uc < 0x80) {
@ -173,12 +172,8 @@ ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
    dst[0] = 0xC0 + (uc >> 6);
    dst[1] = 0x80 + (uc & 0x3F);
    return 2;
-  } else if (uc == 0xFFFF) {
-    dst[0] = 0xFF;
-    return 1;
-  } else if (uc == 0xFFFE) {
-    dst[0] = 0xFE;
-    return 1;
+  // Note: we allow encoding 0xd800-0xdfff here, so as not to change
+  // the API, however, these are actually invalid in UTF-8
  } else if (uc < 0x10000) {
    dst[0] = 0xE0 + (uc >> 12);
    dst[1] = 0x80 + ((uc >> 6) & 0x3F);
@ -193,7 +188,39 @@ ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst) {
  } else return 0;
 }

-const utf8proc_property_t *utf8proc_get_property(int32_t uc) {
+/* internal "unsafe" version that does not check whether uc is in range */
+static utf8proc_ssize_t unsafe_encode_char(utf8proc_int32_t uc, utf8proc_uint8_t *dst) {
+   if (uc < 0x00) {
+      return 0;
+   } else if (uc < 0x80) {
+      dst[0] = uc;
+      return 1;
+   } else if (uc < 0x800) {
+      dst[0] = 0xC0 + (uc >> 6);
+      dst[1] = 0x80 + (uc & 0x3F);
+      return 2;
+   } else if (uc == 0xFFFF) {
+       dst[0] = 0xFF;
+       return 1;
+   } else if (uc == 0xFFFE) {
+       dst[0] = 0xFE;
+       return 1;
+   } else if (uc < 0x10000) {
+      dst[0] = 0xE0 + (uc >> 12);
+      dst[1] = 0x80 + ((uc >> 6) & 0x3F);
+      dst[2] = 0x80 + (uc & 0x3F);
+      return 3;
+   } else if (uc < 0x110000) {
+      dst[0] = 0xF0 + (uc >> 18);
+      dst[1] = 0x80 + ((uc >> 12) & 0x3F);
+      dst[2] = 0x80 + ((uc >> 6) & 0x3F);
+      dst[3] = 0x80 + (uc & 0x3F);
+      return 4;
+   } else return 0;
+}
+
+/* internal "unsafe" version that does not check whether uc is in range */
+static const utf8proc_property_t *unsafe_get_property(utf8proc_int32_t uc) {
  /* ASSERT: uc >= 0 && uc < 0x110000 */
  return utf8proc_properties + (
    utf8proc_stage2table[
@ -202,22 +229,185 @@ const utf8proc_property_t *utf8proc_get_property(int32_t uc) {
  );
 }

+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t uc) {
+  return uc < 0 || uc >= 0x110000 ? utf8proc_properties : unsafe_get_property(uc);
+}
+
+/* return whether there is a grapheme break between boundclasses lbc and tbc
+   (according to the definition of extended grapheme clusters)
+
+  Rule numbering refers to TR29 Version 29 (Unicode 9.0.0):
+  http://www.unicode.org/reports/tr29/tr29-29.html
+
+  CAVEATS:
+   Please note that evaluation of GB10 (grapheme breaks between emoji zwj sequences)
+   and GB 12/13 (regional indicator code points) require knowledge of previous characters
+   and are thus not handled by this function. This may result in an incorrect break before
+   an E_Modifier class codepoint and an incorrectly missing break between two
+   REGIONAL_INDICATOR class code points if such support does not exist in the caller.
+
+   See the special support in grapheme_break_extended, for required bookkeeping by the caller.
+*/
+static utf8proc_bool grapheme_break_simple(int lbc, int tbc) {
+  return
+    (lbc == UTF8PROC_BOUNDCLASS_START) ? true :       // GB1
+    (lbc == UTF8PROC_BOUNDCLASS_CR &&                 // GB3
+     tbc == UTF8PROC_BOUNDCLASS_LF) ? false :         // ---
+    (lbc >= UTF8PROC_BOUNDCLASS_CR && lbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB4
+    (tbc >= UTF8PROC_BOUNDCLASS_CR && tbc <= UTF8PROC_BOUNDCLASS_CONTROL) ? true :  // GB5
+    (lbc == UTF8PROC_BOUNDCLASS_L &&                  // GB6
+     (tbc == UTF8PROC_BOUNDCLASS_L ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_LV ||                // ---
+      tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :      // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LV ||                // GB7
+      lbc == UTF8PROC_BOUNDCLASS_V) &&                // ---
+     (tbc == UTF8PROC_BOUNDCLASS_V ||                 // ---
+      tbc == UTF8PROC_BOUNDCLASS_T)) ? false :        // ---
+    ((lbc == UTF8PROC_BOUNDCLASS_LVT ||               // GB8
+      lbc == UTF8PROC_BOUNDCLASS_T) &&                // ---
+     tbc == UTF8PROC_BOUNDCLASS_T) ? false :          // ---
+    (tbc == UTF8PROC_BOUNDCLASS_EXTEND ||             // GB9
+     tbc == UTF8PROC_BOUNDCLASS_ZWJ ||                // ---
+     tbc == UTF8PROC_BOUNDCLASS_SPACINGMARK ||        // GB9a
+     lbc == UTF8PROC_BOUNDCLASS_PREPEND) ? false :    // GB9b
+    ((lbc == UTF8PROC_BOUNDCLASS_E_BASE ||            // GB10 (requires additional handling below)
+      lbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&       // ----
+     tbc == UTF8PROC_BOUNDCLASS_E_MODIFIER) ? false : // ----
+    (lbc == UTF8PROC_BOUNDCLASS_ZWJ &&                         // GB11
+     (tbc == UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ ||             // ----
+      tbc == UTF8PROC_BOUNDCLASS_E_BASE_GAZ)) ? false :        // ----
+    (lbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR &&          // GB12/13 (requires additional handling below)
+     tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR) ? false :  // ----
+    true; // GB999
+}
+
+static utf8proc_bool grapheme_break_extended(int lbc, int tbc, utf8proc_int32_t *state)
+{
+  int lbc_override = lbc;
+  if (state && *state != UTF8PROC_BOUNDCLASS_START)
+    lbc_override = *state;
+  utf8proc_bool break_permitted = grapheme_break_simple(lbc_override, tbc);
+  if (state) {
+    // Special support for GB 12/13 made possible by GB999. After two RI
+    // class codepoints we want to force a break. Do this by resetting the
+    // second RI's bound class to UTF8PROC_BOUNDCLASS_OTHER, to force a break
+    // after that character according to GB999 (unless of course such a break is
+    // forbidden by a different rule such as GB9).
+    if (*state == tbc && tbc == UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR)
+      *state = UTF8PROC_BOUNDCLASS_OTHER;
+    // Special support for GB10. Fold any EXTEND codepoints into the previous
+    // boundclass if we're dealing with an emoji base boundclass.
+    else if ((*state == UTF8PROC_BOUNDCLASS_E_BASE      ||
+              *state == UTF8PROC_BOUNDCLASS_E_BASE_GAZ) &&
+             tbc == UTF8PROC_BOUNDCLASS_EXTEND)
+      *state = UTF8PROC_BOUNDCLASS_E_BASE;
+    else
+      *state = tbc;
+  }
+  return break_permitted;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t c1, utf8proc_int32_t c2, utf8proc_int32_t *state) {
+
+  return grapheme_break_extended(utf8proc_get_property(c1)->boundclass,
+                                 utf8proc_get_property(c2)->boundclass,
+                                 state);
+}
+
+
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t c1, utf8proc_int32_t c2) {
+  return utf8proc_grapheme_break_stateful(c1, c2, NULL);
+}
+
+static utf8proc_int32_t seqindex_decode_entry(const utf8proc_uint16_t **entry)
+{
+  utf8proc_int32_t entry_cp = **entry;
+  if ((entry_cp & 0xF800) == 0xD800) {
+    *entry = *entry + 1;
+    entry_cp = ((entry_cp & 0x03FF) << 10) | (**entry & 0x03FF);
+    entry_cp += 0x10000;
+  }
+  return entry_cp;
+}
+
+static utf8proc_int32_t seqindex_decode_index(const utf8proc_uint32_t seqindex)
+{
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex];
+  return seqindex_decode_entry(&entry);
+}
+
+static utf8proc_ssize_t seqindex_write_char_decomposed(utf8proc_uint16_t seqindex, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
+  utf8proc_ssize_t written = 0;
+  const utf8proc_uint16_t *entry = &utf8proc_sequences[seqindex & 0x1FFF];
+  int len = seqindex >> 13;
+  if (len >= 7) {
+    len = *entry;
+    entry++;
+  }
+  for (; len >= 0; entry++, len--) {
+    utf8proc_int32_t entry_cp = seqindex_decode_entry(&entry);
+
+    written += utf8proc_decompose_char(entry_cp, dst+written,
+      (bufsize > written) ? (bufsize - written) : 0, options,
+    last_boundclass);
+    if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
+  }
+  return written;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cl = utf8proc_get_property(c)->lowercase_seqindex;
+  return cl != UINT16_MAX ? seqindex_decode_index(cl) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->uppercase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c)
+{
+  utf8proc_int32_t cu = utf8proc_get_property(c)->titlecase_seqindex;
+  return cu != UINT16_MAX ? seqindex_decode_index(cu) : c;
+}
+
+/* return a character width analogous to wcwidth (except portable and
+   hopefully less buggy than most system wcwidth functions). */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->charwidth;
+}
+
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t c) {
+  return utf8proc_get_property(c)->category;
+}
+
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t c) {
+  static const char s[][3] = {"Cn","Lu","Ll","Lt","Lm","Lo","Mn","Mc","Me","Nd","Nl","No","Pc","Pd","Ps","Pe","Pi","Pf","Po","Sm","Sc","Sk","So","Zs","Zl","Zp","Cc","Cf","Cs","Co"};
+  return s[utf8proc_category(c)];
+}
+
+
+
 #define utf8proc_decompose_lump(replacement_uc) \
  return utf8proc_decompose_char((replacement_uc), dst, bufsize, \
  options & ~UTF8PROC_LUMP, last_boundclass)

-ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
-    int options, int *last_boundclass) {
-  /* ASSERT: uc >= 0 && uc < 0x110000 */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(utf8proc_int32_t uc, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize, utf8proc_option_t options, int *last_boundclass) {
  const utf8proc_property_t *property;
  utf8proc_propval_t category;
-  int32_t hangul_sindex;
-  property = utf8proc_get_property(uc);
+  utf8proc_int32_t hangul_sindex;
+  if (uc < 0 || uc >= 0x110000) return UTF8PROC_ERROR_NOTASSIGNED;
+  property = unsafe_get_property(uc);
  category = property->category;
  hangul_sindex = uc - UTF8PROC_HANGUL_SBASE;
  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
    if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT) {
-      int32_t hangul_tindex;
+      utf8proc_int32_t hangul_tindex;
      if (bufsize >= 1) {
        dst[0] = UTF8PROC_HANGUL_LBASE +
          hangul_sindex / UTF8PROC_HANGUL_NCOUNT;
@ -268,79 +458,20 @@ ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
      category == UTF8PROC_CATEGORY_ME) return 0;
  }
  if (options & UTF8PROC_CASEFOLD) {
-    if (property->casefold_mapping) {
-      const int32_t *casefold_entry;
-      ssize_t written = 0;
-      for (casefold_entry = property->casefold_mapping;
-          *casefold_entry >= 0; casefold_entry++) {
-        written += utf8proc_decompose_char(*casefold_entry, dst+written,
-          (bufsize > written) ? (bufsize - written) : 0, options,
-          last_boundclass);
-        if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
-      }
-      return written;
+    if (property->casefold_seqindex != UINT16_MAX) {
+      return seqindex_write_char_decomposed(property->casefold_seqindex, dst, bufsize, options, last_boundclass);
    }
  }
  if (options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) {
-    if (property->decomp_mapping &&
+    if (property->decomp_seqindex != UINT16_MAX &&
        (!property->decomp_type || (options & UTF8PROC_COMPAT))) {
-      const int32_t *decomp_entry;
-      ssize_t written = 0;
-      for (decomp_entry = property->decomp_mapping;
-          *decomp_entry >= 0; decomp_entry++) {
-        written += utf8proc_decompose_char(*decomp_entry, dst+written,
-          (bufsize > written) ? (bufsize - written) : 0, options,
-        last_boundclass);
-        if (written < 0) return UTF8PROC_ERROR_OVERFLOW;
-      }
-      return written;
+      return seqindex_write_char_decomposed(property->decomp_seqindex, dst, bufsize, options, last_boundclass);
    }
  }
  if (options & UTF8PROC_CHARBOUND) {
-    bool boundary;
-    int tbc, lbc;
-    tbc =
-      (uc == 0x000D) ? UTF8PROC_BOUNDCLASS_CR :
-      (uc == 0x000A) ? UTF8PROC_BOUNDCLASS_LF :
-      ((category == UTF8PROC_CATEGORY_ZL ||
-        category == UTF8PROC_CATEGORY_ZP ||
-        category == UTF8PROC_CATEGORY_CC ||
-        category == UTF8PROC_CATEGORY_CF) &&
-        !(uc == 0x200C || uc == 0x200D)) ? UTF8PROC_BOUNDCLASS_CONTROL :
-      property->extend ? UTF8PROC_BOUNDCLASS_EXTEND :
-      ((uc >= UTF8PROC_HANGUL_L_START && uc < UTF8PROC_HANGUL_L_END) ||
-        uc == UTF8PROC_HANGUL_L_FILLER) ? UTF8PROC_BOUNDCLASS_L :
-      (uc >= UTF8PROC_HANGUL_V_START && uc < UTF8PROC_HANGUL_V_END) ?
-        UTF8PROC_BOUNDCLASS_V :
-      (uc >= UTF8PROC_HANGUL_T_START && uc < UTF8PROC_HANGUL_T_END) ?
-        UTF8PROC_BOUNDCLASS_T :
-      (uc >= UTF8PROC_HANGUL_S_START && uc < UTF8PROC_HANGUL_S_END) ? (
-        ((uc-UTF8PROC_HANGUL_SBASE) % UTF8PROC_HANGUL_TCOUNT == 0) ?
-          UTF8PROC_BOUNDCLASS_LV : UTF8PROC_BOUNDCLASS_LVT
-      ) :
-      UTF8PROC_BOUNDCLASS_OTHER;
-    lbc = *last_boundclass;
-    boundary =
-      (tbc == UTF8PROC_BOUNDCLASS_EXTEND) ? false :
-      (lbc == UTF8PROC_BOUNDCLASS_START) ? true :
-      (lbc == UTF8PROC_BOUNDCLASS_CR &&
-       tbc == UTF8PROC_BOUNDCLASS_LF) ? false :
-      (lbc == UTF8PROC_BOUNDCLASS_CONTROL) ? true :
-      (tbc == UTF8PROC_BOUNDCLASS_CONTROL) ? true :
-      (lbc == UTF8PROC_BOUNDCLASS_L &&
-       (tbc == UTF8PROC_BOUNDCLASS_L ||
-        tbc == UTF8PROC_BOUNDCLASS_V ||
-        tbc == UTF8PROC_BOUNDCLASS_LV ||
-        tbc == UTF8PROC_BOUNDCLASS_LVT)) ? false :
-      ((lbc == UTF8PROC_BOUNDCLASS_LV ||
-        lbc == UTF8PROC_BOUNDCLASS_V) &&
-       (tbc == UTF8PROC_BOUNDCLASS_V ||
-        tbc == UTF8PROC_BOUNDCLASS_T)) ? false :
-      ((lbc == UTF8PROC_BOUNDCLASS_LVT ||
-        lbc == UTF8PROC_BOUNDCLASS_T) &&
-       tbc == UTF8PROC_BOUNDCLASS_T) ? false :
-       true;
-    *last_boundclass = tbc;
+    utf8proc_bool boundary;
+    int tbc = property->boundclass;
+    boundary = grapheme_break_extended(*last_boundclass, tbc, last_boundclass);
    if (boundary) {
      if (bufsize >= 1) dst[0] = 0xFFFF;
      if (bufsize >= 2) dst[1] = uc;
@ -351,26 +482,26 @@ ssize_t utf8proc_decompose_char(int32_t uc, int32_t *dst, ssize_t bufsize,
  return 1;
 }

-ssize_t utf8proc_decompose(
-  const uint8_t *str, ssize_t strlen,
-  int32_t *buffer, ssize_t bufsize, int options
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
 ) {
  /* strlen will be ignored, if UTF8PROC_NULLTERM is set in options */
-  ssize_t wpos = 0;
+  utf8proc_ssize_t wpos = 0;
  if ((options & UTF8PROC_COMPOSE) && (options & UTF8PROC_DECOMPOSE))
    return UTF8PROC_ERROR_INVALIDOPTS;
  if ((options & UTF8PROC_STRIPMARK) &&
      !(options & UTF8PROC_COMPOSE) && !(options & UTF8PROC_DECOMPOSE))
    return UTF8PROC_ERROR_INVALIDOPTS;
  {
-    int32_t uc;
-    ssize_t rpos = 0;
-    ssize_t decomp_result;
+    utf8proc_int32_t uc;
+    utf8proc_ssize_t rpos = 0;
+    utf8proc_ssize_t decomp_result;
    int boundclass = UTF8PROC_BOUNDCLASS_START;
    while (1) {
      if (options & UTF8PROC_NULLTERM) {
        rpos += utf8proc_iterate(str + rpos, -1, &uc);
-        /* checking of return value is not neccessary,
+        /* checking of return value is not necessary,
           as 'uc' is < 0 in case of error */
        if (uc < 0) return UTF8PROC_ERROR_INVALIDUTF8;
        if (rpos < 0) return UTF8PROC_ERROR_OVERFLOW;
@ -387,19 +518,20 @@ ssize_t utf8proc_decompose(
      if (decomp_result < 0) return decomp_result;
      wpos += decomp_result;
      /* prohibiting integer overflows due to too long strings: */
-      if (wpos < 0 || wpos > SSIZE_MAX/sizeof(int32_t)/2)
+      if (wpos < 0 ||
+          wpos > (utf8proc_ssize_t)(SSIZE_MAX/sizeof(utf8proc_int32_t)/2))
        return UTF8PROC_ERROR_OVERFLOW;
    }
  }
  if ((options & (UTF8PROC_COMPOSE|UTF8PROC_DECOMPOSE)) && bufsize >= wpos) {
-    ssize_t pos = 0;
+    utf8proc_ssize_t pos = 0;
    while (pos < wpos-1) {
-      int32_t uc1, uc2;
+      utf8proc_int32_t uc1, uc2;
      const utf8proc_property_t *property1, *property2;
      uc1 = buffer[pos];
      uc2 = buffer[pos+1];
-      property1 = utf8proc_get_property(uc1);
-      property2 = utf8proc_get_property(uc2);
+      property1 = unsafe_get_property(uc1);
+      property2 = unsafe_get_property(uc2);
      if (property1->combining_class > property2->combining_class &&
          property2->combining_class > 0) {
        buffer[pos] = uc2;
@ -413,13 +545,13 @@ ssize_t utf8proc_decompose(
  return wpos;
 }

-ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options) {
  /* UTF8PROC_NULLTERM option will be ignored, 'length' is never ignored
     ASSERT: 'buffer' has one spare byte of free space at the end! */
  if (options & (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS | UTF8PROC_STRIPCC)) {
-    ssize_t rpos;
-    ssize_t wpos = 0;
-    int32_t uc;
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t uc;
    for (rpos = 0; rpos < length; rpos++) {
      uc = buffer[rpos];
      if (uc == 0x000D && rpos < length-1 && buffer[rpos+1] == 0x000A) rpos++;
@ -448,23 +580,23 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
    length = wpos;
  }
  if (options & UTF8PROC_COMPOSE) {
-    int32_t *starter = NULL;
-    int32_t current_char;
+    utf8proc_int32_t *starter = NULL;
+    utf8proc_int32_t current_char;
    const utf8proc_property_t *starter_property = NULL, *current_property;
    utf8proc_propval_t max_combining_class = -1;
-    ssize_t rpos;
-    ssize_t wpos = 0;
-    int32_t composition;
+    utf8proc_ssize_t rpos;
+    utf8proc_ssize_t wpos = 0;
+    utf8proc_int32_t composition;
    for (rpos = 0; rpos < length; rpos++) {
      current_char = buffer[rpos];
-      current_property = utf8proc_get_property(current_char);
+      current_property = unsafe_get_property(current_char);
      if (starter && current_property->combining_class > max_combining_class) {
        /* combination perhaps possible */
-        int32_t hangul_lindex;
-        int32_t hangul_sindex;
+        utf8proc_int32_t hangul_lindex;
+        utf8proc_int32_t hangul_sindex;
        hangul_lindex = *starter - UTF8PROC_HANGUL_LBASE;
        if (hangul_lindex >= 0 && hangul_lindex < UTF8PROC_HANGUL_LCOUNT) {
-          int32_t hangul_vindex;
+          utf8proc_int32_t hangul_vindex;
          hangul_vindex = current_char - UTF8PROC_HANGUL_VBASE;
          if (hangul_vindex >= 0 && hangul_vindex < UTF8PROC_HANGUL_VCOUNT) {
            *starter = UTF8PROC_HANGUL_SBASE +
@ -477,7 +609,7 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
        hangul_sindex = *starter - UTF8PROC_HANGUL_SBASE;
        if (hangul_sindex >= 0 && hangul_sindex < UTF8PROC_HANGUL_SCOUNT &&
            (hangul_sindex % UTF8PROC_HANGUL_TCOUNT) == 0) {
-          int32_t hangul_tindex;
+          utf8proc_int32_t hangul_tindex;
          hangul_tindex = current_char - UTF8PROC_HANGUL_TBASE;
          if (hangul_tindex >= 0 && hangul_tindex < UTF8PROC_HANGUL_TCOUNT) {
            *starter += hangul_tindex;
@ -486,19 +618,26 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
          }
        }
        if (!starter_property) {
-          starter_property = utf8proc_get_property(*starter);
+          starter_property = unsafe_get_property(*starter);
        }
-        if (starter_property->comb1st_index >= 0 &&
-            current_property->comb2nd_index >= 0) {
-          composition = utf8proc_combinations[
-            starter_property->comb1st_index +
-            current_property->comb2nd_index
-          ];
-          if (composition >= 0 && (!(options & UTF8PROC_STABLE) ||
-              !(utf8proc_get_property(composition)->comp_exclusion))) {
-            *starter = composition;
-            starter_property = NULL;
-            continue;
+        if (starter_property->comb_index < 0x8000 &&
+            current_property->comb_index != UINT16_MAX &&
+            current_property->comb_index >= 0x8000) {
+          int sidx = starter_property->comb_index;
+          int idx = (current_property->comb_index & 0x3FFF) - utf8proc_combinations[sidx];
+          if (idx >= 0 && idx <= utf8proc_combinations[sidx + 1] ) {
+            idx += sidx + 2;
+            if (current_property->comb_index & 0x4000) {
+              composition = (utf8proc_combinations[idx] << 16) | utf8proc_combinations[idx+1];
+            } else
+              composition = utf8proc_combinations[idx];
+
+            if (composition > 0 && (!(options & UTF8PROC_STABLE) ||
+                !(unsafe_get_property(composition)->comp_exclusion))) {
+              *starter = composition;
+              starter_property = NULL;
+              continue;
+            }
          }
        }
      }
@ -517,26 +656,33 @@ ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options) {
    length = wpos;
  }
  {
-    ssize_t rpos, wpos = 0;
-    int32_t uc;
-    for (rpos = 0; rpos < length; rpos++) {
-      uc = buffer[rpos];
-      wpos += utf8proc_encode_char(uc, ((uint8_t *)buffer) + wpos);
+    utf8proc_ssize_t rpos, wpos = 0;
+    utf8proc_int32_t uc;
+    if (options & UTF8PROC_CHARBOUND) {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += unsafe_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
+    } else {
+        for (rpos = 0; rpos < length; rpos++) {
+            uc = buffer[rpos];
+            wpos += utf8proc_encode_char(uc, ((utf8proc_uint8_t *)buffer) + wpos);
+        }
    }
-    ((uint8_t *)buffer)[wpos] = 0;
+    ((utf8proc_uint8_t *)buffer)[wpos] = 0;
    return wpos;
  }
 }

-ssize_t utf8proc_map(
-  const uint8_t *str, ssize_t strlen, uint8_t **dstptr, int options
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
 ) {
-  int32_t *buffer;
-  ssize_t result;
+  utf8proc_int32_t *buffer;
+  utf8proc_ssize_t result;
  *dstptr = NULL;
  result = utf8proc_decompose(str, strlen, NULL, 0, options);
  if (result < 0) return result;
-  buffer = malloc(result * sizeof(int32_t) + 1);
+  buffer = (utf8proc_int32_t *) malloc(result * sizeof(utf8proc_int32_t) + 1);
  if (!buffer) return UTF8PROC_ERROR_NOMEM;
  result = utf8proc_decompose(str, strlen, buffer, result, options);
  if (result < 0) {
@ -549,37 +695,37 @@ ssize_t utf8proc_map(
    return result;
  }
  {
-    int32_t *newptr;
-    newptr = realloc(buffer, (size_t)result+1);
+    utf8proc_int32_t *newptr;
+    newptr = (utf8proc_int32_t *) realloc(buffer, (size_t)result+1);
    if (newptr) buffer = newptr;
  }
-  *dstptr = (uint8_t *)buffer;
+  *dstptr = (utf8proc_uint8_t *)buffer;
  return result;
 }

-uint8_t *utf8proc_NFD(const uint8_t *str) {
-  uint8_t *retval;
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
    UTF8PROC_DECOMPOSE);
  return retval;
 }

-uint8_t *utf8proc_NFC(const uint8_t *str) {
-  uint8_t *retval;
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
    UTF8PROC_COMPOSE);
  return retval;
 }

-uint8_t *utf8proc_NFKD(const uint8_t *str) {
-  uint8_t *retval;
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
    UTF8PROC_DECOMPOSE | UTF8PROC_COMPAT);
  return retval;
 }

-uint8_t *utf8proc_NFKC(const uint8_t *str) {
-  uint8_t *retval;
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str) {
+  utf8proc_uint8_t *retval;
  utf8proc_map(str, 0, &retval, UTF8PROC_NULLTERM | UTF8PROC_STABLE |
    UTF8PROC_COMPOSE | UTF8PROC_COMPAT);
  return retval;
--- a/3rdparty/utf8proc/utf8proc.h
+++ b/3rdparty/utf8proc/utf8proc.h
@ -1,78 +1,130 @@
 /*
- *  Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
+ * Copyright (c) 2015 Steven G. Johnson, Jiahao Chen, Peter Colberg, Tony Kelman, Scott P. Jones, and other contributors.
+ * Copyright (c) 2009 Public Software Group e. V., Berlin, Germany
 *
- *  Permission is hereby granted, free of charge, to any person obtaining a
- *  copy of this software and associated documentation files (the "Software"),
- *  to deal in the Software without restriction, including without limitation
- *  the rights to use, copy, modify, merge, publish, distribute, sublicense,
- *  and/or sell copies of the Software, and to permit persons to whom the
- *  Software is furnished to do so, subject to the following conditions:
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
 *
- *  The above copyright notice and this permission notice shall be included in
- *  all copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
 *
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- *  DEALINGS IN THE SOFTWARE.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
 */


-/*
- *  File name:    utf8proc.h
+/**
+ * @mainpage
 *
- *  Description:
- *  Header files for libutf8proc, which is a mapping tool for UTF-8 strings
- *  with following features:
- *  - decomposing and composing of strings
- *  - replacing compatibility characters with their equivalents
- *  - stripping of "default ignorable characters"
- *    like SOFT-HYPHEN or ZERO-WIDTH-SPACE
- *  - folding of certain characters for string comparison
- *    (e.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-")
- *    (see "LUMP" option)
- *  - optional rejection of strings containing non-assigned code points
- *  - stripping of control characters
- *  - stripping of character marks (accents, etc.)
- *  - transformation of LF, CRLF, CR and NEL to line-feed (LF)
- *    or to the unicode chararacters for paragraph separation (PS)
- *    or line separation (LS).
- *  - unicode case folding (for case insensitive string comparisons)
- *  - rejection of illegal UTF-8 data
- *    (i.e. UTF-8 encoded UTF-16 surrogates)
- *  - support for korean hangul characters
- *  Unicode Version 5.0.0 is supported.
+ * utf8proc is a free/open-source (MIT/expat licensed) C library
+ * providing Unicode normalization, case-folding, and other operations
+ * for strings in the UTF-8 encoding, supporting Unicode version
+ * 8.0.0.  See the utf8proc home page (http://julialang.org/utf8proc/)
+ * for downloads and other information, or the source code on github
+ * (https://github.com/JuliaLang/utf8proc).
+ *
+ * For the utf8proc API documentation, see: @ref utf8proc.h
+ *
+ * The features of utf8proc include:
+ *
+ * - Transformation of strings (@ref utf8proc_map) to:
+ *    - decompose (@ref UTF8PROC_DECOMPOSE) or compose (@ref UTF8PROC_COMPOSE) Unicode combining characters (http://en.wikipedia.org/wiki/Combining_character)
+ *    - canonicalize Unicode compatibility characters (@ref UTF8PROC_COMPAT)
+ *    - strip "ignorable" (@ref UTF8PROC_IGNORE) characters, control characters (@ref UTF8PROC_STRIPCC), or combining characters such as accents (@ref UTF8PROC_STRIPMARK)
+ *    - case-folding (@ref UTF8PROC_CASEFOLD)
+ * - Unicode normalization: @ref utf8proc_NFD, @ref utf8proc_NFC, @ref utf8proc_NFKD, @ref utf8proc_NFKC
+ * - Detecting grapheme boundaries (@ref utf8proc_grapheme_break and @ref UTF8PROC_CHARBOUND)
+ * - Character-width computation: @ref utf8proc_charwidth
+ * - Classification of characters by Unicode category: @ref utf8proc_category and @ref utf8proc_category_string
+ * - Encode (@ref utf8proc_encode_char) and decode (@ref utf8proc_iterate) Unicode codepoints to/from UTF-8.
 */

+/** @file */

 #ifndef UTF8PROC_H
 #define UTF8PROC_H

+/** @name API version
+ *
+ * The utf8proc API version MAJOR.MINOR.PATCH, following
+ * semantic-versioning rules (http://semver.org) based on API
+ * compatibility.
+ *
+ * This is also returned at runtime by @ref utf8proc_version; however, the
+ * runtime version may append a string like "-dev" to the version number
+ * for prerelease versions.
+ *
+ * @note The shared-library version number in the Makefile
+ *       (and CMakeLists.txt, and MANIFEST) may be different,
+ *       being based on ABI compatibility rather than API compatibility.
+ */
+/** @{ */
+/** The MAJOR version number (increased when backwards API compatibility is broken). */
+#define UTF8PROC_VERSION_MAJOR 2
+/** The MINOR version number (increased when new functionality is added in a backwards-compatible manner). */
+#define UTF8PROC_VERSION_MINOR 0
+/** The PATCH version (increased for fixes that do not change the API). */
+#define UTF8PROC_VERSION_PATCH 2
+/** @} */

 #include <stdlib.h>
 #include <sys/types.h>
 #ifdef _MSC_VER
-typedef signed char int8_t;
-typedef unsigned char uint8_t;
-typedef short int16_t;
-typedef unsigned short uint16_t;
-typedef int int32_t;
-#ifdef _WIN64
-#define ssize_t __int64
-#else
-#define ssize_t int
-#endif
-typedef unsigned char bool;
+typedef signed char utf8proc_int8_t;
+typedef unsigned char utf8proc_uint8_t;
+typedef short utf8proc_int16_t;
+typedef unsigned short utf8proc_uint16_t;
+typedef int utf8proc_int32_t;
+typedef unsigned int utf8proc_uint32_t;
+#  ifdef _WIN64
+typedef __int64 utf8proc_ssize_t;
+typedef unsigned __int64 utf8proc_size_t;
+#  else
+typedef int utf8proc_ssize_t;
+typedef unsigned int utf8proc_size_t;
+#  endif
+#  ifndef __cplusplus
+typedef unsigned char utf8proc_bool;
 enum {false, true};
+#  else
+typedef bool utf8proc_bool;
+#  endif
 #else
-#include <stdbool.h>
-#include <inttypes.h>
+#  include <stdbool.h>
+#  include <inttypes.h>
+typedef int8_t utf8proc_int8_t;
+typedef uint8_t utf8proc_uint8_t;
+typedef int16_t utf8proc_int16_t;
+typedef uint16_t utf8proc_uint16_t;
+typedef int32_t utf8proc_int32_t;
+typedef uint32_t utf8proc_uint32_t;
+typedef size_t utf8proc_size_t;
+typedef ssize_t utf8proc_ssize_t;
+typedef bool utf8proc_bool;
 #endif
 #include <limits.h>

+#ifdef _WIN32
+#  ifdef UTF8PROC_EXPORTS
+#    define UTF8PROC_DLLEXPORT __declspec(dllexport)
+#  else
+#    define UTF8PROC_DLLEXPORT __declspec(dllimport)
+#  endif
+#elif __GNUC__ >= 4
+#  define UTF8PROC_DLLEXPORT __attribute__ ((visibility("default")))
+#else
+#  define UTF8PROC_DLLEXPORT
+#endif
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@ -81,301 +133,497 @@ extern "C" {
 #define SSIZE_MAX ((size_t)SIZE_MAX/2)
 #endif

-#define UTF8PROC_NULLTERM  (1<<0)
-#define UTF8PROC_STABLE    (1<<1)
-#define UTF8PROC_COMPAT    (1<<2)
-#define UTF8PROC_COMPOSE   (1<<3)
-#define UTF8PROC_DECOMPOSE (1<<4)
-#define UTF8PROC_IGNORE    (1<<5)
-#define UTF8PROC_REJECTNA  (1<<6)
-#define UTF8PROC_NLF2LS    (1<<7)
-#define UTF8PROC_NLF2PS    (1<<8)
-#define UTF8PROC_NLF2LF    (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS)
-#define UTF8PROC_STRIPCC   (1<<9)
-#define UTF8PROC_CASEFOLD  (1<<10)
-#define UTF8PROC_CHARBOUND (1<<11)
-#define UTF8PROC_LUMP      (1<<12)
-#define UTF8PROC_STRIPMARK (1<<13)
-/*
- *  Flags being regarded by several functions in the library:
- *  NULLTERM:  The given UTF-8 input is NULL terminated.
- *  STABLE:    Unicode Versioning Stability has to be respected.
- *  COMPAT:    Compatiblity decomposition
- *             (i.e. formatting information is lost)
- *  COMPOSE:   Return a result with composed characters.
- *  DECOMPOSE: Return a result with decomposed characters.
- *  IGNORE:    Strip "default ignorable characters"
- *  REJECTNA:  Return an error, if the input contains unassigned
- *             code points.
- *  NLF2LS:    Indicating that NLF-sequences (LF, CRLF, CR, NEL) are
- *             representing a line break, and should be converted to the
- *             unicode character for line separation (LS).
- *  NLF2PS:    Indicating that NLF-sequences are representing a paragraph
- *             break, and should be converted to the unicode character for
- *             paragraph separation (PS).
- *  NLF2LF:    Indicating that the meaning of NLF-sequences is unknown.
- *  STRIPCC:   Strips and/or convers control characters.
- *             NLF-sequences are transformed into space, except if one of
- *             the NLF2LS/PS/LF options is given.
- *             HorizontalTab (HT) and FormFeed (FF) are treated as a
- *             NLF-sequence in this case.
- *             All other control characters are simply removed.
- *  CASEFOLD:  Performs unicode case folding, to be able to do a
- *             case-insensitive string comparison.
- *  CHARBOUND: Inserts 0xFF bytes at the beginning of each sequence which
- *             is representing a single grapheme cluster (see UAX#29).
- *  LUMP:      Lumps certain characters together
- *             (e.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-").
- *             (See lump.txt for details.)
- *             If NLF2LF is set, this includes a transformation of
- *             paragraph and line separators to ASCII line-feed (LF).
- *  STRIPMARK: Strips all character markings
- *             (non-spacing, spacing and enclosing) (i.e. accents)
- *             NOTE: this option works only with COMPOSE or DECOMPOSE
- */
+#ifndef UINT16_MAX
+#  define UINT16_MAX 65535U
+#endif

+/**
+ * Option flags used by several functions in the library.
+ */
+typedef enum {
+  /** The given UTF-8 input is NULL terminated. */
+  UTF8PROC_NULLTERM  = (1<<0),
+  /** Unicode Versioning Stability has to be respected. */
+  UTF8PROC_STABLE    = (1<<1),
+  /** Compatibility decomposition (i.e. formatting information is lost). */
+  UTF8PROC_COMPAT    = (1<<2),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_COMPOSE   = (1<<3),
+  /** Return a result with decomposed characters. */
+  UTF8PROC_DECOMPOSE = (1<<4),
+  /** Strip "default ignorable characters" such as SOFT-HYPHEN or ZERO-WIDTH-SPACE. */
+  UTF8PROC_IGNORE    = (1<<5),
+  /** Return an error, if the input contains unassigned codepoints. */
+  UTF8PROC_REJECTNA  = (1<<6),
+  /**
+   * Indicating that NLF-sequences (LF, CRLF, CR, NEL) are representing a
+   * line break, and should be converted to the codepoint for line
+   * separation (LS).
+   */
+  UTF8PROC_NLF2LS    = (1<<7),
+  /**
+   * Indicating that NLF-sequences are representing a paragraph break, and
+   * should be converted to the codepoint for paragraph separation
+   * (PS).
+   */
+  UTF8PROC_NLF2PS    = (1<<8),
+  /** Indicating that the meaning of NLF-sequences is unknown. */
+  UTF8PROC_NLF2LF    = (UTF8PROC_NLF2LS | UTF8PROC_NLF2PS),
+  /** Strips and/or convers control characters.
+   *
+   * NLF-sequences are transformed into space, except if one of the
+   * NLF2LS/PS/LF options is given. HorizontalTab (HT) and FormFeed (FF)
+   * are treated as a NLF-sequence in this case.  All other control
+   * characters are simply removed.
+   */
+  UTF8PROC_STRIPCC   = (1<<9),
+  /**
+   * Performs unicode case folding, to be able to do a case-insensitive
+   * string comparison.
+   */
+  UTF8PROC_CASEFOLD  = (1<<10),
+  /**
+   * Inserts 0xFF bytes at the beginning of each sequence which is
+   * representing a single grapheme cluster (see UAX#29).
+   */
+  UTF8PROC_CHARBOUND = (1<<11),
+  /** Lumps certain characters together.
+   *
+   * E.g. HYPHEN U+2010 and MINUS U+2212 to ASCII "-". See lump.md for details.
+   *
+   * If NLF2LF is set, this includes a transformation of paragraph and
+   * line separators to ASCII line-feed (LF).
+   */
+  UTF8PROC_LUMP      = (1<<12),
+  /** Strips all character markings.
+   *
+   * This includes non-spacing, spacing and enclosing (i.e. accents).
+   * @note This option works only with @ref UTF8PROC_COMPOSE or
+   *       @ref UTF8PROC_DECOMPOSE
+   */
+  UTF8PROC_STRIPMARK = (1<<13),
+} utf8proc_option_t;
+
+/** @name Error codes
+ * Error codes being returned by almost all functions.
+ */
+/** @{ */
+/** Memory could not be allocated. */
 #define UTF8PROC_ERROR_NOMEM -1
+/** The given string is too long to be processed. */
 #define UTF8PROC_ERROR_OVERFLOW -2
+/** The given string is not a legal UTF-8 string. */
 #define UTF8PROC_ERROR_INVALIDUTF8 -3
+/** The @ref UTF8PROC_REJECTNA flag was set and an unassigned codepoint was found. */
 #define UTF8PROC_ERROR_NOTASSIGNED -4
+/** Invalid options have been used. */
 #define UTF8PROC_ERROR_INVALIDOPTS -5
-/*
- *  Error codes being returned by almost all functions:
- *  ERROR_NOMEM:       Memory could not be allocated.
- *  ERROR_OVERFLOW:    The given string is too long to be processed.
- *  ERROR_INVALIDUTF8: The given string is not a legal UTF-8 string.
- *  ERROR_NOTASSIGNED: The REJECTNA flag was set,
- *                     and an unassigned code point was found.
- *  ERROR_INVALIDOPTS: Invalid options have been used.
- */
+/** @} */

-typedef int16_t utf8proc_propval_t;
+/* @name Types */
+
+/** Holds the value of a property. */
+typedef utf8proc_int16_t utf8proc_propval_t;
+
+/** Struct containing information about a codepoint. */
 typedef struct utf8proc_property_struct {
+  /**
+   * Unicode category.
+   * @see utf8proc_category_t.
+   */
  utf8proc_propval_t category;
  utf8proc_propval_t combining_class;
+  /**
+   * Bidirectional class.
+   * @see utf8proc_bidi_class_t.
+   */
  utf8proc_propval_t bidi_class;
+  /**
+   * @anchor Decomposition type.
+   * @see utf8proc_decomp_type_t.
+   */
  utf8proc_propval_t decomp_type;
-  const int32_t *decomp_mapping;
+  utf8proc_uint16_t decomp_seqindex;
+  utf8proc_uint16_t casefold_seqindex;
+  utf8proc_uint16_t uppercase_seqindex;
+  utf8proc_uint16_t lowercase_seqindex;
+  utf8proc_uint16_t titlecase_seqindex;
+  utf8proc_uint16_t comb_index;
  unsigned bidi_mirrored:1;
-  int32_t uppercase_mapping;
-  int32_t lowercase_mapping;
-  int32_t titlecase_mapping;
-  int32_t comb1st_index;
-  int32_t comb2nd_index;
  unsigned comp_exclusion:1;
+  /**
+   * Can this codepoint be ignored?
+   *
+   * Used by @ref utf8proc_decompose_char when @ref UTF8PROC_IGNORE is
+   * passed as an option.
+   */
  unsigned ignorable:1;
  unsigned control_boundary:1;
-  unsigned extend:1;
-  const int32_t *casefold_mapping;
+  /** The width of the codepoint. */
+  unsigned charwidth:2;
+  unsigned pad:2;
+  /**
+   * Boundclass.
+   * @see utf8proc_boundclass_t.
+   */
+  unsigned boundclass:8;
 } utf8proc_property_t;

-#define UTF8PROC_CATEGORY_LU  1
-#define UTF8PROC_CATEGORY_LL  2
-#define UTF8PROC_CATEGORY_LT  3
-#define UTF8PROC_CATEGORY_LM  4
-#define UTF8PROC_CATEGORY_LO  5
-#define UTF8PROC_CATEGORY_MN  6
-#define UTF8PROC_CATEGORY_MC  7
-#define UTF8PROC_CATEGORY_ME  8
-#define UTF8PROC_CATEGORY_ND  9
-#define UTF8PROC_CATEGORY_NL 10
-#define UTF8PROC_CATEGORY_NO 11
-#define UTF8PROC_CATEGORY_PC 12
-#define UTF8PROC_CATEGORY_PD 13
-#define UTF8PROC_CATEGORY_PS 14
-#define UTF8PROC_CATEGORY_PE 15
-#define UTF8PROC_CATEGORY_PI 16
-#define UTF8PROC_CATEGORY_PF 17
-#define UTF8PROC_CATEGORY_PO 18
-#define UTF8PROC_CATEGORY_SM 19
-#define UTF8PROC_CATEGORY_SC 20
-#define UTF8PROC_CATEGORY_SK 21
-#define UTF8PROC_CATEGORY_SO 22
-#define UTF8PROC_CATEGORY_ZS 23
-#define UTF8PROC_CATEGORY_ZL 24
-#define UTF8PROC_CATEGORY_ZP 25
-#define UTF8PROC_CATEGORY_CC 26
-#define UTF8PROC_CATEGORY_CF 27
-#define UTF8PROC_CATEGORY_CS 28
-#define UTF8PROC_CATEGORY_CO 29
-#define UTF8PROC_CATEGORY_CN 30
-#define UTF8PROC_BIDI_CLASS_L    1
-#define UTF8PROC_BIDI_CLASS_LRE  2
-#define UTF8PROC_BIDI_CLASS_LRO  3
-#define UTF8PROC_BIDI_CLASS_R    4
-#define UTF8PROC_BIDI_CLASS_AL   5
-#define UTF8PROC_BIDI_CLASS_RLE  6
-#define UTF8PROC_BIDI_CLASS_RLO  7
-#define UTF8PROC_BIDI_CLASS_PDF  8
-#define UTF8PROC_BIDI_CLASS_EN   9
-#define UTF8PROC_BIDI_CLASS_ES  10
-#define UTF8PROC_BIDI_CLASS_ET  11
-#define UTF8PROC_BIDI_CLASS_AN  12
-#define UTF8PROC_BIDI_CLASS_CS  13
-#define UTF8PROC_BIDI_CLASS_NSM 14
-#define UTF8PROC_BIDI_CLASS_BN  15
-#define UTF8PROC_BIDI_CLASS_B   16
-#define UTF8PROC_BIDI_CLASS_S   17
-#define UTF8PROC_BIDI_CLASS_WS  18
-#define UTF8PROC_BIDI_CLASS_ON  19
-#define UTF8PROC_DECOMP_TYPE_FONT      1
-#define UTF8PROC_DECOMP_TYPE_NOBREAK   2
-#define UTF8PROC_DECOMP_TYPE_INITIAL   3
-#define UTF8PROC_DECOMP_TYPE_MEDIAL    4
-#define UTF8PROC_DECOMP_TYPE_FINAL     5
-#define UTF8PROC_DECOMP_TYPE_ISOLATED  6
-#define UTF8PROC_DECOMP_TYPE_CIRCLE    7
-#define UTF8PROC_DECOMP_TYPE_SUPER     8
-#define UTF8PROC_DECOMP_TYPE_SUB       9
-#define UTF8PROC_DECOMP_TYPE_VERTICAL 10
-#define UTF8PROC_DECOMP_TYPE_WIDE     11
-#define UTF8PROC_DECOMP_TYPE_NARROW   12
-#define UTF8PROC_DECOMP_TYPE_SMALL    13
-#define UTF8PROC_DECOMP_TYPE_SQUARE   14
-#define UTF8PROC_DECOMP_TYPE_FRACTION 15
-#define UTF8PROC_DECOMP_TYPE_COMPAT   16
+/** Unicode categories. */
+typedef enum {
+  UTF8PROC_CATEGORY_CN  = 0, /**< Other, not assigned */
+  UTF8PROC_CATEGORY_LU  = 1, /**< Letter, uppercase */
+  UTF8PROC_CATEGORY_LL  = 2, /**< Letter, lowercase */
+  UTF8PROC_CATEGORY_LT  = 3, /**< Letter, titlecase */
+  UTF8PROC_CATEGORY_LM  = 4, /**< Letter, modifier */
+  UTF8PROC_CATEGORY_LO  = 5, /**< Letter, other */
+  UTF8PROC_CATEGORY_MN  = 6, /**< Mark, nonspacing */
+  UTF8PROC_CATEGORY_MC  = 7, /**< Mark, spacing combining */
+  UTF8PROC_CATEGORY_ME  = 8, /**< Mark, enclosing */
+  UTF8PROC_CATEGORY_ND  = 9, /**< Number, decimal digit */
+  UTF8PROC_CATEGORY_NL = 10, /**< Number, letter */
+  UTF8PROC_CATEGORY_NO = 11, /**< Number, other */
+  UTF8PROC_CATEGORY_PC = 12, /**< Punctuation, connector */
+  UTF8PROC_CATEGORY_PD = 13, /**< Punctuation, dash */
+  UTF8PROC_CATEGORY_PS = 14, /**< Punctuation, open */
+  UTF8PROC_CATEGORY_PE = 15, /**< Punctuation, close */
+  UTF8PROC_CATEGORY_PI = 16, /**< Punctuation, initial quote */
+  UTF8PROC_CATEGORY_PF = 17, /**< Punctuation, final quote */
+  UTF8PROC_CATEGORY_PO = 18, /**< Punctuation, other */
+  UTF8PROC_CATEGORY_SM = 19, /**< Symbol, math */
+  UTF8PROC_CATEGORY_SC = 20, /**< Symbol, currency */
+  UTF8PROC_CATEGORY_SK = 21, /**< Symbol, modifier */
+  UTF8PROC_CATEGORY_SO = 22, /**< Symbol, other */
+  UTF8PROC_CATEGORY_ZS = 23, /**< Separator, space */
+  UTF8PROC_CATEGORY_ZL = 24, /**< Separator, line */
+  UTF8PROC_CATEGORY_ZP = 25, /**< Separator, paragraph */
+  UTF8PROC_CATEGORY_CC = 26, /**< Other, control */
+  UTF8PROC_CATEGORY_CF = 27, /**< Other, format */
+  UTF8PROC_CATEGORY_CS = 28, /**< Other, surrogate */
+  UTF8PROC_CATEGORY_CO = 29, /**< Other, private use */
+} utf8proc_category_t;

-extern const int8_t utf8proc_utf8class[256];
+/** Bidirectional character classes. */
+typedef enum {
+  UTF8PROC_BIDI_CLASS_L     = 1, /**< Left-to-Right */
+  UTF8PROC_BIDI_CLASS_LRE   = 2, /**< Left-to-Right Embedding */
+  UTF8PROC_BIDI_CLASS_LRO   = 3, /**< Left-to-Right Override */
+  UTF8PROC_BIDI_CLASS_R     = 4, /**< Right-to-Left */
+  UTF8PROC_BIDI_CLASS_AL    = 5, /**< Right-to-Left Arabic */
+  UTF8PROC_BIDI_CLASS_RLE   = 6, /**< Right-to-Left Embedding */
+  UTF8PROC_BIDI_CLASS_RLO   = 7, /**< Right-to-Left Override */
+  UTF8PROC_BIDI_CLASS_PDF   = 8, /**< Pop Directional Format */
+  UTF8PROC_BIDI_CLASS_EN    = 9, /**< European Number */
+  UTF8PROC_BIDI_CLASS_ES   = 10, /**< European Separator */
+  UTF8PROC_BIDI_CLASS_ET   = 11, /**< European Number Terminator */
+  UTF8PROC_BIDI_CLASS_AN   = 12, /**< Arabic Number */
+  UTF8PROC_BIDI_CLASS_CS   = 13, /**< Common Number Separator */
+  UTF8PROC_BIDI_CLASS_NSM  = 14, /**< Nonspacing Mark */
+  UTF8PROC_BIDI_CLASS_BN   = 15, /**< Boundary Neutral */
+  UTF8PROC_BIDI_CLASS_B    = 16, /**< Paragraph Separator */
+  UTF8PROC_BIDI_CLASS_S    = 17, /**< Segment Separator */
+  UTF8PROC_BIDI_CLASS_WS   = 18, /**< Whitespace */
+  UTF8PROC_BIDI_CLASS_ON   = 19, /**< Other Neutrals */
+  UTF8PROC_BIDI_CLASS_LRI  = 20, /**< Left-to-Right Isolate */
+  UTF8PROC_BIDI_CLASS_RLI  = 21, /**< Right-to-Left Isolate */
+  UTF8PROC_BIDI_CLASS_FSI  = 22, /**< First Strong Isolate */
+  UTF8PROC_BIDI_CLASS_PDI  = 23, /**< Pop Directional Isolate */
+} utf8proc_bidi_class_t;

-const char *utf8proc_version(void);
+/** Decomposition type. */
+typedef enum {
+  UTF8PROC_DECOMP_TYPE_FONT      = 1, /**< Font */
+  UTF8PROC_DECOMP_TYPE_NOBREAK   = 2, /**< Nobreak */
+  UTF8PROC_DECOMP_TYPE_INITIAL   = 3, /**< Initial */
+  UTF8PROC_DECOMP_TYPE_MEDIAL    = 4, /**< Medial */
+  UTF8PROC_DECOMP_TYPE_FINAL     = 5, /**< Final */
+  UTF8PROC_DECOMP_TYPE_ISOLATED  = 6, /**< Isolated */
+  UTF8PROC_DECOMP_TYPE_CIRCLE    = 7, /**< Circle */
+  UTF8PROC_DECOMP_TYPE_SUPER     = 8, /**< Super */
+  UTF8PROC_DECOMP_TYPE_SUB       = 9, /**< Sub */
+  UTF8PROC_DECOMP_TYPE_VERTICAL = 10, /**< Vertical */
+  UTF8PROC_DECOMP_TYPE_WIDE     = 11, /**< Wide */
+  UTF8PROC_DECOMP_TYPE_NARROW   = 12, /**< Narrow */
+  UTF8PROC_DECOMP_TYPE_SMALL    = 13, /**< Small */
+  UTF8PROC_DECOMP_TYPE_SQUARE   = 14, /**< Square */
+  UTF8PROC_DECOMP_TYPE_FRACTION = 15, /**< Fraction */
+  UTF8PROC_DECOMP_TYPE_COMPAT   = 16, /**< Compat */
+} utf8proc_decomp_type_t;

-const char *utf8proc_errmsg(ssize_t errcode);
-/*
- *  Returns a static error string for the given error code.
+/** Boundclass property. (TR29) */
+typedef enum {
+  UTF8PROC_BOUNDCLASS_START              =  0, /**< Start */
+  UTF8PROC_BOUNDCLASS_OTHER              =  1, /**< Other */
+  UTF8PROC_BOUNDCLASS_CR                 =  2, /**< Cr */
+  UTF8PROC_BOUNDCLASS_LF                 =  3, /**< Lf */
+  UTF8PROC_BOUNDCLASS_CONTROL            =  4, /**< Control */
+  UTF8PROC_BOUNDCLASS_EXTEND             =  5, /**< Extend */
+  UTF8PROC_BOUNDCLASS_L                  =  6, /**< L */
+  UTF8PROC_BOUNDCLASS_V                  =  7, /**< V */
+  UTF8PROC_BOUNDCLASS_T                  =  8, /**< T */
+  UTF8PROC_BOUNDCLASS_LV                 =  9, /**< Lv */
+  UTF8PROC_BOUNDCLASS_LVT                = 10, /**< Lvt */
+  UTF8PROC_BOUNDCLASS_REGIONAL_INDICATOR = 11, /**< Regional indicator */
+  UTF8PROC_BOUNDCLASS_SPACINGMARK        = 12, /**< Spacingmark */
+  UTF8PROC_BOUNDCLASS_PREPEND            = 13, /**< Prepend */
+  UTF8PROC_BOUNDCLASS_ZWJ                = 14, /**< Zero Width Joiner */
+  UTF8PROC_BOUNDCLASS_E_BASE             = 15, /**< Emoji Base */
+  UTF8PROC_BOUNDCLASS_E_MODIFIER         = 16, /**< Emoji Modifier */
+  UTF8PROC_BOUNDCLASS_GLUE_AFTER_ZWJ     = 17, /**< Glue_After_ZWJ */
+  UTF8PROC_BOUNDCLASS_E_BASE_GAZ         = 18, /**< E_BASE + GLUE_AFTER_ZJW */
+} utf8proc_boundclass_t;
+
+/**
+ * Array containing the byte lengths of a UTF-8 encoded codepoint based
+ * on the first byte.
 */
+UTF8PROC_DLLEXPORT extern const utf8proc_int8_t utf8proc_utf8class[256];

-ssize_t utf8proc_iterate(const uint8_t *str, ssize_t strlen, int32_t *dst);
-/*
- *  Reads a single char from the UTF-8 sequence being pointed to by 'str'.
- *  The maximum number of bytes read is 'strlen', unless 'strlen' is
- *  negative.
- *  If a valid unicode char could be read, it is stored in the variable
- *  being pointed to by 'dst', otherwise that variable will be set to -1.
- *  In case of success the number of bytes read is returned, otherwise a
- *  negative error code is returned.
+/**
+ * Returns the utf8proc API version as a string MAJOR.MINOR.PATCH
+ * (http://semver.org format), possibly with a "-dev" suffix for
+ * development versions.
 */
+UTF8PROC_DLLEXPORT const char *utf8proc_version(void);

-bool utf8proc_codepoint_valid(int32_t uc);
-/*
- *  Returns 1, if the given unicode code-point is valid, otherwise 0.
+/**
+ * Returns an informative error string for the given utf8proc error code
+ * (e.g. the error codes returned by @ref utf8proc_map).
 */
+UTF8PROC_DLLEXPORT const char *utf8proc_errmsg(utf8proc_ssize_t errcode);

-ssize_t utf8proc_encode_char(int32_t uc, uint8_t *dst);
-/*
- *  Encodes the unicode char with the code point 'uc' as an UTF-8 string in
- *  the byte array being pointed to by 'dst'. This array has to be at least
- *  4 bytes long.
- *  In case of success the number of bytes written is returned,
- *  otherwise 0.
- *  This function does not check if 'uc' is a valid unicode code point.
+/**
+ * Reads a single codepoint from the UTF-8 sequence being pointed to by `str`.
+ * The maximum number of bytes read is `strlen`, unless `strlen` is
+ * negative (in which case up to 4 bytes are read).
+ *
+ * If a valid codepoint could be read, it is stored in the variable
+ * pointed to by `codepoint_ref`, otherwise that variable will be set to -1.
+ * In case of success, the number of bytes read is returned; otherwise, a
+ * negative error code is returned.
 */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_iterate(const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_int32_t *codepoint_ref);

-const utf8proc_property_t *utf8proc_get_property(int32_t uc);
-/*
- *  Returns a pointer to a (constant) struct containing information about
- *  the unicode char with the given code point 'uc'.
- *  If the character is not existent a pointer to a special struct is
- *  returned, where 'category' is a NULL pointer.
- *  WARNING: The parameter 'uc' has to be in the range of 0x0000 to
- *           0x10FFFF, otherwise the program might crash!
+/**
+ * Check if a codepoint is valid (regardless of whether it has been
+ * assigned a value by the current Unicode standard).
+ *
+ * @return 1 if the given `codepoint` is valid and otherwise return 0.
 */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_codepoint_valid(utf8proc_int32_t codepoint);

-ssize_t utf8proc_decompose_char(
-  int32_t uc, int32_t *dst, ssize_t bufsize,
-  int options, int *last_boundclass
+/**
+ * Encodes the codepoint as an UTF-8 string in the byte array pointed
+ * to by `dst`. This array must be at least 4 bytes long.
+ *
+ * In case of success the number of bytes written is returned, and
+ * otherwise 0 is returned.
+ *
+ * This function does not check whether `codepoint` is valid Unicode.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_encode_char(utf8proc_int32_t codepoint, utf8proc_uint8_t *dst);
+
+/**
+ * Look up the properties for a given codepoint.
+ *
+ * @param codepoint The Unicode codepoint.
+ *
+ * @returns
+ * A pointer to a (constant) struct containing information about
+ * the codepoint.
+ * @par
+ * If the codepoint is unassigned or invalid, a pointer to a special struct is
+ * returned in which `category` is 0 (@ref UTF8PROC_CATEGORY_CN).
+ */
+UTF8PROC_DLLEXPORT const utf8proc_property_t *utf8proc_get_property(utf8proc_int32_t codepoint);
+
+/** Decompose a codepoint into an array of codepoints.
+ *
+ * @param codepoint the codepoint.
+ * @param dst the destination buffer.
+ * @param bufsize the size of the destination buffer.
+ * @param options one or more of the following flags:
+ * - @ref UTF8PROC_REJECTNA  - return an error `codepoint` is unassigned
+ * - @ref UTF8PROC_IGNORE    - strip "default ignorable" codepoints
+ * - @ref UTF8PROC_CASEFOLD  - apply Unicode casefolding
+ * - @ref UTF8PROC_COMPAT    - replace certain codepoints with their
+ *                             compatibility decomposition
+ * - @ref UTF8PROC_CHARBOUND - insert 0xFF bytes before each grapheme cluster
+ * - @ref UTF8PROC_LUMP      - lump certain different codepoints together
+ * - @ref UTF8PROC_STRIPMARK - remove all character marks
+ * @param last_boundclass
+ * Pointer to an integer variable containing
+ * the previous codepoint's boundary class if the @ref UTF8PROC_CHARBOUND
+ * option is used.  Otherwise, this parameter is ignored.
+ *
+ * @return
+ * In case of success, the number of codepoints written is returned; in case
+ * of an error, a negative error code is returned (@ref utf8proc_errmsg).
+ * @par
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose_char(
+  utf8proc_int32_t codepoint, utf8proc_int32_t *dst, utf8proc_ssize_t bufsize,
+  utf8proc_option_t options, int *last_boundclass
 );
-/*
- *  Writes a decomposition of the unicode char 'uc' into the array being
- *  pointed to by 'dst'.
- *  Following flags in the 'options' field are regarded:
- *  REJECTNA:  an unassigned unicode code point leads to an error
- *  IGNORE:    "default ignorable" chars are stripped
- *  CASEFOLD:  unicode casefolding is applied
- *  COMPAT:    replace certain characters with their
- *             compatibility decomposition
- *  CHARBOUND: Inserts 0xFF bytes before each grapheme cluster
- *  LUMP:      lumps certain different characters together
- *  STRIPMARK: removes all character marks
- *  The pointer 'last_boundclass' has to point to an integer variable which
- *  is storing the last character boundary class, if the CHARBOUND option
- *  is used.
- *  In case of success the number of chars written is returned,
- *  in case of an error, a negative error code is returned.
- *  If the number of written chars would be bigger than 'bufsize',
- *  the buffer (up to 'bufsize') has inpredictable data, and the needed
- *  buffer size is returned.
- *  WARNING: The parameter 'uc' has to be in the range of 0x0000 to
- *           0x10FFFF, otherwise the program might crash!
- */

-ssize_t utf8proc_decompose(
-  const uint8_t *str, ssize_t strlen,
-  int32_t *buffer, ssize_t bufsize, int options
+/**
+ * The same as @ref utf8proc_decompose_char, but acts on a whole UTF-8
+ * string and orders the decomposed sequences correctly.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in `options` is set, processing
+ * will be stopped, when a NULL byte is encounted, otherwise `strlen`
+ * bytes are processed.  The result (in the form of 32-bit unicode
+ * codepoints) is written into the buffer being pointed to by
+ * `buffer` (which must contain at least `bufsize` entries).  In case of
+ * success, the number of codepoints written is returned; in case of an
+ * error, a negative error code is returned (@ref utf8proc_errmsg).
+ *
+ * If the number of written codepoints would be bigger than `bufsize`, the
+ * required buffer size is returned, while the buffer will be overwritten with
+ * undefined data.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_decompose(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen,
+  utf8proc_int32_t *buffer, utf8proc_ssize_t bufsize, utf8proc_option_t options
 );
-/*
- *  Does the same as 'utf8proc_decompose_char', but acts on a whole UTF-8
- *  string, and orders the decomposed sequences correctly.
- *  If the NULLTERM flag in 'options' is set, processing will be stopped,
- *  when a NULL byte is encounted, otherwise 'strlen' bytes are processed.
- *  The result in form of unicode code points is written into the buffer
- *  being pointed to by 'buffer', having the length of 'bufsize' entries.
- *  In case of success the number of chars written is returned,
- *  in case of an error, a negative error code is returned.
- *  If the number of written chars would be bigger than 'bufsize',
- *  the buffer (up to 'bufsize') has inpredictable data, and the needed
- *  buffer size is returned.
- */

-ssize_t utf8proc_reencode(int32_t *buffer, ssize_t length, int options);
-/*
- *  Reencodes the sequence of unicode characters given by the pointer
- *  'buffer' and 'length' as UTF-8.
- *  The result is stored in the same memory area where the data is read.
- *  Following flags in the 'options' field are regarded:
- *  NLF2LS:  converts LF, CRLF, CR and NEL into LS
- *  NLF2PS:  converts LF, CRLF, CR and NEL into PS
- *  NLF2LF:  converts LF, CRLF, CR and NEL into LF
- *  STRIPCC: strips or converts all non-affected control characters
- *  COMPOSE: tries to combine decomposed characters into composite
- *           characters
- *  STABLE:  prohibits combining characters which would violate
- *           the unicode versioning stability
- *  In case of success the length of the resulting UTF-8 string is
- *  returned, otherwise a negative error code is returned.
- *  WARNING: The amount of free space being pointed to by 'buffer', has to
- *           exceed the amount of the input data by one byte, and the
- *           entries of the array pointed to by 'str' have to be in the
- *           range of 0x0000 to 0x10FFFF, otherwise the program might
- *           crash!
+/**
+ * Reencodes the sequence of `length` codepoints pointed to by `buffer`
+ * UTF-8 data in-place (i.e., the result is also stored in `buffer`).
+ *
+ * @param buffer the (native-endian UTF-32) unicode codepoints to re-encode.
+ * @param length the length (in codepoints) of the buffer.
+ * @param options a bitwise or (`|`) of one or more of the following flags:
+ * - @ref UTF8PROC_NLF2LS  - convert LF, CRLF, CR and NEL into LS
+ * - @ref UTF8PROC_NLF2PS  - convert LF, CRLF, CR and NEL into PS
+ * - @ref UTF8PROC_NLF2LF  - convert LF, CRLF, CR and NEL into LF
+ * - @ref UTF8PROC_STRIPCC - strip or convert all non-affected control characters
+ * - @ref UTF8PROC_COMPOSE - try to combine decomposed codepoints into composite
+ *                           codepoints
+ * - @ref UTF8PROC_STABLE  - prohibit combining characters that would violate
+ *                           the unicode versioning stability
+ *
+ * @return
+ * In case of success, the length (in bytes) of the resulting UTF-8 string is
+ * returned; otherwise, a negative error code is returned (@ref utf8proc_errmsg).
+ *
+ * @warning The amount of free space pointed to by `buffer` must
+ *          exceed the amount of the input data by one byte, and the
+ *          entries of the array pointed to by `str` have to be in the
+ *          range `0x0000` to `0x10FFFF`. Otherwise, the program might crash!
 */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_reencode(utf8proc_int32_t *buffer, utf8proc_ssize_t length, utf8proc_option_t options);

-ssize_t utf8proc_map(
-  const uint8_t *str, ssize_t strlen, uint8_t **dstptr, int options
+/**
+ * Given a pair of consecutive codepoints, return whether a grapheme break is
+ * permitted between them (as defined by the extended grapheme clusters in UAX#29).
+ *
+ * @param state Beginning with Version 29 (Unicode 9.0.0), this algorithm requires
+ *              state to break graphemes. This state can be passed in as a pointer
+ *              in the `state` argument and should initially be set to 0. If the
+ *              state is not passed in (i.e. a null pointer is passed), UAX#29 rules
+ *              GB10/12/13 which require this state will not be applied, essentially
+ *              matching the rules in Unicode 8.0.0.
+ *
+ * @warning If the state parameter is used, `utf8proc_grapheme_break_stateful` must
+ *          be called IN ORDER on ALL potential breaks in a string.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break_stateful(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2, utf8proc_int32_t *state);
+
+/**
+ * Same as @ref utf8proc_grapheme_break_stateful, except without support for the
+ * Unicode 9 additions to the algorithm. Supported for legacy reasons.
+ */
+UTF8PROC_DLLEXPORT utf8proc_bool utf8proc_grapheme_break(
+    utf8proc_int32_t codepoint1, utf8proc_int32_t codepoint2);
+
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * lower-case character, if any; otherwise (if there is no lower-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_tolower(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * upper-case character, if any; otherwise (if there is no upper-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_toupper(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint `c`, return the codepoint of the corresponding
+ * title-case character, if any; otherwise (if there is no title-case
+ * variant, or if `c` is not a valid codepoint) return `c`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_int32_t utf8proc_totitle(utf8proc_int32_t c);
+
+/**
+ * Given a codepoint, return a character width analogous to `wcwidth(codepoint)`,
+ * except that a width of 0 is returned for non-printable codepoints
+ * instead of -1 as in `wcwidth`.
+ * 
+ * @note
+ * If you want to check for particular types of non-printable characters,
+ * (analogous to `isprint` or `iscntrl`), use @ref utf8proc_category. */
+UTF8PROC_DLLEXPORT int utf8proc_charwidth(utf8proc_int32_t codepoint);
+
+/**
+ * Return the Unicode category for the codepoint (one of the
+ * @ref utf8proc_category_t constants.)
+ */
+UTF8PROC_DLLEXPORT utf8proc_category_t utf8proc_category(utf8proc_int32_t codepoint);
+
+/**
+ * Return the two-letter (nul-terminated) Unicode category string for
+ * the codepoint (e.g. `"Lu"` or `"Co"`).
+ */
+UTF8PROC_DLLEXPORT const char *utf8proc_category_string(utf8proc_int32_t codepoint);
+
+/**
+ * Maps the given UTF-8 string pointed to by `str` to a new UTF-8
+ * string, allocated dynamically by `malloc` and returned via `dstptr`.
+ *
+ * If the @ref UTF8PROC_NULLTERM flag in the `options` field is set,
+ * the length is determined by a NULL terminator, otherwise the
+ * parameter `strlen` is evaluated to determine the string length, but
+ * in any case the result will be NULL terminated (though it might
+ * contain NULL characters with the string if `str` contained NULL
+ * characters). Other flags in the `options` field are passed to the
+ * functions defined above, and regarded as described.
+ *
+ * In case of success the length of the new string is returned,
+ * otherwise a negative error code is returned.
+ *
+ * @note The memory of the new UTF-8 string will have been allocated
+ * with `malloc`, and should therefore be deallocated with `free`.
+ */
+UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map(
+  const utf8proc_uint8_t *str, utf8proc_ssize_t strlen, utf8proc_uint8_t **dstptr, utf8proc_option_t options
 );
-/*
- *  Maps the given UTF-8 string being pointed to by 'str' to a new UTF-8
- *  string, which is allocated dynamically, and afterwards pointed to by
- *  the pointer being pointed to by 'dstptr'.
- *  If the NULLTERM flag in the 'options' field is set, the length is
- *  determined by a NULL terminator, otherwise the parameter 'strlen' is
- *  evaluated to determine the string length, but in any case the result
- *  will be NULL terminated (though it might contain NULL characters
- *  before). Other flags in the 'options' field are passed to the functions
- *  defined above, and regarded as described.
- *  In case of success the length of the new string is returned,
- *  otherwise a negative error code is returned.
- *  NOTICE: The memory of the new UTF-8 string will have been allocated with
- *          'malloc', and has theirfore to be freed with 'free'.
- */

-uint8_t *utf8proc_NFD(const uint8_t *str);
-uint8_t *utf8proc_NFC(const uint8_t *str);
-uint8_t *utf8proc_NFKD(const uint8_t *str);
-uint8_t *utf8proc_NFKC(const uint8_t *str);
-/*
- *  Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
- *  normalized version of the null-terminated string 'str'.
+/** @name Unicode normalization
+ *
+ * Returns a pointer to newly allocated memory of a NFD, NFC, NFKD or NFKC
+ * normalized version of the null-terminated string `str`.  These
+ * are shortcuts to calling @ref utf8proc_map with @ref UTF8PROC_NULLTERM
+ * combined with @ref UTF8PROC_STABLE and flags indicating the normalization.
 */
+/** @{ */
+/** NFD normalization (@ref UTF8PROC_DECOMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFD(const utf8proc_uint8_t *str);
+/** NFC normalization (@ref UTF8PROC_COMPOSE). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFC(const utf8proc_uint8_t *str);
+/** NFD normalization (@ref UTF8PROC_DECOMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKD(const utf8proc_uint8_t *str);
+/** NFD normalization (@ref UTF8PROC_COMPOSE and @ref UTF8PROC_COMPAT). */
+UTF8PROC_DLLEXPORT utf8proc_uint8_t *utf8proc_NFKC(const utf8proc_uint8_t *str);
+/** @} */

 #ifdef __cplusplus
 }
--- a/3rdparty/utf8proc/utf8proc_data.c
+++ b/3rdparty/utf8proc/utf8proc_data.c
--- a/3rdparty/utf8proc/utils.cmake
+++ b/3rdparty/utf8proc/utils.cmake
@ -0,0 +1,20 @@
+
+function (disallow_intree_builds)
+  # Adapted from LLVM's toplevel CMakeLists.txt file
+  if( CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND NOT MSVC_IDE )
+    message(FATAL_ERROR "
+      In-source builds are not allowed. CMake would overwrite the
+      makefiles distributed with utf8proc. Please create a directory
+      and run cmake from there. Building in a subdirectory is
+      fine, e.g.:
+      
+        mkdir build
+        cd build
+        cmake ..
+      
+      This process created the file `CMakeCache.txt' and the
+      directory `CMakeFiles'. Please delete them.
+      
+      ")
+  endif()
+endfunction()
--- a/env/strings.cpp
+++ b/env/strings.cpp
@ -17,10 +17,10 @@ string MakeNormalizeAndLowerUtf8(string const & s)

  vector<int32_t> buffer(count);
  int sz = utf8proc_decompose(reinterpret_cast<uint8_t const *>(s.c_str()), count, buffer.data(), count,
-                              UTF8PROC_CASEFOLD | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPMARK);
+                              utf8proc_option_t(UTF8PROC_CASEFOLD | UTF8PROC_DECOMPOSE | UTF8PROC_STRIPMARK));
  CHECK(sz >= 0 && sz <= count, ());

-  sz = utf8proc_reencode(buffer.data(), sz, 0);
+  sz = utf8proc_reencode(buffer.data(), sz, utf8proc_option_t(0));
  return string(reinterpret_cast<char *>(buffer.data()), sz);
 }