From 32df24937c62e20a877408bde0a76d4e0ab84df5 Mon Sep 17 00:00:00 2001
From: Alex Zolotarev <deathbaba@gmail.com>
Date: Mon, 23 May 2011 20:09:39 +0200
Subject: [PATCH] Added GetUniString() to strings::TokenizeIterator

---
 base/base_tests/string_utils_test.cpp | 16 ++++++++++++++++
 base/string_utils.hpp                 | 16 +++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/base/base_tests/string_utils_test.cpp b/base/base_tests/string_utils_test.cpp
index 4e402db222..b8e9e0d3c2 100644
--- a/base/base_tests/string_utils_test.cpp
+++ b/base/base_tests/string_utils_test.cpp
@@ -121,3 +121,19 @@ UNIT_TEST(LastUniChar)
   TEST_EQUAL(strings::LastUniChar(" \xD0\x90"), 0x0410, ());
 
 }
+
+UNIT_TEST(GetUniString)
+{
+  strings::SimpleTokenizer iter("Hello, \xD0\x9C\xD0\xB8\xD0\xBD\xD1\x81\xD0\xBA!", ", !");
+  {
+    strings::UniChar const s[] = { 'H', 'e', 'l', 'l', 'o' };
+    strings::UniString us(&s[0], &s[0] + ARRAY_SIZE(s));
+    TEST_EQUAL(iter.GetUniString(), us, ());
+  }
+  ++iter;
+  {
+    strings::UniChar const s[] = { 0x041C, 0x0438, 0x043D, 0x0441, 0x043A };
+    strings::UniString us(&s[0], &s[0] + ARRAY_SIZE(s));
+    TEST_EQUAL(iter.GetUniString(), us, ());
+  }
+}
diff --git a/base/string_utils.hpp b/base/string_utils.hpp
index badc6f1468..52530381d0 100644
--- a/base/string_utils.hpp
+++ b/base/string_utils.hpp
@@ -18,7 +18,8 @@ typedef buffer_vector<UniChar, 32> UniString;
 template <typename DelimFuncT>
 class TokenizeIterator
 {
-  utf8::unchecked::iterator<string::const_iterator> m_beg, m_end, m_finish;
+  typedef utf8::unchecked::iterator<string::const_iterator> Utf8IterT;
+  Utf8IterT m_beg, m_end, m_finish;
   DelimFuncT m_delimFunc;
 
   void move()
@@ -53,6 +54,7 @@ public:
     ASSERT( m_beg != m_finish, ("dereferencing of empty iterator") );
     return string(m_beg.base(), m_end.base());
   }
+
   operator bool() const { return m_beg != m_finish; }
 
   TokenizeIterator & operator++()
@@ -69,6 +71,18 @@ public:
     ++copy;
     return !copy;
   }
+
+  UniString GetUniString() const
+  {
+    UniString result;
+    Utf8IterT iter(m_beg);
+    while (iter != m_end)
+    {
+      result.push_back(*iter);
+      ++iter;
+    }
+    return result;
+  }
 };
 
 class SimpleDelimiter