From f0122d60b5264645e2679220215c52a8a0083ce0 Mon Sep 17 00:00:00 2001
From: vng <viktor.govako@gmail.com>
Date: Fri, 20 Sep 2013 12:53:16 +0300
Subject: [PATCH] Add some tokenize tests.

---
 base/base_tests/string_utils_test.cpp     |  5 +++
 search/search_tests/string_match_test.cpp | 45 +++++++++++++++++++++--
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/base/base_tests/string_utils_test.cpp b/base/base_tests/string_utils_test.cpp
index fd367a66bd..773207c087 100644
--- a/base/base_tests/string_utils_test.cpp
+++ b/base/base_tests/string_utils_test.cpp
@@ -295,6 +295,11 @@ UNIT_TEST(SimpleTokenizer)
              " -\xD9\x87", tokens);
   }
 
+  {
+    char const * s[] = {"1", "2"};
+    tokens.assign(&s[0], &s[0] + ARRAY_SIZE(s));
+    TestIter("/1/2/", "/", tokens);
+  }
 }
 
 UNIT_TEST(LastUniChar)
diff --git a/search/search_tests/string_match_test.cpp b/search/search_tests/string_match_test.cpp
index 95307f87d1..abe6b90a6d 100644
--- a/search/search_tests/string_match_test.cpp
+++ b/search/search_tests/string_match_test.cpp
@@ -1,21 +1,29 @@
 #include "../../testing/testing.hpp"
 #include "../approximate_string_match.hpp"
+
 #include "match_cost_mock.hpp"
+
+#include "../../indexer/search_delimiters.hpp"
+
+#include "../../base/stl_add.hpp"
+
 #include "../../std/memcpy.hpp"
 
+
+using namespace search;
+using namespace strings;
+
 namespace
 {
 
 uint32_t FullMatchCost(char const * a, char const * b, uint32_t maxCost = 1000)
 {
-  return ::search::StringMatchCost(a, strlen(a), b, strlen(b),
-                                   search::MatchCostMock<char>(), maxCost);
+  return StringMatchCost(a, strlen(a), b, strlen(b), MatchCostMock<char>(), maxCost);
 }
 
 uint32_t PrefixMatchCost(char const * a, char const * b)
 {
-  return ::search::StringMatchCost(a, strlen(a), b, strlen(b),
-                                   search::MatchCostMock<char>(), 1000, true);
+  return StringMatchCost(a, strlen(a), b, strlen(b), MatchCostMock<char>(), 1000, true);
 }
 
 }
@@ -67,3 +75,32 @@ UNIT_TEST(StringMatchCost_PrefixMatch)
   TEST_EQUAL(PrefixMatchCost("Helpo", "Hello!"), 1, ());
   TEST_EQUAL(PrefixMatchCost("Happo", "Hello!"), 3, ());
 }
+
+namespace
+{
+
+void TestEqual(vector<UniString> const v, char const * arr[])
+{
+  for (size_t i = 0; i < v.size(); ++i)
+  {
+    TEST_EQUAL(ToUtf8(v[i]), arr[i], ());
+    TEST_EQUAL(v[i], MakeUniString(arr[i]), ());
+  }
+}
+
+}
+
+UNIT_TEST(StringSplit_Smoke)
+{
+  vector<UniString> tokens;
+
+  {
+    string const s = "1/2";
+    UniString const s1 = NormalizeAndSimplifyString(s);
+    TEST_EQUAL(ToUtf8(s1), s, ());
+
+    char const * arr[] = { "1", "2" };
+    SplitUniString(s1, MakeBackInsertFunctor(tokens), Delimiters());
+    TestEqual(tokens, arr);
+  }
+}