Added string tokenizer.

env/strings.hpp

@@ -16,4 +16,25 @@ template <class T> string ToString(T const & t)
 
 string MakeNormalizeAndLowerUtf8(string const & s);
 
+template <class IterT>
+IterT Tokenize(string const & s, char const * delims, IterT out)
+{
+  size_t i = 0;
+  while (i < s.size())
+  {
+    size_t j = s.find_first_of(delims, i);
+    if (j == string::npos)
+      j = s.size();
+    if (j > i)
+    {
+      *out++ = s.substr(i, j-i);
+      i = j+1;
+    }
+    else
+      ++i;
+  }
+
+  return out;
+}
+
 }

env/tests/env_tests.cpp

@@ -79,3 +79,14 @@ TEST(Env, MakeNormalizeAndLowerUtf8)
   for (size_t i = 0; i < ArraySize(arr); ++i)
     EXPECT_EQ(str::MakeNormalizeAndLowerUtf8(arr[i]), res[i]);
 }
+
+TEST(Env, Tokenizer)
+{
+  string in = "\taaa, bbb?,\tccc ";
+  string out[] = { "aaa", "bbb", "ccc" };
+
+  vector<string> v;
+  str::Tokenize(in, " ,\t?", back_inserter(v));
+
+  EXPECT_TRUE(equal(out, out + ArraySize(out), v.begin()));
+}