Cuisine normalisation #https://trello.com/c/TKooF4Q0/25-cuisine

base/string_utils.cpp

@@ -6,8 +6,7 @@
 #include "std/cmath.hpp"
 #include "std/iomanip.hpp"
 
-#include <boost/algorithm/string.hpp> // boost::trim
-
+#include <boost/algorithm/string/trim.hpp>
 
 namespace strings
 {

generator/generator_tests/generator_tests.pro

@@ -22,10 +22,11 @@ SOURCES += \
     feature_builder_test.cpp \
     feature_merger_test.cpp \
     metadata_parser_test.cpp \
+    osm2meta_test.cpp \
     osm_id_test.cpp \
     osm_o5m_source_test.cpp \
     osm_type_test.cpp \
+    source_data.cpp \
+    source_to_element_test.cpp \
     tesselator_test.cpp \
     triangles_tree_coding_test.cpp \
-    source_to_element_test.cpp \
-    source_data.cpp \

generator/generator_tests/osm2meta_test.cpp

@@ -0,0 +1,14 @@
+#include <testing/testing.hpp>
+
+#include <generator/osm2meta.hpp>
+
+UNIT_TEST(ValidateAndFormat_cuisine_test)
+{
+  FeatureParams params;
+  MetadataTagProcessorImpl tagProc(params);
+  TEST_EQUAL(tagProc.ValidateAndFormat_cuisine(" ,ABC, CDE;   FgH,   "), "abc;cde;fgh", ());
+  TEST_EQUAL(tagProc.ValidateAndFormat_cuisine(";;;ABc,   cef,,,"), "abc;cef", ());
+  TEST_EQUAL(tagProc.ValidateAndFormat_cuisine("abc bca"), "abc_bca", ());
+  TEST_EQUAL(tagProc.ValidateAndFormat_cuisine("abc      def  gh"), "abc_def_gh", ());
+  TEST_EQUAL(tagProc.ValidateAndFormat_cuisine(""), "", ());
+}

generator/osm2meta.cpp

@@ -5,7 +5,137 @@
 #include "base/logging.hpp"
 #include "base/string_utils.hpp"
 
-string MetadataTagProcessor::ValidateAndFormat_wikipedia(string v) const
+#include "std/regex.hpp"
+#include "std/cctype.hpp"
+
+string MetadataTagProcessorImpl::ValidateAndFormat_maxspeed(string const & v) const
+{
+  if (!ftypes::IsSpeedCamChecker::Instance()(m_params.m_Types))
+    return string();
+
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_stars(string const & v) const
+{
+  if (v.empty())
+    return string();
+
+  // We are accepting stars from 1 to 7.
+  if (v[0] <= '0' || v[0] > '7')
+    return string();
+
+  // Ignore numbers large then 9.
+  if (v.size() > 1 && ::isdigit(v[1]))
+    return string();
+
+  return string(1, v[0]);
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_operator(string const & v) const
+{
+  auto const & isATM = ftypes::IsATMChecker::Instance();
+  auto const & isFuelStation = ftypes::IsFuelStationChecker::Instance();
+
+  if (!(isATM(m_params.m_Types) || isFuelStation(m_params.m_Types)))
+    return string();
+
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_url(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_phone(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_opening_hours(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_ele(string const & v) const
+{
+  auto const & isPeak = ftypes::IsPeakChecker::Instance();
+  if (!isPeak(m_params.m_Types))
+    return string();
+  double val = 0;
+  if(!strings::to_double(v, val) || val == 0)
+    return string();
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_turn_lanes(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_turn_lanes_forward(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_turn_lanes_backward(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_email(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_postcode(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_flats(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_height(string const & v) const
+{
+  double val = 0;
+  string corrected(v, 0, v.find(" "));
+  if(!strings::to_double(corrected, val) || val == 0)
+    return string();
+  ostringstream ss;
+  ss << fixed << setprecision(1) << val;
+  return ss.str();
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_building_levels(string const & v) const
+{
+  double constexpr kMetersPerLevel = 3;
+  double val = 0;
+  if(!strings::to_double(v, val) || val == 0)
+    return string();
+  ostringstream ss;
+  ss << fixed << setprecision(1) << (val * kMetersPerLevel);
+  return ss.str();
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_denomination(string const & v) const
+{
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_cuisine(string v) const
+{
+  strings::MakeLowerCaseInplace(v);
+  v = regex_replace(v, regex("[;,]\\s*"), ";");
+  v = regex_replace(v, regex("\\s+"), "_");
+  strings::Trim(v, ";_");
+  return v;
+}
+
+string MetadataTagProcessorImpl::ValidateAndFormat_wikipedia(string v) const
 {
   strings::Trim(v);
   // Normalize by converting full URL to "lang:title" if necessary

generator/osm2meta.hpp

@@ -4,21 +4,46 @@
 #include "indexer/classificator.hpp"
 #include "indexer/ftypes_matcher.hpp"
 
-#include "base/string_utils.hpp"
-
 #include "std/string.hpp"
 
-
-class MetadataTagProcessor
+struct MetadataTagProcessorImpl
 {
-  FeatureParams & m_params;
-
-public:
-  MetadataTagProcessor(FeatureParams &params)
+  MetadataTagProcessorImpl(FeatureParams &params)
   : m_params(params)
   {
   }
 
+  string ValidateAndFormat_maxspeed(string const & v) const;
+  string ValidateAndFormat_stars(string const & v) const;
+  string ValidateAndFormat_cuisine(string v) const;
+  string ValidateAndFormat_operator(string const & v) const;
+  string ValidateAndFormat_url(string const & v) const;
+  string ValidateAndFormat_phone(string const & v) const;
+  string ValidateAndFormat_opening_hours(string const & v) const;
+  string ValidateAndFormat_ele(string const & v) const;
+  string ValidateAndFormat_turn_lanes(string const & v) const;
+  string ValidateAndFormat_turn_lanes_forward(string const & v) const;
+  string ValidateAndFormat_turn_lanes_backward(string const & v) const;
+  string ValidateAndFormat_email(string const & v) const;
+  string ValidateAndFormat_postcode(string const & v) const;
+  string ValidateAndFormat_flats(string const & v) const;
+  string ValidateAndFormat_height(string const & v) const;
+  string ValidateAndFormat_building_levels(string const & v) const;
+  string ValidateAndFormat_denomination(string const & v) const;
+  string ValidateAndFormat_wikipedia(string v) const;
+
+protected:
+  FeatureParams & m_params;
+};
+
+class MetadataTagProcessor : private MetadataTagProcessorImpl
+{
+public:
+  using MetadataTagProcessorImpl::MetadataTagProcessorImpl;
+
+  /// Since it is used as a functor wich stops iteration in ftype::ForEachTag
+  /// and the is no need for interrupting it always returns false.
+  /// TODO(mgsergio): Move to cpp after merge with https://github.com/mapsme/omim/pull/1314
   bool operator() (string const & k, string const & v)
   {
     if (v.empty())
@@ -163,118 +188,7 @@ public:
       if (!value.empty())
         md.Set(Metadata::FMD_DENOMINATION, value);
     }
+
     return false;
   }
-
-protected:
-  /// Validation and formatting functions
-
-  string ValidateAndFormat_maxspeed(string const & v) const
-  {
-    if (!ftypes::IsSpeedCamChecker::Instance()(m_params.m_Types))
-      return string();
-
-    return v;
-  }
-
-  string ValidateAndFormat_stars(string const & v) const
-  {
-    if (v.empty())
-      return string();
-
-    // we accepting stars from 1 to 7
-    if (v[0] <= '0' || v[0] > '7')
-      return string();
-
-    // ignore numbers large then 9
-    if (v.size() > 1 && (v[1] >= '0' && v[1] <= '9'))
-      return string();
-
-    return string(1, v[0]);
-  }
-
-  string ValidateAndFormat_cuisine(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_operator(string const & v) const
-  {
-    static ftypes::IsATMChecker const IsATM;
-    static ftypes::IsFuelStationChecker const IsFuelStation;
-
-    if (!(IsATM(m_params.m_Types) || IsFuelStation(m_params.m_Types)))
-      return string();
-
-    return v;
-  }
-  string ValidateAndFormat_url(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_phone(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_opening_hours(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_ele(string const & v) const
-  {
-    static ftypes::IsPeakChecker const IsPeak;
-    if (!IsPeak(m_params.m_Types))
-      return string();
-    double val = 0;
-    if(!strings::to_double(v, val) || val == 0)
-      return string();
-    return v;
-  }
-  string ValidateAndFormat_turn_lanes(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_turn_lanes_forward(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_turn_lanes_backward(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_email(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_postcode(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_flats(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_height(string const & v) const
-  {
-    double val = 0;
-    string corrected(v, 0, v.find(" "));
-    if(!strings::to_double(corrected, val) || val == 0)
-      return string();
-    stringstream ss;
-    ss << fixed << setw(2) << setprecision(1) << val;
-    return ss.str();
-  }
-  string ValidateAndFormat_building_levels(string const & v) const
-  {
-    double val = 0;
-    if(!strings::to_double(v, val) || val == 0)
-      return string();
-    stringstream ss;
-    ss << fixed << setw(2) << setprecision(1) << (val * 3 /*levels multiplied by 3 meters per level*/);
-    return ss.str();
-  }
-  string ValidateAndFormat_denomination(string const & v) const
-  {
-    return v;
-  }
-  string ValidateAndFormat_wikipedia(string v) const;
 };

std/regex.hpp

@@ -10,6 +10,7 @@ using std::regex;
 using std::regex_match;
 using std::regex_search;
 using std::sregex_token_iterator;
+using std::regex_replace;
 
 #if defined(DEBUG_NEW)
 #define new DEBUG_NEW