From 36ef1408f454de4fde543bf1b7577e0a59a326dd Mon Sep 17 00:00:00 2001
From: Maxim Pimenov <m@maps.me>
Date: Wed, 22 Jan 2020 17:40:09 +0300
Subject: [PATCH] [search][search_quality] More liberal matching of linear
 features.

---
 indexer/search_string_utils.hpp               |   8 +
 .../assessment_tool/main_model.cpp            |   2 +-
 .../assessment_tool/search_request_runner.cpp |   2 +-
 .../features_collector_tool.cpp               |   2 +-
 search/search_quality/matcher.cpp             | 198 ++++++++++++++----
 search/search_quality/matcher.hpp             |  16 +-
 6 files changed, 178 insertions(+), 50 deletions(-)

diff --git a/indexer/search_string_utils.hpp b/indexer/search_string_utils.hpp
index 7ac9e27d53..aea59822b3 100644
--- a/indexer/search_string_utils.hpp
+++ b/indexer/search_string_utils.hpp
@@ -10,6 +10,7 @@
 #include <functional>
 #include <string>
 #include <utility>
+#include <vector>
 
 namespace search
 {
@@ -54,6 +55,13 @@ void NormalizeAndTokenizeAsUtf8(std::string const & s, Tokens & tokens)
   SplitUniString(NormalizeAndSimplifyString(s), fn, search::Delimiters());
 }
 
+inline std::vector<std::string> NormalizeAndTokenizeAsUtf8(std::string const & s)
+{
+  std::vector<std::string> result;
+  NormalizeAndTokenizeAsUtf8(s, result);
+  return result;
+}
+
 template <typename Fn>
 void ForEachNormalizedToken(std::string const & s, Fn && fn)
 {
diff --git a/search/search_quality/assessment_tool/main_model.cpp b/search/search_quality/assessment_tool/main_model.cpp
index 64d47644b1..da92cfe93c 100644
--- a/search/search_quality/assessment_tool/main_model.cpp
+++ b/search/search_quality/assessment_tool/main_model.cpp
@@ -399,7 +399,7 @@ void MainModel::ForAnyMatchingEntry(Context & context, FeatureID const & id, Fn
   for (size_t i = 0; i < nonFoundResults.size(); ++i)
   {
     auto const & result = context.m_nonFoundResults[i];
-    if (matcher.Matches(result, *ft))
+    if (matcher.Matches(context.m_sample.m_query, result, *ft))
       return fn(context.m_nonFoundResultsEdits, i);
   }
 }
diff --git a/search/search_quality/assessment_tool/search_request_runner.cpp b/search/search_quality/assessment_tool/search_request_runner.cpp
index 78d0f7c645..ff12a7390f 100644
--- a/search/search_quality/assessment_tool/search_request_runner.cpp
+++ b/search/search_quality/assessment_tool/search_request_runner.cpp
@@ -103,7 +103,7 @@ void SearchRequestRunner::RunRequest(size_t index, bool background, size_t times
       search::Matcher matcher(loader);
 
       vector<search::Result> const actual(results.begin(), results.end());
-      matcher.Match(sample.m_results, actual, goldenMatching, actualMatching);
+      matcher.Match(sample, actual, goldenMatching, actualMatching);
       relevances.resize(actual.size());
       for (size_t i = 0; i < goldenMatching.size(); ++i)
       {
diff --git a/search/search_quality/features_collector_tool/features_collector_tool.cpp b/search/search_quality/features_collector_tool/features_collector_tool.cpp
index 34d6857b6d..44eb525531 100644
--- a/search/search_quality/features_collector_tool/features_collector_tool.cpp
+++ b/search/search_quality/features_collector_tool/features_collector_tool.cpp
@@ -170,7 +170,7 @@ int main(int argc, char * argv[])
 
     vector<size_t> goldenMatching;
     vector<size_t> actualMatching;
-    matcher.Match(sample.m_results, results, goldenMatching, actualMatching);
+    matcher.Match(sample, results, goldenMatching, actualMatching);
 
     for (size_t j = 0; j < results.size(); ++j)
     {
diff --git a/search/search_quality/matcher.cpp b/search/search_quality/matcher.cpp
index 91f449dcd9..60ba76dda6 100644
--- a/search/search_quality/matcher.cpp
+++ b/search/search_quality/matcher.cpp
@@ -1,28 +1,121 @@
 #include "search/search_quality/matcher.hpp"
 
 #include "search/feature_loader.hpp"
+#include "search/house_numbers_matcher.hpp"
 
 #include "indexer/feature.hpp"
 #include "indexer/feature_algo.hpp"
 #include "indexer/search_string_utils.hpp"
 
-#include "base/string_utils.hpp"
-
 #include "geometry/mercator.hpp"
+#include "geometry/parametrized_segment.hpp"
+#include "geometry/point2d.hpp"
+#include "geometry/polyline2d.hpp"
 
+#include "base/assert.hpp"
 #include "base/control_flow.hpp"
 #include "base/stl_helpers.hpp"
+#include "base/string_utils.hpp"
+
+#include <algorithm>
+
+namespace
+{
+double DistanceToFeature(m2::PointD const & pt, FeatureType & ft)
+{
+  if (ft.GetGeomType() != feature::GeomType::Line)
+    return mercator::DistanceOnEarth(pt, feature::GetCenter(ft));
+
+  ft.ParseGeometry(FeatureType::BEST_GEOMETRY);
+  std::vector<m2::PointD> points(ft.GetPointsCount());
+  for (size_t i = 0; i < points.size(); ++i)
+    points[i] = ft.GetPoint(i);
+
+  auto const & [dummy, segId] = m2::CalcMinSquaredDistance(points.begin(), points.end(), pt);
+  CHECK_LESS(segId + 1, points.size(), ());
+  m2::ParametrizedSegment<m2::PointD> segment(points[segId], points[segId + 1]);
+
+  return mercator::DistanceOnEarth(pt, segment.ClosestPointTo(pt));
+}
+
+template <typename Iter>
+bool StartsWithHouseNumber(Iter beg, Iter end)
+{
+  using namespace search::house_numbers;
+
+  std::string s;
+  for (auto it = beg; it != end; ++it)
+  {
+    s.append(*it);
+    if (LooksLikeHouseNumber(s, false /* isPrefix */))
+      return true;
+  }
+  return false;
+}
+
+// todo(@m) This function looks very slow.
+template <typename Iter>
+bool EndsWithHouseNumber(Iter beg, Iter end)
+{
+  using namespace search::house_numbers;
+
+  if (beg == end)
+    return false;
+
+  std::string s;
+  for (auto it = --end;; --it)
+  {
+    s = *it + s;
+    if (LooksLikeHouseNumber(s, false /* isPrefix */))
+      return true;
+    if (it == beg)
+      break;
+  }
+  return false;
+}
+
+bool StreetMatches(std::string const & name, std::vector<std::string> const & queryTokens)
+{
+  auto const nameTokens = search::NormalizeAndTokenizeAsUtf8(name);
+
+  if (nameTokens.empty())
+    return false;
+
+  for (size_t i = 0; i + nameTokens.size() <= queryTokens.size(); ++i)
+  {
+    bool found = true;
+    for (size_t j = 0; j < nameTokens.size(); ++j)
+    {
+      if (queryTokens[i + j] != nameTokens[j])
+      {
+        found = false;
+        break;
+      }
+    }
+
+    if (!found)
+      continue;
+
+    if (!EndsWithHouseNumber(queryTokens.begin(), queryTokens.begin() + i) &&
+        !StartsWithHouseNumber(queryTokens.begin() + i + nameTokens.size(), queryTokens.end()))
+    {
+      return true;
+    }
+  }
+
+  return false;
+}
+}  // namespace
 
 namespace search
 {
-// static
-size_t constexpr Matcher::kInvalidId;
-
 Matcher::Matcher(FeatureLoader & loader) : m_loader(loader) {}
 
-void Matcher::Match(std::vector<Sample::Result> const & golden, std::vector<Result> const & actual,
+void Matcher::Match(Sample const & goldenSample, std::vector<Result> const & actual,
                     std::vector<size_t> & goldenMatching, std::vector<size_t> & actualMatching)
 {
+  auto const & golden = goldenSample.m_results;
+
   auto const n = golden.size();
   auto const m = actual.size();
 
@@ -42,7 +135,7 @@ void Matcher::Match(std::vector<Sample::Result> const & golden, std::vector<Resu
         continue;
 
       auto const & a = actual[j];
-      if (Matches(g, a))
+      if (Matches(goldenSample.m_query, g, a))
       {
         goldenMatching[i] = j;
         actualMatching[j] = i;
@@ -52,39 +145,8 @@ void Matcher::Match(std::vector<Sample::Result> const & golden, std::vector<Resu
   }
 }
 
-bool Matcher::Matches(Sample::Result const & golden, FeatureType & ft)
-{
-  static double constexpr kToleranceMeters = 50;
-
-  auto const houseNumber = ft.GetHouseNumber();
-  auto const center = feature::GetCenter(ft);
-
-  bool nameMatches = false;
-  if (golden.m_name.empty())
-  {
-    nameMatches = true;
-  }
-  else
-  {
-    ft.ForEachName([&golden, &nameMatches](int8_t /* lang */, std::string const & name) {
-      if (NormalizeAndSimplifyString(ToUtf8(golden.m_name)) == NormalizeAndSimplifyString(name))
-      {
-        nameMatches = true;
-        return base::ControlFlow::Break;
-      }
-      return base::ControlFlow::Continue;
-    });
-  }
-
-  bool houseNumberMatches = true;
-  if (!golden.m_houseNumber.empty() && !houseNumber.empty())
-    houseNumberMatches = golden.m_houseNumber == houseNumber;
-
-  return nameMatches && houseNumberMatches &&
-         mercator::DistanceOnEarth(golden.m_pos, center) < kToleranceMeters;
-}
-
-bool Matcher::Matches(Sample::Result const & golden, search::Result const & actual)
+bool Matcher::Matches(strings::UniString const & query, Sample::Result const & golden,
+                      search::Result const & actual)
 {
   if (actual.GetResultType() != Result::Type::Feature)
     return false;
@@ -93,6 +155,60 @@ bool Matcher::Matches(Sample::Result const & golden, search::Result const & actu
   if (!ft)
     return false;
 
-  return Matches(golden, *ft);
+  return Matches(query, golden, *ft);
+}
+
+bool Matcher::Matches(strings::UniString const & query, Sample::Result const & golden,
+                      FeatureType & ft)
+{
+  static double constexpr kToleranceMeters = 50;
+
+  auto const houseNumber = ft.GetHouseNumber();
+
+  auto const queryTokens = NormalizeAndTokenizeAsUtf8(ToUtf8(query));
+
+  bool nameMatches = false;
+
+  // The golden result may have an empty name. What is more likely, though, is that the
+  // sample was not obtained as a result of a previous run of our search_quality tools.
+  // Probably it originates from a third-party source.
+  if (golden.m_name.empty())
+  {
+    if (ft.GetGeomType() == feature::GeomType::Line)
+    {
+      nameMatches = StreetMatches(ft.GetParams().ref, queryTokens);
+    }
+    else
+    {
+      // Don't try to guess: it's enough to match by distance.
+      // |ft| with GeomType::Point here is usually a POI and |ft| with GeomType::Area is a building.
+      nameMatches = true;
+    }
+  }
+
+  ft.ForEachName(
+      [&queryTokens, &ft, &golden, &nameMatches](int8_t /* lang */, std::string const & name) {
+        if (NormalizeAndSimplifyString(ToUtf8(golden.m_name)) == NormalizeAndSimplifyString(name))
+        {
+          nameMatches = true;
+          return base::ControlFlow::Break;
+        }
+
+        if (golden.m_name.empty() && ft.GetGeomType() == feature::GeomType::Line &&
+            StreetMatches(name, queryTokens))
+        {
+          nameMatches = true;
+          return base::ControlFlow::Break;
+        }
+
+        return base::ControlFlow::Continue;
+      });
+
+  bool houseNumberMatches = true;
+  if (!golden.m_houseNumber.empty() && !houseNumber.empty())
+    houseNumberMatches = golden.m_houseNumber == houseNumber;
+
+  return nameMatches && houseNumberMatches &&
+         DistanceToFeature(golden.m_pos, ft) < kToleranceMeters;
 }
 }  // namespace search
diff --git a/search/search_quality/matcher.hpp b/search/search_quality/matcher.hpp
index 741e042692..db4a4528cf 100644
--- a/search/search_quality/matcher.hpp
+++ b/search/search_quality/matcher.hpp
@@ -1,8 +1,11 @@
 #pragma once
 
-#include "search/result.hpp"
 #include "search/search_quality/sample.hpp"
 
+#include "search/result.hpp"
+
+#include "base/string_utils.hpp"
+
 #include <cstddef>
 #include <limits>
 #include <vector>
@@ -16,21 +19,22 @@ class FeatureLoader;
 class Matcher
 {
 public:
-  static size_t constexpr kInvalidId = std::numeric_limits<size_t>::max();
+  inline static size_t constexpr kInvalidId = std::numeric_limits<size_t>::max();
 
   explicit Matcher(FeatureLoader & loader);
 
-  // Matches the |golden| results loaded from a Sample with |actual| results
+  // Matches the results loaded from |goldenSample| with |actual| results
   // found by the search engine using the params from the Sample.
   // goldenMatching[i] is the index of the result in |actual| that matches
   // the sample result number i.
   // actualMatching[j] is the index of the sample in |golden| that matches
   // the golden result number j.
-  void Match(std::vector<Sample::Result> const & golden, std::vector<Result> const & actual,
+  void Match(Sample const & goldenSample, std::vector<Result> const & actual,
              std::vector<size_t> & goldenMatching, std::vector<size_t> & actualMatching);
 
-  bool Matches(Sample::Result const & golden, FeatureType & ft);
-  bool Matches(Sample::Result const & golden, Result const & actual);
+  bool Matches(strings::UniString const & query, Sample::Result const & golden,
+               Result const & actual);
+  bool Matches(strings::UniString const & query, Sample::Result const & golden, FeatureType & ft);
 
 private:
   FeatureLoader & m_loader;