From 11da6c08fe8c414a3a2f1d3d5df9cd497a8b117b Mon Sep 17 00:00:00 2001 From: Yuri Gorshenin Date: Wed, 11 Oct 2017 14:29:11 +0300 Subject: [PATCH] [search] Reduced number of cache invalidations in LocalityFinder. --- coding/multilang_utf8_string.hpp | 17 ++--- search/city_finder.hpp | 4 +- search/locality_finder.cpp | 74 ++++++------------- search/locality_finder.hpp | 66 +++++++++++++---- search/processor.cpp | 1 - search/ranker.cpp | 8 +- search/ranker.hpp | 3 +- search/search_tests/locality_finder_test.cpp | 8 +- .../search_tests/locality_selector_test.cpp | 47 ++++++++---- 9 files changed, 125 insertions(+), 103 deletions(-) diff --git a/coding/multilang_utf8_string.hpp b/coding/multilang_utf8_string.hpp index e4bd083c8e..c78fe14a5e 100644 --- a/coding/multilang_utf8_string.hpp +++ b/coding/multilang_utf8_string.hpp @@ -75,15 +75,8 @@ public: /// @returns empty string if langCode is invalid. static char const * GetTransliteratorIdByCode(int8_t langCode); - inline bool operator== (StringUtf8Multilang const & rhs) const - { - return (m_s == rhs.m_s); - } - - inline bool operator!= (StringUtf8Multilang const & rhs) const - { - return !(*this == rhs); - } + inline bool operator==(StringUtf8Multilang const & rhs) const { return (m_s == rhs.m_s); } + inline bool operator!=(StringUtf8Multilang const & rhs) const { return !(*this == rhs); } inline void Clear() { m_s.clear(); } inline bool IsEmpty() const { return m_s.empty(); } @@ -124,12 +117,14 @@ public: int8_t FindString(string const & utf8s) const; - template void Write(TSink & sink) const + template + void Write(TSink & sink) const { utils::WriteString(sink, m_s); } - template void Read(TSource & src) + template + void Read(TSource & src) { utils::ReadString(src, m_s); } diff --git a/search/city_finder.hpp b/search/city_finder.hpp index 9f1b0c0500..4b94c59944 100644 --- a/search/city_finder.hpp +++ b/search/city_finder.hpp @@ -22,8 +22,8 @@ public: string GetCityName(m2::PointD const & p, int8_t lang) { string city; - m_finder.SetLanguage(lang); - m_finder.GetLocality(p, city); + m_finder.GetLocality( + p, [&](LocalityItem const & item) { item.GetSpecifiedOrDefaultName(lang, city); }); return city; } diff --git a/search/locality_finder.cpp b/search/locality_finder.cpp index 210885efa2..c84416c66e 100644 --- a/search/locality_finder.cpp +++ b/search/locality_finder.cpp @@ -12,7 +12,9 @@ #include "base/assert.hpp" #include "base/stl_helpers.hpp" -#include "std/algorithm.hpp" +#include + +using namespace std; namespace search { @@ -55,12 +57,10 @@ private: class LocalitiesLoader { public: - LocalitiesLoader(MwmContext const & ctx, Filter const & filter, int8_t lang, - LocalityFinder::Holder & holder, + LocalitiesLoader(MwmContext const & ctx, Filter const & filter, LocalityFinder::Holder & holder, map> & loadedIds) : m_ctx(ctx) , m_filter(filter) - , m_lang(lang) , m_holder(holder) , m_loadedIds(loadedIds[m_ctx.GetId()]) { @@ -96,21 +96,20 @@ public: if (population == 0) return; - // read item - string name; - if (!ft.GetName(m_lang, name) && !ft.GetName(0, name)) - return; - + StringUtf8Multilang names; + ft.ForEachName([&](int8_t lang, string const & name) { + names.AddString(lang, name); + return true; + }); auto const center = ft.GetCenter(); - m_holder.Add(LocalityItem(name, center, population)); + m_holder.Add(LocalityItem(names, center, population)); m_loadedIds.insert(id); } private: MwmContext const & m_ctx; Filter const & m_filter; - int8_t const m_lang; LocalityFinder::Holder & m_holder; unordered_set & m_loadedIds; @@ -118,26 +117,23 @@ private: } // namespace // LocalityItem ------------------------------------------------------------------------------------ -LocalityItem::LocalityItem(string const & name, m2::PointD const & center, uint64_t population) - : m_name(name), m_center(center), m_population(population) +LocalityItem::LocalityItem(StringUtf8Multilang const & names, m2::PointD const & center, + uint64_t population) + : m_names(names), m_center(center), m_population(population) { } string DebugPrint(LocalityItem const & item) { - stringstream ss; - ss << "Name = " << item.m_name << ", Center = " << DebugPrint(item.m_center) - << ", Population = " << item.m_population; - return ss.str(); + stringstream os; + os << "Names = " << DebugPrint(item.m_names) << ", "; + os << "Center = " << DebugPrint(item.m_center) << ", "; + os << "Population = " << item.m_population; + return os.str(); } // LocalitySelector -------------------------------------------------------------------------------- -LocalitySelector::LocalitySelector(string & name, m2::PointD const & p) - : m_name(name) - , m_p(p) - , m_bestScore(numeric_limits::max()) -{ -} +LocalitySelector::LocalitySelector(m2::PointD const & p) : m_p(p) {} void LocalitySelector::operator()(LocalityItem const & item) { @@ -150,7 +146,7 @@ void LocalitySelector::operator()(LocalityItem const & item) if (score < m_bestScore) { m_bestScore = score; - m_name = item.m_name; + m_bestLocality = &item; } } @@ -200,35 +196,12 @@ void LocalityFinder::Holder::Clear() LocalityFinder::LocalityFinder(Index const & index, VillagesCache & villagesCache) : m_index(index) , m_villagesCache(villagesCache) - , m_lang(0) , m_cities(kMaxCityRadiusMeters) , m_villages(kMaxVillageRadiusMeters) , m_mapsLoaded(false) { } -void LocalityFinder::SetLanguage(int8_t lang) -{ - if (m_lang == lang) - return; - - ClearCache(); - m_lang = lang; -} - -void LocalityFinder::GetLocality(m2::PointD const & p, string & name) -{ - m2::RectD const crect = m_cities.GetRect(p); - m2::RectD const vrect = m_villages.GetRect(p); - - LoadVicinity(p, !m_cities.IsCovered(crect) /* loadCities */, - !m_villages.IsCovered(vrect) /* loadVillages */); - - LocalitySelector selector(name, p); - m_cities.ForEachInVicinity(crect, selector); - m_villages.ForEachInVicinity(vrect, selector); -} - void LocalityFinder::ClearCache() { m_ranks.reset(); @@ -259,8 +232,7 @@ void LocalityFinder::LoadVicinity(m2::PointD const & p, bool loadCities, bool lo m_ranks = make_unique(); MwmContext ctx(move(handle)); - ctx.ForEachIndex(crect, - LocalitiesLoader(ctx, CityFilter(*m_ranks), m_lang, m_cities, m_loadedIds)); + ctx.ForEachIndex(crect, LocalitiesLoader(ctx, CityFilter(*m_ranks), m_cities, m_loadedIds)); } m_cities.SetCovered(p); @@ -275,8 +247,8 @@ void LocalityFinder::LoadVicinity(m2::PointD const & p, bool loadCities, bool lo return; MwmContext ctx(move(handle)); - ctx.ForEachIndex(vrect, LocalitiesLoader(ctx, VillageFilter(ctx, m_villagesCache), m_lang, - m_villages, m_loadedIds)); + ctx.ForEachIndex(vrect, LocalitiesLoader(ctx, VillageFilter(ctx, m_villagesCache), m_villages, + m_loadedIds)); }); m_villages.SetCovered(p); diff --git a/search/locality_finder.hpp b/search/locality_finder.hpp index fffdd5d85d..7a6de185d5 100644 --- a/search/locality_finder.hpp +++ b/search/locality_finder.hpp @@ -3,14 +3,20 @@ #include "indexer/mwm_set.hpp" #include "indexer/rank_table.hpp" +#include "coding/multilang_utf8_string.hpp" + #include "geometry/point2d.hpp" #include "geometry/rect2d.hpp" #include "geometry/tree4d.hpp" #include "base/macros.hpp" -#include "std/unique_ptr.hpp" -#include "std/unordered_set.hpp" +#include +#include +#include +#include +#include +#include class Index; @@ -20,9 +26,16 @@ class VillagesCache; struct LocalityItem { - LocalityItem(string const & name, m2::PointD const & center, uint64_t population); + LocalityItem(StringUtf8Multilang const & names, m2::PointD const & center, uint64_t population); - string m_name; + bool GetName(int8_t lang, string & name) const { return m_names.GetString(lang, name); } + + bool GetSpecifiedOrDefaultName(int8_t lang, string & name) const + { + return GetName(lang, name) || GetName(StringUtf8Multilang::kDefaultCode, name); + } + + StringUtf8Multilang m_names; m2::PointD m_center; uint64_t m_population; }; @@ -32,14 +45,23 @@ string DebugPrint(LocalityItem const & item); class LocalitySelector { public: - LocalitySelector(string & name, m2::PointD const & p); + LocalitySelector(m2::PointD const & p); void operator()(LocalityItem const & item); + template + bool WithBestLocality(Fn && fn) const + { + if (!m_bestLocality) + return false; + fn(*m_bestLocality); + return true; + } + private: - string & m_name; - m2::PointD m_p; - double m_bestScore; + m2::PointD const m_p; + double m_bestScore = std::numeric_limits::max(); + LocalityItem const * m_bestLocality = nullptr; }; class LocalityFinder @@ -47,7 +69,7 @@ class LocalityFinder public: class Holder { - public: + public: Holder(double radiusMeters); bool IsCovered(m2::RectD const & rect) const; @@ -61,8 +83,7 @@ public: void Clear(); - private: - + private: double const m_radiusMeters; m4::Tree m_coverage; m4::Tree m_localities; @@ -72,8 +93,22 @@ public: LocalityFinder(Index const & index, VillagesCache & villagesCache); - void SetLanguage(int8_t lang); - void GetLocality(m2::PointD const & p, string & name); + template + bool GetLocality(m2::PointD const & p, Fn && fn) + { + m2::RectD const crect = m_cities.GetRect(p); + m2::RectD const vrect = m_villages.GetRect(p); + + LoadVicinity(p, !m_cities.IsCovered(crect) /* loadCities */, + !m_villages.IsCovered(vrect) /* loadVillages */); + + LocalitySelector selector(p); + m_cities.ForEachInVicinity(crect, selector); + m_villages.ForEachInVicinity(vrect, selector); + + return selector.WithBestLocality(std::forward(fn)); + } + void ClearCache(); private: @@ -82,7 +117,6 @@ private: Index const & m_index; VillagesCache & m_villagesCache; - int8_t m_lang; Holder m_cities; Holder m_villages; @@ -91,8 +125,8 @@ private: MwmSet::MwmId m_worldId; bool m_mapsLoaded; - unique_ptr m_ranks; + std::unique_ptr m_ranks; - map> m_loadedIds; + std::map> m_loadedIds; }; } // namespace search diff --git a/search/processor.cpp b/search/processor.cpp index 35b4da3089..eb1f25acef 100644 --- a/search/processor.cpp +++ b/search/processor.cpp @@ -228,7 +228,6 @@ void Processor::SetPreferredLocale(string const & locale) // Default initialization. // If you want to reset input language, call SetInputLocale before search. SetInputLocale(locale); - m_ranker.SetLocalityFinderLanguage(code); } void Processor::SetInputLocale(string const & locale) diff --git a/search/ranker.cpp b/search/ranker.cpp index 405fd99459..a7e83cc001 100644 --- a/search/ranker.cpp +++ b/search/ranker.cpp @@ -380,9 +380,11 @@ Result Ranker::MakeResult(PreResult2 const & r) const MakeResultHighlight(res); if (ftypes::IsLocalityChecker::Instance().GetType(r.GetTypes()) == ftypes::NONE) { - string city; - m_localities.GetLocality(res.GetFeatureCenter(), city); - res.AppendCity(city); + m_localities.GetLocality(res.GetFeatureCenter(), [&](LocalityItem const & item) { + string city; + if (item.GetSpecifiedOrDefaultName(m_localityLang, city)) + res.AppendCity(city); + }); } res.SetRankingInfo(r.GetRankingInfo()); diff --git a/search/ranker.hpp b/search/ranker.hpp index 867f247d17..d045ba61a8 100644 --- a/search/ranker.hpp +++ b/search/ranker.hpp @@ -96,7 +96,7 @@ public: void ClearCaches(); - inline void SetLocalityFinderLanguage(int8_t code) { m_localities.SetLanguage(code); } + inline void SetLocalityLanguage(int8_t code) { m_localityLang = code; } inline void SetLanguage(pair const & ind, int8_t lang) { @@ -131,6 +131,7 @@ private: KeywordLangMatcher m_keywordsScorer; mutable LocalityFinder m_localities; + int8_t m_localityLang = StringUtf8Multilang::kDefaultCode; Index const & m_index; storage::CountryInfoGetter const & m_infoGetter; diff --git a/search/search_tests/locality_finder_test.cpp b/search/search_tests/locality_finder_test.cpp index c79f5f359b..c36deb012b 100644 --- a/search/search_tests/locality_finder_test.cpp +++ b/search/search_tests/locality_finder_test.cpp @@ -52,8 +52,6 @@ public: { LOG(LERROR, ("Read World.mwm error:", ex.Msg())); } - - m_finder.SetLanguage(StringUtf8Multilang::kEnglishCode); } ~LocalityFinderTest() @@ -66,7 +64,10 @@ public: for (size_t i = 0; i < input.size(); ++i) { string result; - m_finder.GetLocality(MercatorBounds::FromLatLon(input[i]), result); + m_finder.GetLocality( + MercatorBounds::FromLatLon(input[i]), [&](search::LocalityItem const & item) { + item.GetSpecifiedOrDefaultName(StringUtf8Multilang::kEnglishCode, result); + }); TEST_EQUAL(result, results[i], ()); } } @@ -75,7 +76,6 @@ public: void ClearCaches() { m_finder.ClearCache(); } }; - } // namespace UNIT_CLASS_TEST(LocalityFinderTest, Smoke) diff --git a/search/search_tests/locality_selector_test.cpp b/search/search_tests/locality_selector_test.cpp index 29cf8d6270..e23f27c24a 100644 --- a/search/search_tests/locality_selector_test.cpp +++ b/search/search_tests/locality_selector_test.cpp @@ -2,16 +2,38 @@ #include "search/locality_finder.hpp" +#include "base/string_utils.hpp" + using namespace search; namespace { -string GetMatchedCity(m2::PointD const & point, vector const & items) +StringUtf8Multilang ToMultilang(string const & name) { + StringUtf8Multilang s; + s.AddString(StringUtf8Multilang::kEnglishCode, name); + return s; +} + +struct City +{ + City(string const & name, m2::PointD const & center, uint64_t population) + : m_item(ToMultilang(name), center, population) + { + } + + LocalityItem m_item; +}; + +string GetMatchedCity(m2::PointD const & point, vector const & cities) +{ + LocalitySelector selector(point); + for (auto const & city : cities) + selector(city.m_item); + string name; - LocalitySelector selector(name, point); - for (auto const & item : items) - selector(item); + selector.WithBestLocality( + [&](LocalityItem const & item) { item.GetName(StringUtf8Multilang::kEnglishCode, name); }); return name; } @@ -30,22 +52,19 @@ string GetMatchedCity(m2::PointD const & point, vector const & ite UNIT_TEST(LocalitySelector_Test2) { - vector const localities = { - {"Moscow", m2::PointD(37.61751, 67.45398), 11971516}, - {"Krasnogorsk", m2::PointD(37.34040, 67.58036), 135735}, - {"Khimki", m2::PointD(37.44499, 67.70070), 240463}, - {"Mytishchi", m2::PointD(37.73394, 67.73675), 180663}, - {"Dolgoprudny", m2::PointD(37.51425, 67.78073), 101979}}; + vector const cities = {{"Moscow", m2::PointD(37.61751, 67.45398), 11971516}, + {"Krasnogorsk", m2::PointD(37.34040, 67.58036), 135735}, + {"Khimki", m2::PointD(37.44499, 67.70070), 240463}, + {"Mytishchi", m2::PointD(37.73394, 67.73675), 180663}, + {"Dolgoprudny", m2::PointD(37.51425, 67.78073), 101979}}; { - auto const name = - GetMatchedCity(m2::PointD(37.53826, 67.53554), localities); + auto const name = GetMatchedCity(m2::PointD(37.53826, 67.53554), cities); TEST_EQUAL(name, "Moscow", ()); } { - auto const name = - GetMatchedCity(m2::PointD(37.46980, 67.66650), localities); + auto const name = GetMatchedCity(m2::PointD(37.46980, 67.66650), cities); TEST_EQUAL(name, "Khimki", ()); } }