From f4b0b9e8cff7d47b3c388e420bf87722a5c8978d Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Tue, 30 Jun 2020 10:30:20 +0300 Subject: [PATCH] [search] Use languages similar to device and query languages. --- indexer/feature_utils.cpp | 34 +++++++++------ indexer/feature_utils.hpp | 5 +++ search/processor.cpp | 5 ++- .../processor_test.cpp | 41 +++++++++++++++++++ 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/indexer/feature_utils.cpp b/indexer/feature_utils.cpp index f37716d331..bd62178906 100644 --- a/indexer/feature_utils.cpp +++ b/indexer/feature_utils.cpp @@ -95,14 +95,15 @@ bool GetBestName(StringUtf8Multilang const & src, vector const & priorit } vector GetSimilarToDeviceLanguages(int8_t deviceLang) -{ - static unordered_map> const kSimilarToDeviceLanguages = - { - {GetIndex("be"), {GetIndex("ru")}} - }; +{ + static unordered_map> const kSimilarLanguages = { + {GetIndex("be"), {GetIndex("ru")}}, + {GetIndex("ja"), {GetIndex("ja_kana"), GetIndex("ja_rm")}}, + {GetIndex("ko"), {GetIndex("ko_rm")}}, + {GetIndex("zh"), {GetIndex("zh_pinyin")}}}; - auto const it = kSimilarToDeviceLanguages.find(deviceLang); - if (it != kSimilarToDeviceLanguages.cend()) + auto const it = kSimilarLanguages.find(deviceLang); + if (it != kSimilarLanguages.cend()) return it->second; return {}; @@ -122,7 +123,7 @@ bool IsNativeLang(feature::RegionData const & regionData, int8_t deviceLang) return false; } -vector MakePrimaryNamePriorityList(int8_t deviceLang, bool preferDefault) +vector MakeLanguagesPriorityList(int8_t deviceLang, bool preferDefault) { vector langPriority = {deviceLang}; if (preferDefault) @@ -138,7 +139,7 @@ vector MakePrimaryNamePriorityList(int8_t deviceLang, bool preferDefault void GetReadableNameImpl(feature::RegionData const & regionData, StringUtf8Multilang const & src, int8_t deviceLang, bool preferDefault, bool allowTranslit, string & out) { - vector langPriority = MakePrimaryNamePriorityList(deviceLang, preferDefault); + vector langPriority = MakeLanguagesPriorityList(deviceLang, preferDefault); if (GetBestName(src, langPriority, out)) return; @@ -323,6 +324,15 @@ int GetFeatureViewportScale(TypesHolder const & types) return impl::GetFeatureEstimator().GetViewportScale(types); } +vector GetSimilar(int8_t lang) +{ + vector langs = {lang}; + + auto const similarLangs = GetSimilarToDeviceLanguages(lang); + langs.insert(langs.cend(), similarLangs.cbegin(), similarLangs.cend()); + return langs; +} + void GetPreferredNames(RegionData const & regionData, StringUtf8Multilang const & src, int8_t const deviceLang, bool allowTranslit, string & primary, string & secondary) { @@ -337,7 +347,7 @@ void GetPreferredNames(RegionData const & regionData, StringUtf8Multilang const if (IsNativeLang(regionData, deviceLang)) return GetReadableNameImpl(regionData, src, deviceLang, true, allowTranslit, primary); - vector primaryCodes = MakePrimaryNamePriorityList(deviceLang, false); + vector primaryCodes = MakeLanguagesPriorityList(deviceLang, false); if (!GetBestName(src, primaryCodes, primary) && allowTranslit) GetTransliteratedName(regionData, src, primary); @@ -396,14 +406,14 @@ int8_t GetNameForSearchOnBooking(RegionData const & regionData, StringUtf8Multil bool GetPreferredName(StringUtf8Multilang const & src, int8_t deviceLang, string & out) { - auto const priorityList = MakePrimaryNamePriorityList(deviceLang, true /* preferDefault */); + auto const priorityList = MakeLanguagesPriorityList(deviceLang, true /* preferDefault */); return GetBestName(src, priorityList, out); } vector GetDescriptionLangPriority(RegionData const & regionData, int8_t const deviceLang) { bool const preferDefault = IsNativeLang(regionData, deviceLang); - return MakePrimaryNamePriorityList(deviceLang, preferDefault); + return MakeLanguagesPriorityList(deviceLang, preferDefault); } vector GetCuisines(TypesHolder const & types) diff --git a/indexer/feature_utils.hpp b/indexer/feature_utils.hpp index eb0ac9f991..83af595c96 100644 --- a/indexer/feature_utils.hpp +++ b/indexer/feature_utils.hpp @@ -18,6 +18,11 @@ namespace feature /// Get viewport scale to show given feature. Used in search. int GetFeatureViewportScale(TypesHolder const & types); + // Returns following languages given |lang|: + // - |lang|; + // - similar to |lang| languages if provided; + std::vector GetSimilar(int8_t deviceLang); + /// When the language of the device is equal to one of the languages of the MWM /// (or similar to device languages) only single name scheme is used. See GetReadableName method. /// Primary name using priority: diff --git a/search/processor.cpp b/search/processor.cpp index 8eb673a880..1c76f2c943 100644 --- a/search/processor.cpp +++ b/search/processor.cpp @@ -29,6 +29,7 @@ #include "indexer/feature_covering.hpp" #include "indexer/feature_data.hpp" #include "indexer/feature_impl.hpp" +#include "indexer/feature_utils.hpp" #include "indexer/features_vector.hpp" #include "indexer/ftypes_matcher.hpp" #include "indexer/postcodes_matcher.hpp" @@ -202,7 +203,7 @@ void Processor::SetPreferredLocale(string const & locale) LOG(LINFO, ("New preferred locale:", locale)); int8_t const code = StringUtf8Multilang::GetLangIndex(languages::Normalize(locale)); - m_keywordsScorer.SetLanguages(LanguageTier::LANGUAGE_TIER_CURRENT, {code}); + m_keywordsScorer.SetLanguages(LanguageTier::LANGUAGE_TIER_CURRENT, feature::GetSimilar(code)); m_currentLocaleCode = CategoriesHolder::MapLocaleToInteger(locale); @@ -219,7 +220,7 @@ void Processor::SetInputLocale(string const & locale) int8_t const code = StringUtf8Multilang::GetLangIndex(languages::Normalize(locale)); LOG(LDEBUG, ("New input locale:", locale, "locale code:", code)); - m_keywordsScorer.SetLanguages(LanguageTier::LANGUAGE_TIER_INPUT, {code}); + m_keywordsScorer.SetLanguages(LanguageTier::LANGUAGE_TIER_INPUT, feature::GetSimilar(code)); m_inputLocaleCode = CategoriesHolder::MapLocaleToInteger(locale); } diff --git a/search/search_integration_tests/processor_test.cpp b/search/search_integration_tests/processor_test.cpp index 2b4dddd9ec..61e8c9bf70 100644 --- a/search/search_integration_tests/processor_test.cpp +++ b/search/search_integration_tests/processor_test.cpp @@ -2937,5 +2937,46 @@ UNIT_CLASS_TEST(ProcessorTest, StreetWithNumber) TEST(ResultsMatch("11-я Магистральная 11 ", rules), ()); } } + +UNIT_CLASS_TEST(ProcessorTest, SimilarLanguage) +{ + string const countryName = "Wonderland"; + + auto testLanguage = [&](string language, string searchString, bool expectedRes) { + auto request = MakeRequest(searchString, language); + auto const & results = request->Results(); + TEST_EQUAL(results.size(), expectedRes ? 1 : 0, (results, language, searchString, expectedRes)); + }; + + TestMultilingualPOI poi(m2::PointD(0.0, 0.0), "default", + {{"en", "Jiyugaoka Station"}, + {"int_name", "international"}, + {"ja", "自由が丘"}, + {"ja_kana", "じゆうがおか"}, + {"ko", "지유가오카"}}); + + auto wonderlandId = + BuildCountry(countryName, [&](TestMwmBuilder & builder) { builder.Add(poi); }); + + SetViewport(m2::RectD(-1, -1, 1, 1)); + + // Languages to search: default, English, international, device language (English by default), + // languages similar to device language, input language, languages similar to input language. + + // Search must use default, English and international languages for any query locale. + testLanguage("it", "default", true); + testLanguage("it", "Jiyugaoka Station", true); + testLanguage("it", "international", true); + + testLanguage("ja", "自由が丘", true); + // ja_kana is similar to ja. Search must use both ja and ja_kana for ja locale. + testLanguage("ja", "じゆうがおか", true); + + // ko is not in language list if query locale is not ko. + testLanguage("ja", "지유가오카", false); + testLanguage("en", "지유가오카", false); + // ko is in language list if query locale is ko. + testLanguage("ko", "지유가오카", true); +} } // namespace } // namespace search