From c14111a5bbe87cc0004b30a4f1d080c0d4ee161b Mon Sep 17 00:00:00 2001 From: Alex Zolotarev Date: Sun, 3 Aug 2014 01:27:41 +0300 Subject: [PATCH] [localizations] Split languages for data and for categories, they should be different, to support Chinese variations and Danish correctly --- indexer/categories_holder.cpp | 77 ++++++++++++++++++++--- indexer/categories_holder.hpp | 19 +++--- indexer/indexer_tests/categories_test.cpp | 38 ++++++++--- indexer/search_index_builder.cpp | 8 +-- map/address_finder.cpp | 19 +++--- search/intermediate_result.cpp | 17 ++--- search/intermediate_result.hpp | 6 +- search/params.cpp | 20 +----- search/params.hpp | 7 +-- search/search_engine.cpp | 17 ++--- search/search_query.cpp | 36 +++++++---- search/search_query.hpp | 14 +++-- 12 files changed, 173 insertions(+), 105 deletions(-) diff --git a/indexer/categories_holder.cpp b/indexer/categories_holder.cpp index 97a1ba241e..788f3ff7f4 100644 --- a/indexer/categories_holder.cpp +++ b/indexer/categories_holder.cpp @@ -5,7 +5,6 @@ #include "../coding/reader.hpp" #include "../coding/reader_streambuf.hpp" -#include "../coding/multilang_utf8_string.hpp" #include "../base/logging.hpp" #include "../base/stl_add.hpp" @@ -23,6 +22,8 @@ enum State } // unnamed namespace +int8_t const CategoriesHolder::UNSUPPORTED_LOCALE_CODE; + CategoriesHolder::CategoriesHolder(Reader * reader) { ReaderStreamBuf buffer(reader); @@ -127,17 +128,13 @@ void CategoriesHolder::LoadFromStream(istream & s) continue; } - int8_t const langCode = StringUtf8Multilang::GetLangIndex(*iter); - if (langCode == StringUtf8Multilang::UNSUPPORTED_LANGUAGE_CODE) - { - LOG(LWARNING, ("Invalid language code:", *iter, "at line:", lineNumber)); - continue; - } + int8_t const langCode = MapLocaleToInteger(*iter); + CHECK_NOT_EQUAL(langCode, UNSUPPORTED_LOCALE_CODE, ("Invalid language code:", *iter, "at line:", lineNumber)); while (++iter) { Category::Name name; - name.m_lang = langCode; + name.m_locale = langCode; name.m_name = *iter; if (name.m_name.empty()) @@ -165,7 +162,7 @@ void CategoriesHolder::LoadFromStream(istream & s) AddCategory(cat, types); } -bool CategoriesHolder::GetNameByType(uint32_t type, int8_t lang, string & name) const +bool CategoriesHolder::GetNameByType(uint32_t type, int8_t locale, string & name) const { pair const range = m_type2cat.equal_range(type); @@ -173,7 +170,7 @@ bool CategoriesHolder::GetNameByType(uint32_t type, int8_t lang, string & name) { Category const & cat = *i->second; for (size_t j = 0; j < cat.m_synonyms.size(); ++j) - if (cat.m_synonyms[j].m_lang == lang) + if (cat.m_synonyms[j].m_locale == locale) { name = cat.m_synonyms[j].m_name; return true; @@ -188,3 +185,63 @@ bool CategoriesHolder::GetNameByType(uint32_t type, int8_t lang, string & name) return false; } + +bool CategoriesHolder::IsTypeExist(uint32_t type) const +{ + pair const range = m_type2cat.equal_range(type); + return range.first != range.second; +} + +namespace +{ +struct Mapping +{ + char const * m_name; + int8_t m_code; +}; +} // namespace + +int8_t CategoriesHolder::MapLocaleToInteger(string const & locale) +{ + static const Mapping mapping[] = { + {"en", 1 }, + {"ru", 2 }, + {"uk", 3 }, + {"de", 4 }, + {"fr", 5 }, + {"it", 6 }, + {"es", 7 }, + {"ko", 8 }, + {"ja", 9 }, + {"cs", 10 }, + {"nl", 11 }, + {"zh-Hant", 12 }, + {"pl", 13 }, + {"pt", 14 }, + {"hu", 15 }, + {"th", 16 }, + {"zh-Hans", 17 }, + {"ar", 18 }, + {"da", 19 }, + }; + for (size_t i = 0; i < ARRAY_SIZE(mapping); ++i) + if (locale.find(mapping[i].m_name) == 0) + return mapping[i].m_code; + + // Special cases for different Chinese variations + if (locale.find("zh") == 0) + { + string lower = locale; + strings::AsciiToLower(lower); + + if (lower.find("hant") != string::npos + || lower.find("tw") != string::npos + || lower.find("hk") != string::npos + || lower.find("mo") != string::npos) + return 12; // Traditional Chinese + + return 17; // Simplified Chinese by default for all other cases + } + + return UNSUPPORTED_LOCALE_CODE; +} diff --git a/indexer/categories_holder.hpp b/indexer/categories_holder.hpp index 1b740b8cc4..198716be08 100644 --- a/indexer/categories_holder.hpp +++ b/indexer/categories_holder.hpp @@ -20,7 +20,9 @@ public: struct Name { string m_name; - int8_t m_lang; + /// This language/locale code is completely different from our built-in langs in multilang_utf8_string.cpp + /// and is only used for mapping user's input language to our values in categories.txt file + int8_t m_locale; uint8_t m_prefixLengthToSuggest; }; @@ -76,17 +78,12 @@ public: } } - /// Search name for type with preffered language. + /// Search name for type with preffered locale language. /// If no name for this language, return first (en) name. /// @return false if no categories for type. - bool GetNameByType(uint32_t type, int8_t lang, string & name) const; + bool GetNameByType(uint32_t type, int8_t locale, string & name) const; - inline bool IsTypeExist(uint32_t type) const - { - // pass any language - string dummy; - return GetNameByType(type, 0, dummy); - } + bool IsTypeExist(uint32_t type) const; inline void Swap(CategoriesHolder & r) { @@ -94,6 +91,10 @@ public: m_name2type.swap(r.m_name2type); } + /// Converts any language locale from UI to internal integer code + static int8_t MapLocaleToInteger(string const & locale); + static int8_t const UNSUPPORTED_LOCALE_CODE = -1; + private: void AddCategory(Category & cat, vector & types); static bool ValidKeyToken(StringT const & s); diff --git a/indexer/indexer_tests/categories_test.cpp b/indexer/indexer_tests/categories_test.cpp index 7191611731..9b9847a1a2 100644 --- a/indexer/indexer_tests/categories_test.cpp +++ b/indexer/indexer_tests/categories_test.cpp @@ -13,6 +13,9 @@ char const * TEST_STRING = "amenity-bench\n" "en:1bench|sit down|to sit\n" "de:0bank|auf die strafbank schicken\n" + "zh-Hans:长凳\n" + "zh-Hant:長板凳\n" + "da:bænk\n" "\n" "place-village|place-hamlet\n" "en:village\n" @@ -28,32 +31,47 @@ struct Checker { case 0: { - TEST_EQUAL(cat.m_synonyms.size(), 5, ()); - TEST_EQUAL(cat.m_synonyms[0].m_lang, StringUtf8Multilang::GetLangIndex("en"), ()); + TEST_EQUAL(cat.m_synonyms.size(), 8, ()); + TEST_EQUAL(cat.m_synonyms[0].m_locale, CategoriesHolder::MapLocaleToInteger("en"), ()); TEST_EQUAL(cat.m_synonyms[0].m_name, "bench", ()); TEST_EQUAL(cat.m_synonyms[0].m_prefixLengthToSuggest, 1, ()); - TEST_EQUAL(cat.m_synonyms[1].m_lang, StringUtf8Multilang::GetLangIndex("en"), ()); + TEST_EQUAL(cat.m_synonyms[1].m_locale, CategoriesHolder::MapLocaleToInteger("en"), ()); TEST_EQUAL(cat.m_synonyms[1].m_name, "sit down", ()); TEST_EQUAL(cat.m_synonyms[1].m_prefixLengthToSuggest, 10, ()); - TEST_EQUAL(cat.m_synonyms[2].m_lang, StringUtf8Multilang::GetLangIndex("en"), ()); + TEST_EQUAL(cat.m_synonyms[2].m_locale, CategoriesHolder::MapLocaleToInteger("en"), ()); TEST_EQUAL(cat.m_synonyms[2].m_name, "to sit", ()); - TEST_EQUAL(cat.m_synonyms[3].m_lang, StringUtf8Multilang::GetLangIndex("de"), ()); + TEST_EQUAL(cat.m_synonyms[3].m_locale, CategoriesHolder::MapLocaleToInteger("de"), ()); TEST_EQUAL(cat.m_synonyms[3].m_name, "bank", ()); TEST_EQUAL(cat.m_synonyms[3].m_prefixLengthToSuggest, 0, ()); - TEST_EQUAL(cat.m_synonyms[4].m_lang, StringUtf8Multilang::GetLangIndex("de"), ()); + TEST_EQUAL(cat.m_synonyms[4].m_locale, CategoriesHolder::MapLocaleToInteger("de"), ()); TEST_EQUAL(cat.m_synonyms[4].m_name, "auf die strafbank schicken", ()); + TEST_EQUAL(cat.m_synonyms[5].m_locale, CategoriesHolder::MapLocaleToInteger("zh_CN"), ()); + TEST_EQUAL(cat.m_synonyms[5].m_locale, CategoriesHolder::MapLocaleToInteger("zh_rCN"), ()); + TEST_EQUAL(cat.m_synonyms[5].m_locale, CategoriesHolder::MapLocaleToInteger("zh_HANS_CN"), ()); + TEST_EQUAL(cat.m_synonyms[5].m_locale, CategoriesHolder::MapLocaleToInteger("zh-Hans"), ()); + TEST_EQUAL(cat.m_synonyms[5].m_name, "长凳", ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh_TW"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh-MO"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh-rTW"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh_HANT_HK"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh_HK"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_locale, CategoriesHolder::MapLocaleToInteger("zh-Hant"), ()); + TEST_EQUAL(cat.m_synonyms[6].m_name, "長板凳", ()); + TEST_EQUAL(cat.m_synonyms[7].m_locale, CategoriesHolder::MapLocaleToInteger("da"), ()); + TEST_EQUAL(cat.m_synonyms[7].m_name, "bænk", ()); ++m_count; } break; - case 1: case 2: + case 1: + case 2: { TEST_EQUAL(cat.m_synonyms.size(), 3, ()); - TEST_EQUAL(cat.m_synonyms[0].m_lang, StringUtf8Multilang::GetLangIndex("en"), ()); + TEST_EQUAL(cat.m_synonyms[0].m_locale, CategoriesHolder::MapLocaleToInteger("en"), ()); TEST_EQUAL(cat.m_synonyms[0].m_name, "village", ()); - TEST_EQUAL(cat.m_synonyms[1].m_lang, StringUtf8Multilang::GetLangIndex("de"), ()); + TEST_EQUAL(cat.m_synonyms[1].m_locale, CategoriesHolder::MapLocaleToInteger("de"), ()); TEST_EQUAL(cat.m_synonyms[1].m_name, "dorf", ()); TEST_EQUAL(cat.m_synonyms[1].m_prefixLengthToSuggest, 2, ()); - TEST_EQUAL(cat.m_synonyms[2].m_lang, StringUtf8Multilang::GetLangIndex("de"), ()); + TEST_EQUAL(cat.m_synonyms[2].m_locale, CategoriesHolder::MapLocaleToInteger("de"), ()); TEST_EQUAL(cat.m_synonyms[2].m_name, "weiler", ()); TEST_EQUAL(cat.m_synonyms[2].m_prefixLengthToSuggest, 4, ()); ++m_count; diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index e069ad5821..cf90d108da 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -170,10 +170,10 @@ class FeatureInserter m_valueSaver.Save(sink, v); } - /// There are 3 different ways of search index skipping: \n - /// - skip features in any case (m_skipFeatures) \n - /// - skip features with empty names (m_enFeature) \n - /// - skip specified types for features with empty names (m_enTypes) \n + /// There are 3 different ways of search index skipping: + /// - skip features in any case (m_skipFeatures) + /// - skip features with empty names (m_enFeature) + /// - skip specified types for features with empty names (m_enTypes) class SkipIndexing { /// Array index (0, 1) means type level for checking (1, 2). diff --git a/map/address_finder.cpp b/map/address_finder.cpp index 72772efa63..0daaa6a967 100644 --- a/map/address_finder.cpp +++ b/map/address_finder.cpp @@ -4,6 +4,7 @@ #include "../indexer/classificator.hpp" #include "../indexer/feature_visibility.hpp" +#include "../indexer/categories_holder.hpp" #include "../platform/preferred_languages.hpp" @@ -319,7 +320,7 @@ namespace return true; } - static void GetReadableTypes(search::Engine const * eng, int8_t lang, + static void GetReadableTypes(search::Engine const * eng, int8_t locale, feature::TypesHolder & types, search::AddressInfo & info) { @@ -330,7 +331,7 @@ namespace for (size_t i = 0; i < count; ++i) { string s; - if (eng->GetNameByType(types[i], lang, s)) + if (eng->GetNameByType(types[i], locale, s)) info.m_types.push_back(s); } @@ -356,9 +357,9 @@ namespace } } - void FillAddress(search::Engine const * eng, search::AddressInfo & info) + void FillAddress(search::Engine const * eng, search::AddressInfo & info, string const & lang) { - int8_t const lang = eng->GetCurrentLanguage(); + int8_t const locale = CategoriesHolder::MapLocaleToInteger(lang); SortResults(); @@ -381,7 +382,7 @@ namespace { info.m_name = m_cont[i].m_name; - GetReadableTypes(eng, lang, m_cont[i].m_types, info); + GetReadableTypes(eng, locale, m_cont[i].m_types, info); } } @@ -484,9 +485,10 @@ void Framework::GetAddressInfoForGlobalPoint(m2::PointD const & pt, search::Addr DoGetAddressInfo getAddress(pt, scale, GetChecker(), addressR); m_model.ForEachFeature(rect, getAddress, scale); - getAddress.FillAddress(GetSearchEngine(), info); + // @TODO Pass language as a parameter from UI + getAddress.FillAddress(GetSearchEngine(), info, languages::CurrentLanguage()); - /// @todo Temporarily commented - it's slow and not used in UI + // @todo Temporarily commented - it's slow and not used in UI //GetLocality(pt, info); } @@ -507,7 +509,8 @@ void Framework::GetAddressInfo(FeatureType const & ft, m2::PointD const & pt, se // FeatureType::WORST_GEOMETRY - no need to check on visibility DoGetAddressInfo getAddress(pt, FeatureType::WORST_GEOMETRY, GetChecker(), addressR); getAddress(ft); - getAddress.FillAddress(GetSearchEngine(), info); + // @TODO Pass language as a parameter from UI + getAddress.FillAddress(GetSearchEngine(), info, languages::CurrentLanguage()); /// @todo Temporarily commented - it's slow and not used in UI //GetLocality(pt, info); diff --git a/search/intermediate_result.cpp b/search/intermediate_result.cpp index 03afe967fe..82a82709a5 100644 --- a/search/intermediate_result.cpp +++ b/search/intermediate_result.cpp @@ -216,7 +216,7 @@ Result PreResult2::GenerateFinalResult( storage::CountryInfoGetter const * pInfo, CategoriesHolder const * pCat, set const * pTypes, - int8_t lang) const + int8_t locale) const { storage::CountryInfo info; @@ -234,14 +234,14 @@ Result PreResult2::GenerateFinalResult( switch (m_resultType) { case RESULT_FEATURE: - return Result(m_id, GetCenter(), m_str, info.m_name, GetFeatureType(pCat, pTypes, lang) + return Result(m_id, GetCenter(), m_str, info.m_name, ReadableFeatureType(pCat, pTypes, locale) #ifdef DEBUG + ' ' + strings::to_string(static_cast(m_rank)) #endif , type); case RESULT_BUILDING: - return Result(GetCenter(), m_str, info.m_name, GetFeatureType(pCat, pTypes, lang)); + return Result(GetCenter(), m_str, info.m_name, ReadableFeatureType(pCat, pTypes, locale)); default: ASSERT_EQUAL(m_resultType, RESULT_LATLON, ()); @@ -252,9 +252,9 @@ Result PreResult2::GenerateFinalResult( Result PreResult2::GeneratePointResult( CategoriesHolder const * pCat, set const * pTypes, - int8_t lang) const + int8_t locale) const { - return Result(GetCenter(), m_str, GetFeatureType(pCat, pTypes, lang)); + return Result(GetCenter(), m_str, ReadableFeatureType(pCat, pTypes, locale)); } bool PreResult2::LessRank(PreResult2 const & r1, PreResult2 const & r2) @@ -361,19 +361,20 @@ uint32_t PreResult2::GetBestType(set const * pPrefferedTypes) const return t; } -string PreResult2::GetFeatureType(CategoriesHolder const * pCat, +string PreResult2::ReadableFeatureType(CategoriesHolder const * pCat, set const * pTypes, - int8_t lang) const + int8_t locale) const { ASSERT_EQUAL(m_resultType, RESULT_FEATURE, ()); + // @TODO print all types, not just one uint32_t const type = GetBestType(pTypes); ASSERT_NOT_EQUAL(type, 0, ()); if (pCat) { string name; - if (pCat->GetNameByType(type, lang, name)) + if (pCat->GetNameByType(type, locale, name)) return name; } diff --git a/search/intermediate_result.hpp b/search/intermediate_result.hpp index 6334ff68c3..2f56e062c2 100644 --- a/search/intermediate_result.hpp +++ b/search/intermediate_result.hpp @@ -95,7 +95,7 @@ public: Result GeneratePointResult(CategoriesHolder const * pCat, set const * pTypes, - int8_t lang) const; + int8_t locale) const; static bool LessRank(PreResult2 const & r1, PreResult2 const & r2); static bool LessDistance(PreResult2 const & r1, PreResult2 const & r2); @@ -138,9 +138,9 @@ private: bool IsEqualCommon(PreResult2 const & r) const; - string GetFeatureType(CategoriesHolder const * pCat, + string ReadableFeatureType(CategoriesHolder const * pCat, set const * pTypes, - int8_t lang) const; + int8_t locale) const; FeatureID m_id; feature::TypesHolder m_types; diff --git a/search/params.cpp b/search/params.cpp index 244db35e1a..46443de225 100644 --- a/search/params.cpp +++ b/search/params.cpp @@ -6,9 +6,7 @@ namespace search { -SearchParams::SearchParams() -: m_inputLanguageCode(StringUtf8Multilang::UNSUPPORTED_LANGUAGE_CODE), - m_searchMode(ALL), m_forceSearch(false), m_validPos(false) +SearchParams::SearchParams() : m_searchMode(ALL), m_forceSearch(false), m_validPos(false) { } @@ -19,24 +17,10 @@ void SearchParams::SetPosition(double lat, double lon) m_validPos = true; } -void SearchParams::SetInputLanguage(string const & language) -{ - /// @todo take into an account zh_pinyin, ko_rm and ja_rm - size_t const delimPos = language.find_first_of("-_"); - - m_inputLanguageCode = StringUtf8Multilang::GetLangIndex( - delimPos == string::npos ? language : language.substr(0, delimPos)); -} - -bool SearchParams::IsLanguageValid() const -{ - return (m_inputLanguageCode != StringUtf8Multilang::UNSUPPORTED_LANGUAGE_CODE); -} - bool SearchParams::IsEqualCommon(SearchParams const & rhs) const { return (m_query == rhs.m_query && - m_inputLanguageCode == rhs.m_inputLanguageCode && + m_inputLanguage == rhs.m_inputLanguage && m_validPos == rhs.m_validPos && m_searchMode == rhs.m_searchMode); } diff --git a/search/params.hpp b/search/params.hpp index 3a03e947f0..efd79f1397 100644 --- a/search/params.hpp +++ b/search/params.hpp @@ -42,8 +42,7 @@ namespace search bool IsValidPosition() const { return m_validPos; } /// @param[in] language can be "fr", "en-US", "ru_RU" etc. - void SetInputLanguage(string const & language); - bool IsLanguageValid() const; + void SetInputLanguage(string const & language) { m_inputLanguage = language; } bool IsEqualCommon(SearchParams const & rhs) const; @@ -53,9 +52,7 @@ namespace search SearchCallbackT m_callback; string m_query; - /// Can be -1 (@see StringUtf8Multilang::UNSUPPORTED_LANGUAGE_CODE), - /// in the case when input language is unknown. - int8_t m_inputLanguageCode; + string m_inputLanguage; double m_lat, m_lon; diff --git a/search/search_engine.cpp b/search/search_engine.cpp index 488fc6cdba..958ed0ec54 100644 --- a/search/search_engine.cpp +++ b/search/search_engine.cpp @@ -43,7 +43,7 @@ namespace class InitSuggestions { - // Key - is a string with language. + // Key - is a string with suggestion's _locale_ code, not a language from multilang_utf8_string.cpp typedef map, uint8_t> SuggestMapT; SuggestMapT m_suggests; @@ -54,7 +54,7 @@ public: { strings::UniString const uniName = NormalizeAndSimplifyString(name.m_name); - uint8_t & score = m_suggests[make_pair(uniName, name.m_lang)]; + uint8_t & score = m_suggests[make_pair(uniName, name.m_locale)]; if (score == 0 || score > name.m_prefixLengthToSuggest) score = name.m_prefixLengthToSuggest; } @@ -288,8 +288,8 @@ void Engine::SearchAsync(bool viewportPoints) m_pQuery->SetSearchInWorld(params.NeedSearch(SearchParams::SEARCH_WORLD)); m_pQuery->SetSortByViewport(params.IsSortByViewport()); - if (params.IsLanguageValid()) - m_pQuery->SetInputLanguage(params.m_inputLanguageCode); + // Language validity is checked inside + m_pQuery->SetInputLanguage(params.m_inputLanguage); m_pQuery->SetQuery(params.m_query); bool const emptyQuery = m_pQuery->IsEmptyQuery(); @@ -399,12 +399,12 @@ string Engine::GetCountryName(string const & id) return GetCountryNameT(id); } -bool Engine::GetNameByType(uint32_t type, int8_t lang, string & name) const +bool Engine::GetNameByType(uint32_t type, int8_t locale, string & name) const { uint8_t level = ftype::GetLevel(type); while (level >= 2) { - if (m_pData->m_categories.GetNameByType(type, lang, name)) + if (m_pData->m_categories.GetNameByType(type, locale, name)) return true; ftype::TruncValue(type, --level); } @@ -416,11 +416,6 @@ m2::RectD Engine::GetCountryBounds(string const & file) const return m_pData->m_infoGetter.CalcLimitRect(file); } -int8_t Engine::GetCurrentLanguage() const -{ - return m_pQuery->GetPrefferedLanguage(); -} - void Engine::ClearViewportsCache() { threads::MutexGuard guard(m_searchMutex); diff --git a/search/search_query.cpp b/search/search_query.cpp index fe43da6974..607b89f9e1 100644 --- a/search/search_query.cpp +++ b/search/search_query.cpp @@ -74,7 +74,8 @@ Query::Query(Index const * pIndex, CategoriesHolder const * pCategories, StringsToSuggestVectorT const * pStringsToSuggest, storage::CountryInfoGetter const * pInfoGetter) - : m_pIndex(pIndex), + : m_inputLocaleCode(CategoriesHolder::UNSUPPORTED_LOCALE_CODE), + m_pIndex(pIndex), m_pCategories(pCategories), m_pStringsToSuggest(pStringsToSuggest), m_pInfoGetter(pInfoGetter), @@ -203,22 +204,27 @@ void Query::SetPreferredLanguage(string const & lang) // Default initialization. // If you want to reset input language, call SetInputLanguage before search. - SetInputLanguage(code); + SetInputLanguage(lang); #ifdef FIND_LOCALITY_TEST m_locality.SetLanguage(code); #endif } -void Query::SetInputLanguage(int8_t lang) +void Query::SetInputLanguage(string const & lang) { - LOG(LDEBUG, ("New input language = ", lang)); - SetLanguage(LANG_INPUT, lang); -} + if (!lang.empty()) + { + LOG(LDEBUG, ("New input language = ", lang)); -int8_t Query::GetPrefferedLanguage() const -{ - return GetLanguage(LANG_CURRENT); + // For "data" language we need only 2-letter code + size_t const delimPos = lang.find_first_of("-_"); + int8_t const code = StringUtf8Multilang::GetLangIndex( + delimPos == string::npos ? lang : lang.substr(0, delimPos)); + SetLanguage(LANG_INPUT, code); + + m_inputLocaleCode = CategoriesHolder::MapLocaleToInteger(lang); + } } void Query::ClearCaches() @@ -2103,7 +2109,7 @@ void Query::SuggestStrings(Results & res) } } -bool Query::MatchForSuggestionsImpl(strings::UniString const & token, int8_t lang, string const & prolog, Results & res) +bool Query::MatchForSuggestionsImpl(strings::UniString const & token, int8_t locale, string const & prolog, Results & res) { bool ret = false; @@ -2113,7 +2119,7 @@ bool Query::MatchForSuggestionsImpl(strings::UniString const & token, int8_t lan strings::UniString const & s = it->m_name; if ((it->m_prefixLength <= token.size()) && (token != s) && // do not push suggestion if it already equals to token - (it->m_lang == lang) && // push suggestions only for needed language + (it->m_locale == locale) && // push suggestions only for needed language StartsWith(s.begin(), s.end(), token.begin(), token.end())) { string const utf8Str = strings::ToUtf8(s); @@ -2132,8 +2138,12 @@ void Query::MatchForSuggestions(strings::UniString const & token, Results & res) string prolog; RemoveStringPrefix(*m_query, prolog); - if (!MatchForSuggestionsImpl(token, GetLanguage(LANG_INPUT), prolog, res)) - MatchForSuggestionsImpl(token, GetLanguage(LANG_EN), prolog, res); + static int8_t const enLocaleCode = CategoriesHolder::MapLocaleToInteger("en"); + if (!MatchForSuggestionsImpl(token, m_inputLocaleCode, prolog, res) + && m_inputLocaleCode != enLocaleCode) + { + MatchForSuggestionsImpl(token, enLocaleCode, prolog, res); + } } m2::RectD const & Query::GetViewport(ViewportID vID /*= DEFAULT_V*/) const diff --git a/search/search_query.hpp b/search/search_query.hpp index 0b5df7bfe4..d5eb031fdf 100644 --- a/search/search_query.hpp +++ b/search/search_query.hpp @@ -60,10 +60,10 @@ public: { strings::UniString m_name; uint8_t m_prefixLength; - int8_t m_lang; + int8_t m_locale; - SuggestT(strings::UniString const & name, uint8_t len, int8_t lang) - : m_name(name), m_prefixLength(len), m_lang(lang) + SuggestT(strings::UniString const & name, uint8_t len, int8_t locale) + : m_name(name), m_prefixLength(len), m_locale(locale) { } }; @@ -91,9 +91,11 @@ public: inline void SetSearchInWorld(bool b) { m_worldSearch = b; } inline void SetSortByViewport(bool b) { m_sortByViewport = b; } + /// Suggestions language code, not the same as we use in mwm data + int8_t m_inputLocaleCode; + void SetPreferredLanguage(string const & lang); - void SetInputLanguage(int8_t lang); - int8_t GetPrefferedLanguage() const; + void SetInputLanguage(string const & lang); void SetQuery(string const & query); inline bool IsEmptyQuery() const { return (m_prefix.empty() && m_tokens.empty()); } @@ -206,7 +208,7 @@ private: //@} void SuggestStrings(Results & res); - bool MatchForSuggestionsImpl(strings::UniString const & token, int8_t lang, string const & prolog, Results & res); + bool MatchForSuggestionsImpl(strings::UniString const & token, int8_t locale, string const & prolog, Results & res); void MatchForSuggestions(strings::UniString const & token, Results & res); void GetBestMatchName(FeatureType const & f, string & name) const;