[categories] Do correct distinguishing of current language and current locale (zh, zh-Hans, zh-Hant).

android/jni/com/mapswithme/maps/Framework.cpp

@@ -923,7 +923,7 @@ extern "C"
       m_giClass = m_env->FindClass("com/mapswithme/maps/guides/GuideInfo");
       m_methodID = m_env->GetMethodID(m_giClass,
                 "<init>", "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V");
-      m_lang = languages::CurrentLanguage();
+      m_lang = languages::GetCurrentNorm();
     }
 
     jclass GetClass() const { return m_giClass; }

android/jni/com/mapswithme/maps/SearchActivity.cpp

@@ -207,7 +207,7 @@ Java_com_mapswithme_maps_SearchActivity_nativeRunSearch(
   search::SearchParams params;
 
   params.m_query = jni::ToNativeString(env, s);
-  params.SetInputLanguage(jni::ToNativeString(env, lang));
+  params.SetInputLocale(jni::ToNativeString(env, lang));
 
   params.SetSearchMode(searchMode);
 
@@ -304,7 +304,7 @@ Java_com_mapswithme_maps_SearchActivity_runInteractiveSearch(JNIEnv * env, jobje
 {
   search::SearchParams params;
   params.m_query = jni::ToNativeString(env, query);
-  params.SetInputLanguage(jni::ToNativeString(env, lang));
+  params.SetInputLocale(jni::ToNativeString(env, lang));
 
   g_framework->NativeFramework()->StartInteractiveSearch(params);
 }

android/jni/com/mapswithme/platform/Language.cpp

@@ -27,12 +27,25 @@ string GetAndroidSystemLanguage()
   jobject localeInstance = env->CallStaticObjectMethod(localeClass, localeGetDefaultId);
   ASSERT(localeInstance, ());
 
-  jmethodID localeGetLanguageId = env->GetMethodID(localeClass, "getLanguage", "()Ljava/lang/String;");
+  jmethodID localeGetLanguageId = env->GetMethodID(localeClass, "toString", "()Ljava/lang/String;");
   ASSERT(localeGetLanguageId, ());
 
   jstring langString = (jstring)env->CallObjectMethod(localeInstance, localeGetLanguageId);
   ASSERT(langString, ());
 
+  // I'm out of ideas here. This simple code works at startup, but crashes a bit later.
+  // Probably, some kind of specific threads routine.
+  /*
+  jclass langClass = env->FindClass("com/mapswithme/util/Language");
+  ASSERT(langClass, ());
+
+  jmethodID langGetDefaultId = env->GetStaticMethodID(langClass, "getDefault", "()Ljava/lang/String;");
+  ASSERT(langGetDefaultId, ());
+
+  jstring langString = (jstring)env->CallStaticObjectMethod(langClass, langGetDefaultId);
+  ASSERT(langString, ());
+  */
+
   string res = jni::ToNativeString(env, langString);
   if (res.empty())
     res = DEFAULT_LANG;

indexer/feature.cpp

@@ -212,9 +212,7 @@ struct BestMatchedLangNames
   bool operator()(int8_t code, string const & name)
   {
     static int8_t defaultCode = StringUtf8Multilang::GetLangIndex("default");
-    // @TODO support list of preferred languages
-    // We can get them also from input keyboard languages
-    static int8_t const nativeCode = StringUtf8Multilang::GetLangIndex(languages::CurrentLanguage());
+    static int8_t const nativeCode = StringUtf8Multilang::GetLangIndex(languages::GetCurrentNorm());
     static int8_t const intCode = StringUtf8Multilang::GetLangIndex("int_name");
     static int8_t const englishCode = StringUtf8Multilang::GetLangIndex("en");
 

iphone/Maps/Classes/MapsAppDelegate.mm

@@ -438,7 +438,7 @@ void InitLocalizedStrings()
   notification.timeZone = [NSTimeZone defaultTimeZone];
   notification.soundName = UILocalNotificationDefaultSoundName;
 
-  string const lang = languages::CurrentLanguage();
+  string const lang = languages::GetCurrentNorm();
   NSString * message = [NSString stringWithUTF8String:guide.GetAdMessage(lang).c_str()];
   notification.alertBody = message;
   notification.userInfo = @{

iphone/Maps/Classes/SearchView.mm

@@ -311,7 +311,7 @@ __weak SearchView * selfPointer;
   sp.SetSearchMode(search::SearchParams::ALL);
   sp.m_query = [[newQuery precomposedStringWithCompatibilityMapping] UTF8String];
   sp.m_callback = bind(&onSearchResultCallback, _1);
-  sp.SetInputLanguage([GetKeyboardInputLanguage() UTF8String]);
+  sp.SetInputLocale([GetKeyboardInputLanguage() UTF8String]);
   sp.SetForceSearch(force == YES);
 }
 
@@ -444,7 +444,7 @@ static void onSearchResultCallback(search::Results const & results)
 
     search::SearchParams params;
     params.m_query = [[self.searchBar.textField.text precomposedStringWithCompatibilityMapping] UTF8String];
-    params.SetInputLanguage([GetKeyboardInputLanguage() UTF8String]);
+    params.SetInputLocale([GetKeyboardInputLanguage() UTF8String]);
 
     f.StartInteractiveSearch(params);
 

iphone/Maps/Settings/CountriesViewController.mm

@@ -63,7 +63,7 @@ static bool getGuideName(string & name, storage::TIndex const & index)
   Framework & f = GetFramework();
   if ((f.Storage().CountriesCount(index) == 0) && f.GetGuideInfo(index, info))
   {
-    string const lang = languages::CurrentLanguage();
+    string const lang = languages::GetCurrentNorm();
     name = info.GetAdTitle(lang);
     return true;
   }
@@ -266,7 +266,7 @@ static bool getGuideName(string & name, storage::TIndex const & index)
     NSString * title = [actionSheet buttonTitleAtIndex:buttonIndex];
     guides::GuideInfo info;
     BOOL isGuideAvailable = f.GetGuideInfo(m_clickedIndex, info);
-    string const lang = languages::CurrentLanguage();
+    string const lang = languages::GetCurrentNorm();
     if (isGuideAvailable && [title isEqualToString:[NSString stringWithUTF8String:info.GetAdTitle(lang).c_str()]])
     {
       NSURL * guideUrl = [NSURL URLWithString:[NSString stringWithUTF8String:info.GetAppID().c_str()]];
@@ -478,7 +478,6 @@ static bool getGuideName(string & name, storage::TIndex const & index)
   string guideAdevertiseString;
   if (getGuideName(guideAdevertiseString, m_clickedIndex))
   {
-    string const lang = languages::CurrentLanguage();
     [as addButtonWithTitle:[NSString stringWithUTF8String:guideAdevertiseString.c_str()]];
     ++numOfButtons;
   }

map/address_finder.cpp

@@ -357,9 +357,9 @@ namespace
       }
     }
 
-    void FillAddress(search::Engine const * eng, search::AddressInfo & info, string const & lang)
+    void FillAddress(search::Engine const * eng, search::AddressInfo & info)
     {
-      int8_t const locale = CategoriesHolder::MapLocaleToInteger(lang);
+      int8_t const locale = CategoriesHolder::MapLocaleToInteger(languages::GetCurrentOrig());
 
       SortResults();
 
@@ -485,8 +485,7 @@ void Framework::GetAddressInfoForGlobalPoint(m2::PointD const & pt, search::Addr
   DoGetAddressInfo getAddress(pt, scale, GetChecker(), addressR);
 
   m_model.ForEachFeature(rect, getAddress, scale);
-  // @TODO Pass language as a parameter from UI
-  getAddress.FillAddress(GetSearchEngine(), info, languages::CurrentLanguage());
+  getAddress.FillAddress(GetSearchEngine(), info);
 
   // @todo Temporarily commented - it's slow and not used in UI
   //GetLocality(pt, info);
@@ -509,8 +508,7 @@ void Framework::GetAddressInfo(FeatureType const & ft, m2::PointD const & pt, se
   // FeatureType::WORST_GEOMETRY - no need to check on visibility
   DoGetAddressInfo getAddress(pt, FeatureType::WORST_GEOMETRY, GetChecker(), addressR);
   getAddress(ft);
-  // @TODO Pass language as a parameter from UI
-  getAddress.FillAddress(GetSearchEngine(), info, languages::CurrentLanguage());
+  getAddress.FillAddress(GetSearchEngine(), info);
 
   /// @todo Temporarily commented - it's slow and not used in UI
   //GetLocality(pt, info);

map/framework.cpp

@@ -269,6 +269,8 @@ Framework::Framework()
   // Restore temporary states from persistent Settings storage
   RestoreSesame();
   LOG(LDEBUG, ("Sesame initialized"));
+
+  LOG(LINFO, ("System languages:", languages::GetPreferred()));
 }
 
 Framework::~Framework()
@@ -1111,7 +1113,7 @@ search::Engine * Framework::GetSearchEngine() const
                               pl.GetReader(SEARCH_CATEGORIES_FILE_NAME),
                               pl.GetReader(PACKED_POLYGONS_FILE),
                               pl.GetReader(COUNTRIES_FILE),
-                              languages::CurrentLanguage()));
+                              languages::GetCurrentOrig()));
 
       m_pSearchEngine->SupportOldFormat(m_lowestMapVersion < feature::DataHeader::v3);
     }

map/map_tests/bookmarks_test.cpp

@@ -401,7 +401,7 @@ UNIT_TEST(Bookmarks_AddressInfo)
   fm.OnSize(800, 600);
 
   // assume that developers have English or Russian system language :)
-  string const lang = languages::CurrentLanguage();
+  string const lang = languages::GetCurrentNorm();
   LOG(LINFO, ("Current language =", lang));
 
   // default name (street in russian, category in english).

platform/platform_tests/language_test.cpp

@@ -1,48 +1,31 @@
 #include "../../testing/testing.hpp"
 
-#include "../../std/string.hpp"
-#include "../../std/vector.hpp"
-
 #include "../preferred_languages.hpp"
 
-namespace languages
+#include "../../std/string.hpp"
+
+
+UNIT_TEST(LangNormalize_Smoke)
 {
-  void FilterLanguages(vector<string> & langs);
+  char const * arr1[] = { "en", "en-GB", "zh", "es-SP", "zh-penyn", "en-US", "ru_RU", "es" };
+  char const * arr2[] = { "en", "en", "zh", "es", "zh", "en", "ru", "es" };
+  STATIC_ASSERT(ARRAY_SIZE(arr1) == ARRAY_SIZE(arr2));
+
+  for (size_t i = 0; i < ARRAY_SIZE(arr1); ++i)
+    TEST_EQUAL(arr2[i], languages::Normalize(arr1[i]), ());
 }
 
-UNIT_TEST(LangFilter)
+UNIT_TEST(PrefLanguages_Smoke)
 {
-  vector<string> v;
-  v.push_back("en");
-  v.push_back("en-GB");
-  v.push_back("zh");
-  v.push_back("es-SP");
-  v.push_back("zh-penyn");
-  v.push_back("en-US");
-  v.push_back("ru_RU");
-  v.push_back("es");
+  string s = languages::GetPreferred();
+  TEST(!s.empty(), ());
+  cout << "Preferred langs: " << s << endl;
 
-  languages::FilterLanguages(v);
+  s = languages::GetCurrentOrig();
+  TEST(!s.empty(), ());
+  cout << "Current original lang: " << s << endl;
 
-  vector<string> c;
-  c.push_back("en");
-  c.push_back("zh");
-  c.push_back("es");
-  c.push_back("ru");
-
-  TEST_EQUAL(v.size(), c.size(), (v, c));
-  for (size_t i = 0; i < c.size(); ++i)
-  {
-    TEST_EQUAL(c[i], v[i], (v, c));
-  }
-}
-
-UNIT_TEST(PrefLanguagesSmoke)
-{
-  string const prefLangs = languages::PreferredLanguages();
-  TEST_GREATER(prefLangs.size(), 0, ());
-  cout << "Preferred langs: " << prefLangs << endl;
-  string const currLang = languages::CurrentLanguage();
-  TEST_GREATER(currLang.size(), 0, ());
-  cout << "Current lang: " << currLang << endl;
+  s = languages::GetCurrentNorm();
+  TEST(!s.empty(), ());
+  cout << "Current normalized lang: " << s << endl;
 }

platform/preferred_languages.cpp

@@ -45,45 +45,7 @@ static const MSLocale gLocales[] = {{0x1,"ar"},{0x2,"bg"},{0x3,"ca"},{0x4,"zh-Ha
 namespace languages
 {
 
-class LangFilter
-{
-  set<string> & m_known;
-public:
-  LangFilter(set<string> & known) : m_known(known) {}
-  bool operator()(string const & t)
-  {
-    return !m_known.insert(t).second;
-  }
-};
-
-class NormalizeFilter
-{
-public:
-  void operator()(string & t)
-  {
-    strings::SimpleTokenizer const iter(t, "-_ ");
-    if (iter)
-      t = *iter;
-    else
-    {
-      LOG(LWARNING, ("Invalid language"));
-    }
-  }
-};
-
-void FilterLanguages(vector<string> & langs)
-{
-  // normalize languages: en-US -> en, ru_RU -> ru etc.
-  for_each(langs.begin(), langs.end(), NormalizeFilter());
-  {
-    // tmp storage
-    set<string> known;
-    // remove duplicate languages
-    langs.erase(remove_if(langs.begin(), langs.end(), LangFilter(known)), langs.end());
-  }
-}
-
-void SystemPreferredLanguages(vector<string> & languages)
+void GetSystemPreferred(vector<string> & languages)
 {
 #if defined(OMIM_OS_MAC) || defined(OMIM_OS_IPHONE)
   // Mac and iOS implementation
@@ -166,14 +128,12 @@ void SystemPreferredLanguages(vector<string> & languages)
 #else
   #error "Define language preferences for your platform"
 #endif
-
-  FilterLanguages(languages);
 }
 
-string PreferredLanguages()
+string GetPreferred()
 {
   vector<string> arr;
-  SystemPreferredLanguages(arr);
+  GetSystemPreferred(arr);
 
   // generate output string
   string result;
@@ -182,6 +142,7 @@ string PreferredLanguages()
     result.append(arr[i]);
     result.push_back('|');
   }
+
   if (result.empty())
     result = "default";
   else
@@ -189,14 +150,26 @@ string PreferredLanguages()
   return result;
 }
 
-string CurrentLanguage()
+string GetCurrentOrig()
 {
   vector<string> arr;
-  SystemPreferredLanguages(arr);
+  GetSystemPreferred(arr);
   if (arr.empty())
     return "en";
   else
     return arr[0];
 }
 
+string Normalize(string const & lang)
+{
+  strings::SimpleTokenizer const iter(lang, "-_ ");
+  ASSERT(iter, ());
+  return *iter;
+}
+
+string GetCurrentNorm()
+{
+  return Normalize(GetCurrentOrig());
+}
+
 }

platform/preferred_languages.hpp

@@ -7,13 +7,22 @@ namespace languages
 
 /// @note This functions are heavy enough to call them often. Be careful.
 //@{
-/// @return system language preferences in the form "en|ru|es|zh"
-string PreferredLanguages();
-/// @return language code for current user in the form "en"
+
+/// @return List of original system languages in the form "en-US|ru-RU|es|zh-Hant".
+string GetPreferred();
+
+/// @return Original language code for the current user in the form "en-US", "zh-Hant".
+string GetCurrentOrig();
+
+/// @return Normalized language code for the current user in the form "en", "zh".
+//@{
 /// Returned languages are normalized to our supported languages in the core, see multilang_utf8_string.cpp
 /// and should not be used with any sub-locales like zh-Hans/zh-Hant.
-/// Some langs like Danish (da) are not supported too in the core
-string CurrentLanguage();
+/// Some langs like Danish (da) are not supported in the core too, but used as a locale.
+string Normalize(string const & lang);
+string GetCurrentNorm();
+//@}
+
 //@}
 
 }

search/params.cpp

@@ -20,7 +20,7 @@ void SearchParams::SetPosition(double lat, double lon)
 bool SearchParams::IsEqualCommon(SearchParams const & rhs) const
 {
   return (m_query == rhs.m_query &&
-          m_inputLanguage == rhs.m_inputLanguage &&
+          m_inputLocale == rhs.m_inputLocale &&
           m_validPos == rhs.m_validPos &&
           m_searchMode == rhs.m_searchMode);
 }
@@ -28,7 +28,9 @@ bool SearchParams::IsEqualCommon(SearchParams const & rhs) const
 string DebugPrint(SearchParams const & params)
 {
   ostringstream stream;
-  stream << "search::SearchParams: Query = " << params.m_query << " Mode = " << params.m_searchMode;
+  stream << "{ Query = " << params.m_query <<
+            ", Locale = " << params.m_inputLocale <<
+            ", Mode = " << params.m_searchMode << " }";
   return stream.str();
 }
 

search/params.hpp

@@ -41,8 +41,8 @@ namespace search
     void SetPosition(double lat, double lon);
     bool IsValidPosition() const { return m_validPos; }
 
-    /// @param[in] language can be "fr", "en-US", "ru_RU" etc.
-    void SetInputLanguage(string const & language) { m_inputLanguage = language; }
+    /// @param[in] locale can be "fr", "en-US", "ru_RU" etc.
+    void SetInputLocale(string const & locale) { m_inputLocale = locale; }
 
     bool IsEqualCommon(SearchParams const & rhs) const;
 
@@ -52,7 +52,7 @@ namespace search
     SearchCallbackT m_callback;
 
     string m_query;
-    string m_inputLanguage;
+    string m_inputLocale;
 
     double m_lat, m_lon;
 

search/search_engine.cpp

@@ -73,7 +73,7 @@ public:
 
 Engine::Engine(IndexType const * pIndex, Reader * pCategoriesR,
                ModelReaderPtr polyR, ModelReaderPtr countryR,
-               string const & lang)
+               string const & locale)
   : m_readyThread(false),
     m_pData(new EngineData(pCategoriesR, polyR, countryR))
 {
@@ -85,7 +85,7 @@ Engine::Engine(IndexType const * pIndex, Reader * pCategoriesR,
                            &m_pData->m_categories,
                            &m_pData->m_stringsToSuggest,
                            &m_pData->m_infoGetter));
-  m_pQuery->SetPreferredLanguage(lang);
+  m_pQuery->SetPreferredLocale(locale);
 }
 
 Engine::~Engine()
@@ -289,7 +289,7 @@ void Engine::SearchAsync(bool viewportPoints)
   m_pQuery->SetSortByViewport(params.IsSortByViewport());
 
   // Language validity is checked inside
-  m_pQuery->SetInputLanguage(params.m_inputLanguage);
+  m_pQuery->SetInputLocale(params.m_inputLocale);
 
   m_pQuery->SetQuery(params.m_query);
   bool const emptyQuery = m_pQuery->IsEmptyQuery();

search/search_engine.hpp

@@ -34,7 +34,7 @@ public:
   // Doesn't take ownership of @pIndex. Takes ownership of pCategories
   Engine(IndexType const * pIndex, Reader * pCategoriesR,
          ModelReaderPtr polyR, ModelReaderPtr countryR,
-         string const & lang);
+         string const & locale);
   ~Engine();
 
   void SupportOldFormat(bool b);

search/search_query.cpp

@@ -18,6 +18,8 @@
 #include "../indexer/categories_holder.hpp"
 #include "../indexer/classificator.hpp"
 
+#include "../platform/preferred_languages.hpp"
+
 #include "../coding/multilang_utf8_string.hpp"
 #include "../coding/reader_wrapper.hpp"
 
@@ -112,7 +114,7 @@ Query::Query(Index const * pIndex,
   langPriorities[3].push_back(StringUtf8Multilang::GetLangIndex("default"));
   m_keywordsScorer.SetLanguages(langPriorities);
 
-  SetPreferredLanguage("en");
+  SetPreferredLocale("en");
 }
 
 Query::~Query()
@@ -196,40 +198,31 @@ void Query::NullPosition()
   m_region.clear();
 }
 
-void Query::SetPreferredLanguage(string const & lang)
+void Query::SetPreferredLocale(string const & locale)
 {
-  // We have normalized language string here.
-  ASSERT(lang.find_first_of("-_") == string::npos, (lang));
-  LOG(LDEBUG, ("Preffered (system) language = ", lang));
+  LOG(LDEBUG, ("New preffered locale:", locale));
 
-  int8_t const code = StringUtf8Multilang::GetLangIndex(lang);
+  int8_t const code = StringUtf8Multilang::GetLangIndex(languages::Normalize(locale));
   SetLanguage(LANG_CURRENT, code);
 
-  m_currentLocaleCode = CategoriesHolder::MapLocaleToInteger(lang);
+  m_currentLocaleCode = CategoriesHolder::MapLocaleToInteger(locale);
 
   // Default initialization.
-  // If you want to reset input language, call SetInputLanguage before search.
-  SetInputLanguage(lang);
+  // If you want to reset input language, call SetInputLocale before search.
+  SetInputLocale(locale);
 
 #ifdef FIND_LOCALITY_TEST
   m_locality.SetLanguage(code);
 #endif
 }
 
-void Query::SetInputLanguage(string const & lang)
+void Query::SetInputLocale(string const & locale)
 {
-  if (!lang.empty())
-  {
-    LOG(LDEBUG, ("New input language = ", lang));
+  LOG(LDEBUG, ("New input locale:", locale));
 
-    // For "data" language we need normalized code.
-    size_t const delimPos = lang.find_first_of("-_");
-    int8_t const code = StringUtf8Multilang::GetLangIndex(
-          delimPos == string::npos ? lang : lang.substr(0, delimPos));
-    SetLanguage(LANG_INPUT, code);
+  SetLanguage(LANG_INPUT, StringUtf8Multilang::GetLangIndex(languages::Normalize(locale)));
 
-    m_inputLocaleCode = CategoriesHolder::MapLocaleToInteger(lang);
-  }
+  m_inputLocaleCode = CategoriesHolder::MapLocaleToInteger(locale);
 }
 
 void Query::ClearCaches()

search/search_query.hpp

@@ -93,8 +93,8 @@ public:
   /// Suggestions language code, not the same as we use in mwm data
   int8_t m_inputLocaleCode, m_currentLocaleCode;
 
-  void SetPreferredLanguage(string const & lang);
-  void SetInputLanguage(string const & lang);
+  void SetPreferredLocale(string const & locale);
+  void SetInputLocale(string const & locale);
 
   void SetQuery(string const & query);
   inline bool IsEmptyQuery() const { return (m_prefix.empty() && m_tokens.empty()); }