From 339f7324bf0667731c2aabb0212beac029cabba9 Mon Sep 17 00:00:00 2001 From: Maxim Pimenov Date: Fri, 1 Dec 2017 14:28:56 +0300 Subject: [PATCH] [coding] An uninterruptible version of ForEach in StringUtf8Multilang. --- .../multilang_utf8_string_test.cpp | 40 ++++++++----- coding/multilang_utf8_string.cpp | 7 ++- coding/multilang_utf8_string.hpp | 60 ++++++++++++------- feature_list/feature_list.cpp | 7 +-- generator/dumper.cpp | 10 +--- .../restaurants_info/restaurants_info.cpp | 6 +- generator/search_index_builder.cpp | 5 +- indexer/editable_map_object.cpp | 11 +--- indexer/feature.cpp | 5 +- .../editable_map_object_test.cpp | 10 +--- search/ranker.cpp | 4 +- 11 files changed, 81 insertions(+), 84 deletions(-) diff --git a/coding/coding_tests/multilang_utf8_string_test.cpp b/coding/coding_tests/multilang_utf8_string_test.cpp index 8773171e8a..874785c4d0 100644 --- a/coding/coding_tests/multilang_utf8_string_test.cpp +++ b/coding/coding_tests/multilang_utf8_string_test.cpp @@ -19,21 +19,6 @@ lang_string gArr[] = {{"default", "default"}, {"ru", "\xD0\xA0\xD0\xB0\xD1\x88\xD0\xBA\xD0\xB0"}, {"be", "\xE2\x82\xAC\xF0\xA4\xAD\xA2"}}; -struct LangChecker -{ - LangChecker() = default; - - base::ControlFlow operator()(char lang, string const & utf8s) - { - TEST_EQUAL(lang, StringUtf8Multilang::GetLangIndex(gArr[m_index].m_lang), ()); - TEST_EQUAL(utf8s, gArr[m_index].m_str, ()); - ++m_index; - return base::ControlFlow::Continue; - } - - size_t m_index = 0; -}; - void TestMultilangString(lang_string const * arr, size_t count) { StringUtf8Multilang s; @@ -75,7 +60,30 @@ UNIT_TEST(MultilangString_ForEach) for (size_t i = 0; i < ARRAY_SIZE(gArr); ++i) s.AddString(gArr[i].m_lang, gArr[i].m_str); - s.ForEach(LangChecker()); + { + size_t index = 0; + s.ForEach([&index](char lang, string const & utf8s) { + TEST_EQUAL(lang, StringUtf8Multilang::GetLangIndex(gArr[index].m_lang), ()); + TEST_EQUAL(utf8s, gArr[index].m_str, ()); + ++index; + }); + TEST_EQUAL(index, ARRAY_SIZE(gArr), ()); + } + + { + size_t index = 0; + vector const expected = {"default", "en", "ru"}; + vector actual; + s.ForEach([&index, &actual](char lang, string const & utf8s) { + actual.push_back(gArr[index].m_lang); + ++index; + if (index == 3) + return base::ControlFlow::Break; + return base::ControlFlow::Continue; + }); + TEST_EQUAL(index, 3, ()); + TEST_EQUAL(actual, expected, ()); + } } UNIT_TEST(MultilangString_Unique) diff --git a/coding/multilang_utf8_string.cpp b/coding/multilang_utf8_string.cpp index 46b1c0fd75..51c5b58357 100644 --- a/coding/multilang_utf8_string.cpp +++ b/coding/multilang_utf8_string.cpp @@ -2,6 +2,8 @@ #include "defines.hpp" +using namespace std; + namespace { // TODO(AlexZ): Review and replace invalid languages which does not map correctly to @@ -213,7 +215,7 @@ int8_t StringUtf8Multilang::FindString(string const & utf8s) const { int8_t result = kUnsupportedLanguageCode; - ForEach([&utf8s, &result](int8_t code, string const & name) -> base::ControlFlow { + ForEach([&utf8s, &result](int8_t code, string const & name) { if (name == utf8s) { result = code; @@ -229,9 +231,8 @@ string DebugPrint(StringUtf8Multilang const & s) { string result; - s.ForEach([&result](int8_t code, string const & name) -> base::ControlFlow { + s.ForEach([&result](int8_t code, string const & name) { result += string(StringUtf8Multilang::GetLangByCode(code)) + string(":") + name + " "; - return base::ControlFlow::Continue; }); return result; diff --git a/coding/multilang_utf8_string.hpp b/coding/multilang_utf8_string.hpp index c016fadc55..e89f9b495d 100644 --- a/coding/multilang_utf8_string.hpp +++ b/coding/multilang_utf8_string.hpp @@ -7,13 +7,14 @@ #include "base/assert.hpp" #include "base/control_flow.hpp" -#include "std/array.hpp" -#include "std/string.hpp" +#include +#include +#include namespace utils { template -void WriteString(TSink & sink, string const & s) +void WriteString(TSink & sink, std::string const & s) { if (EnableExceptions && s.empty()) MYTHROW(Writer::WriteException, ("String is empty")); @@ -26,7 +27,7 @@ void WriteString(TSink & sink, string const & s) } template -void ReadString(TSource & src, string & s) +void ReadString(TSource & src, std::string & s) { uint32_t const sz = ReadVarUint(src) + 1; s.resize(sz); @@ -41,19 +42,7 @@ void ReadString(TSource & src, string & s) class StringUtf8Multilang { - string m_s; - - size_t GetNextIndex(size_t i) const; - public: - static int8_t constexpr kUnsupportedLanguageCode = -1; - static int8_t constexpr kDefaultCode = 0; - static int8_t constexpr kEnglishCode = 1; - static int8_t constexpr kInternationalCode = 7; - /// How many languages we support on indexing stage. See full list in cpp file. - /// TODO(AlexZ): Review and replace invalid languages by valid ones. - static int8_t constexpr kMaxSupportedLanguages = 64; - struct Lang { /// OSM language code (e.g. for name:en it's "en" part). @@ -63,7 +52,16 @@ public: /// Transliterator to latin id. char const * m_transliteratorId; }; - using Languages = array; + + static int8_t constexpr kUnsupportedLanguageCode = -1; + static int8_t constexpr kDefaultCode = 0; + static int8_t constexpr kEnglishCode = 1; + static int8_t constexpr kInternationalCode = 7; + /// How many languages we support on indexing stage. See full list in cpp file. + /// TODO(AlexZ): Review and replace invalid languages by valid ones. + static int8_t constexpr kMaxSupportedLanguages = 64; + + using Languages = std::array; static Languages const & GetSupportedLanguages(); @@ -90,8 +88,13 @@ public: AddString(l, utf8s); } - template - void ForEach(T && fn) const + // Calls |fn| for each pair of |lang| and |utf8s| stored in this multilang string. + // To avoid excessive calls, |fn| may signal the end of execution via its return value. + template + typename enable_if< + is_same::type, base::ControlFlow>::value, + void>::type + ForEach(Fn && fn) const { size_t i = 0; size_t const sz = m_s.size(); @@ -104,6 +107,18 @@ public: } } + // Calls |fn| for each pair of |lang| and |utf8s| stored in this multilang string. + template + typename enable_if::type, void>::value, + void>::type + ForEach(Fn && fn) const + { + ForEach([&](int8_t lang, std::string utf8s) { + fn(lang, utf8s); + return base::ControlFlow::Continue; + }); + } + bool GetString(int8_t lang, string & utf8s) const; bool GetString(string const & lang, string & utf8s) const { @@ -129,6 +144,11 @@ public: { utils::ReadString(src, m_s); } + +private: + size_t GetNextIndex(size_t i) const; + + std::string m_s; }; -string DebugPrint(StringUtf8Multilang const & s); +std::string DebugPrint(StringUtf8Multilang const & s); diff --git a/feature_list/feature_list.cpp b/feature_list/feature_list.cpp index e83aed49f8..b88d062b4e 100644 --- a/feature_list/feature_list.cpp +++ b/feature_list/feature_list.cpp @@ -25,8 +25,6 @@ #include "storage/index.hpp" #include "storage/storage.hpp" -#include "base/control_flow.hpp" - #include "std/algorithm.hpp" #include "std/iostream.hpp" @@ -122,10 +120,7 @@ string BuildUniqueId(ms::LatLon const & coords, string const & name) void AppendNames(FeatureType const & f, vector & columns) { vector names(kLangCount); - f.GetNames().ForEach([&names](int8_t code, string const & name) -> base::ControlFlow { - names[code] = string(name); - return base::ControlFlow::Continue; - }); + f.GetNames().ForEach([&names](int8_t code, string const & name) { names[code] = name; }); columns.insert(columns.end(), next(names.begin()), names.end()); } diff --git a/generator/dumper.cpp b/generator/dumper.cpp index 35b607ed8b..3c1a1f8dae 100644 --- a/generator/dumper.cpp +++ b/generator/dumper.cpp @@ -11,7 +11,6 @@ #include "coding/multilang_utf8_string.hpp" -#include "base/control_flow.hpp" #include "base/logging.hpp" #include @@ -124,7 +123,7 @@ namespace feature public: TokensContainerT m_stats; - base::ControlFlow operator()(int8_t langCode, string const & name) + void operator()(int8_t langCode, string const & name) { CHECK(!name.empty(), ("Feature name is empty")); @@ -133,7 +132,7 @@ namespace feature MakeBackInsertFunctor(tokens), search::Delimiters()); if (tokens.empty()) - return base::ControlFlow::Continue; + return; for (size_t i = 1; i < tokens.size(); ++i) { @@ -148,7 +147,6 @@ namespace feature if (!found.second) found.first->second.first++; } - return base::ControlFlow::Continue; } void operator()(FeatureType & f, uint32_t) @@ -218,13 +216,12 @@ namespace feature void DumpFeatureNames(string const & fPath, string const & lang) { int8_t const langIndex = StringUtf8Multilang::GetLangIndex(lang); - auto printName = [&](int8_t langCode, string const & name) -> base::ControlFlow { + auto printName = [&](int8_t langCode, string const & name) { CHECK(!name.empty(), ("Feature name is empty")); if (langIndex == StringUtf8Multilang::kUnsupportedLanguageCode) cout << StringUtf8Multilang::GetLangByCode(langCode) << ' ' << name << endl; else if (langCode == langIndex) cout << name << endl; - return base::ControlFlow::Continue; }; feature::ForEachFromDat(fPath, [&](FeatureType & f, uint32_t) @@ -232,5 +229,4 @@ namespace feature f.ForEachName(printName); }); } - } // namespace feature diff --git a/generator/restaurants_info/restaurants_info.cpp b/generator/restaurants_info/restaurants_info.cpp index 7414bb7de3..a67d0c3cb5 100644 --- a/generator/restaurants_info/restaurants_info.cpp +++ b/generator/restaurants_info/restaurants_info.cpp @@ -104,15 +104,13 @@ void DumpRestaurants(std::vector const & features, std::ostream std::string defaultName; std::vector translations; multilangName.ForEach( - [&translations, &defaultName](uint8_t const langCode, - std::string const & name) -> base::ControlFlow { + [&translations, &defaultName](uint8_t const langCode, std::string const & name) { if (langCode == StringUtf8Multilang::kDefaultCode) { defaultName = name; - return base::ControlFlow::Continue; + return; } translations.push_back(name); - return base::ControlFlow::Continue; }); auto const center = MercatorBounds::ToLatLon(f.GetKeyPoint()); diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index 168cccbd13..406976ddd4 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -29,7 +29,6 @@ #include "coding/writer.hpp" #include "base/assert.hpp" -#include "base/control_flow.hpp" #include "base/logging.hpp" #include "base/scope_guard.hpp" #include "base/stl_add.hpp" @@ -162,7 +161,7 @@ struct FeatureNameInserter m_keyValuePairs.emplace_back(key, m_val); } - base::ControlFlow operator()(signed char lang, string const & name) const + void operator()(signed char lang, string const & name) const { strings::UniString const uniName = search::NormalizeAndSimplifyString(name); @@ -201,8 +200,6 @@ struct FeatureNameInserter for (auto const & token : tokens) AddToken(lang, token); } - - return base::ControlFlow::Continue; } }; diff --git a/indexer/editable_map_object.cpp b/indexer/editable_map_object.cpp index fdea9701fd..c2e976c808 100644 --- a/indexer/editable_map_object.cpp +++ b/indexer/editable_map_object.cpp @@ -170,8 +170,6 @@ void RemoveFakesFromName(osm::FakeNames const & fakeNames, StringUtf8Multilang & auto const it = find(codesToExclude.begin(), codesToExclude.end(), langCode); if (it == codesToExclude.end()) nameWithoutFakes.AddString(langCode, value); - - return base::ControlFlow::Continue; }); name = nameWithoutFakes; @@ -266,11 +264,10 @@ NamesDataSource EditableMapObject::GetNamesDataSource(StringUtf8Multilang const ++mandatoryCount; // Push other languages. - source.ForEach([&names, mandatoryCount](int8_t const code, - string const & name) -> base::ControlFlow { + source.ForEach([&names, mandatoryCount](int8_t const code, string const & name) { // Exclude default name. if (StringUtf8Multilang::kDefaultCode == code) - return base::ControlFlow::Continue; + return; auto const mandatoryNamesEnd = names.begin() + mandatoryCount; // Exclude languages which are already in container (languages with top priority). @@ -280,8 +277,6 @@ NamesDataSource EditableMapObject::GetNamesDataSource(StringUtf8Multilang const if (mandatoryNamesEnd == it) names.emplace_back(code, name); - - return base::ControlFlow::Continue; }); return result; @@ -549,8 +544,6 @@ void EditableMapObject::RemoveBlankAndDuplicationsForDefault() auto const duplicate = langCode != StringUtf8Multilang::kDefaultCode && defaultName == name; if (!name.empty() && !duplicate) editedName.AddString(langCode, name); - - return base::ControlFlow::Continue; }); m_name = editedName; diff --git a/indexer/feature.cpp b/indexer/feature.cpp index 4bb3b72acd..f52913981d 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -13,7 +13,6 @@ #include "geometry/distance.hpp" #include "geometry/robust_orientation.hpp" -#include "base/control_flow.hpp" #include "base/range_iterator.hpp" #include "base/stl_helpers.hpp" @@ -128,7 +127,6 @@ editor::XMLFeature FeatureType::ToXML(bool serializeType) const ForEachName([&feature](uint8_t const & lang, string const & name) { feature.SetName(lang, name); - return base::ControlFlow::Continue; }); string const house = GetHouseNumber(); @@ -378,10 +376,9 @@ void FeatureType::SetNames(StringUtf8Multilang const & newNames) { m_params.name.Clear(); // Validate passed string to clean up empty names (if any). - newNames.ForEach([this](int8_t langCode, string const & name) -> base::ControlFlow { + newNames.ForEach([this](int8_t langCode, string const & name) { if (!name.empty()) m_params.name.AddString(langCode, name); - return base::ControlFlow::Continue; }); if (m_params.name.IsEmpty()) diff --git a/indexer/indexer_tests/editable_map_object_test.cpp b/indexer/indexer_tests/editable_map_object_test.cpp index f30692ca32..f28591ffa8 100644 --- a/indexer/indexer_tests/editable_map_object_test.cpp +++ b/indexer/indexer_tests/editable_map_object_test.cpp @@ -4,8 +4,6 @@ #include "indexer/classificator_loader.hpp" #include "indexer/editable_map_object.hpp" -#include "base/control_flow.hpp" - namespace { using osm::EditableMapObject; @@ -29,7 +27,7 @@ string DebugPrint(ExpectedName const & expectedName) void CheckExpectations(StringUtf8Multilang const & s, vector const & expectations) { size_t counter = 0; - s.ForEach([&expectations, &counter](int8_t const code, string const & name) -> base::ControlFlow { + s.ForEach([&expectations, &counter](int8_t const code, string const & name) { auto const it = find_if(expectations.begin(), expectations.end(), [&code](ExpectedName const & item) { return GetLangCode(item.m_lang.c_str()) == code; @@ -40,7 +38,6 @@ void CheckExpectations(StringUtf8Multilang const & s, vector const TEST_EQUAL(name, it->m_value, ()); ++counter; - return base::ControlFlow::Continue; }); TEST_EQUAL(counter, expectations.size(), ("Unexpected count of names, expected ", expectations.size(), @@ -581,10 +578,7 @@ UNIT_TEST(EditableMapObject_RemoveBlankNames) { auto const getCountOfNames = [](StringUtf8Multilang const & names) { size_t counter = 0; - names.ForEach([&counter](int8_t const, string const &) -> base::ControlFlow { - ++counter; - return base::ControlFlow::Continue; - }); + names.ForEach([&counter](int8_t const, string const &) { ++counter; }); return counter; }; diff --git a/search/ranker.cpp b/search/ranker.cpp index 87b18e48c1..fde2673363 100644 --- a/search/ranker.cpp +++ b/search/ranker.cpp @@ -11,7 +11,6 @@ #include "indexer/feature_algo.hpp" #include "indexer/search_string_utils.hpp" -#include "base/control_flow.hpp" #include "base/logging.hpp" #include "base/string_utils.hpp" @@ -515,14 +514,13 @@ void Ranker::MakeRankerResults(Geocoder::Params const & geocoderParams, void Ranker::GetBestMatchName(FeatureType const & f, string & name) const { KeywordLangMatcher::Score bestScore; - auto bestNameFinder = [&](int8_t lang, string const & s) -> base::ControlFlow { + auto bestNameFinder = [&](int8_t lang, string const & s) { auto const score = m_keywordsScorer.CalcScore(lang, s); if (bestScore < score) { bestScore = score; name = s; } - return base::ControlFlow::Continue; }; UNUSED_VALUE(f.ForEachName(bestNameFinder)); }