AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[coding] An uninterruptible version of ForEach in StringUtf8Multilang.

Browse source
This commit is contained in:
Maxim Pimenov 2017-12-01 14:28:56 +03:00 • committed by Vladimir Byko-Ianko
parent 6595b3e342
commit 339f7324bf
11 changed files with 81 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
coding/coding_tests/multilang_utf8_string_test.cpp
View file

@ -19,21 +19,6 @@ lang_string gArr[] = {{"default", "default"},
{"ru", "\xD0\xA0\xD0\xB0\xD1\x88\xD0\xBA\xD0\xB0"},
{"be", "\xE2\x82\xAC\xF0\xA4\xAD\xA2"}};
struct LangChecker
{
LangChecker() = default;
base::ControlFlow operator()(char lang, string const & utf8s)
{
TEST_EQUAL(lang, StringUtf8Multilang::GetLangIndex(gArr[m_index].m_lang), ());
TEST_EQUAL(utf8s, gArr[m_index].m_str, ());
++m_index;
return base::ControlFlow::Continue;
}
size_t m_index = 0;
};
void TestMultilangString(lang_string const * arr, size_t count)
{
StringUtf8Multilang s;
@ -75,7 +60,30 @@ UNIT_TEST(MultilangString_ForEach)
for (size_t i = 0; i < ARRAY_SIZE(gArr); ++i)
s.AddString(gArr[i].m_lang, gArr[i].m_str);
s.ForEach(LangChecker());
{
size_t index = 0;
s.ForEach([&index](char lang, string const & utf8s) {
TEST_EQUAL(lang, StringUtf8Multilang::GetLangIndex(gArr[index].m_lang), ());
TEST_EQUAL(utf8s, gArr[index].m_str, ());
++index;
});
TEST_EQUAL(index, ARRAY_SIZE(gArr), ());
}
{
size_t index = 0;
vector<string> const expected = {"default", "en", "ru"};
vector<string> actual;
s.ForEach([&index, &actual](char lang, string const & utf8s) {
actual.push_back(gArr[index].m_lang);
++index;
if (index == 3)
return base::ControlFlow::Break;
return base::ControlFlow::Continue;
});
TEST_EQUAL(index, 3, ());
TEST_EQUAL(actual, expected, ());
}
}
UNIT_TEST(MultilangString_Unique)

7
coding/multilang_utf8_string.cpp
View file

@ -2,6 +2,8 @@
#include "defines.hpp"
using namespace std;
namespace
{
// TODO(AlexZ): Review and replace invalid languages which does not map correctly to
@ -213,7 +215,7 @@ int8_t StringUtf8Multilang::FindString(string const & utf8s) const
{
int8_t result = kUnsupportedLanguageCode;
ForEach([&utf8s, &result](int8_t code, string const & name) -> base::ControlFlow {
ForEach([&utf8s, &result](int8_t code, string const & name) {
if (name == utf8s)
{
result = code;
@ -229,9 +231,8 @@ string DebugPrint(StringUtf8Multilang const & s)
{
string result;
s.ForEach([&result](int8_t code, string const & name) -> base::ControlFlow {
s.ForEach([&result](int8_t code, string const & name) {
result += string(StringUtf8Multilang::GetLangByCode(code)) + string(":") + name + " ";
return base::ControlFlow::Continue;
});
return result;

60
coding/multilang_utf8_string.hpp
View file

@ -7,13 +7,14 @@
#include "base/assert.hpp"
#include "base/control_flow.hpp"
#include "std/array.hpp"
#include "std/string.hpp"
#include <array>
#include <string>
#include <type_traits>
namespace utils
{
template <class TSink, bool EnableExceptions = false>
void WriteString(TSink & sink, string const & s)
void WriteString(TSink & sink, std::string const & s)
{
if (EnableExceptions && s.empty())
MYTHROW(Writer::WriteException, ("String is empty"));
@ -26,7 +27,7 @@ void WriteString(TSink & sink, string const & s)
}
template <class TSource, bool EnableExceptions = false>
void ReadString(TSource & src, string & s)
void ReadString(TSource & src, std::string & s)
{
uint32_t const sz = ReadVarUint<uint32_t>(src) + 1;
s.resize(sz);
@ -41,19 +42,7 @@ void ReadString(TSource & src, string & s)
class StringUtf8Multilang
{
string m_s;
size_t GetNextIndex(size_t i) const;
public:
static int8_t constexpr kUnsupportedLanguageCode = -1;
static int8_t constexpr kDefaultCode = 0;
static int8_t constexpr kEnglishCode = 1;
static int8_t constexpr kInternationalCode = 7;
/// How many languages we support on indexing stage. See full list in cpp file.
/// TODO(AlexZ): Review and replace invalid languages by valid ones.
static int8_t constexpr kMaxSupportedLanguages = 64;
struct Lang
{
/// OSM language code (e.g. for name:en it's "en" part).
@ -63,7 +52,16 @@ public:
/// Transliterator to latin id.
char const * m_transliteratorId;
};
using Languages = array<Lang, kMaxSupportedLanguages>;
static int8_t constexpr kUnsupportedLanguageCode = -1;
static int8_t constexpr kDefaultCode = 0;
static int8_t constexpr kEnglishCode = 1;
static int8_t constexpr kInternationalCode = 7;
/// How many languages we support on indexing stage. See full list in cpp file.
/// TODO(AlexZ): Review and replace invalid languages by valid ones.
static int8_t constexpr kMaxSupportedLanguages = 64;
using Languages = std::array<Lang, kMaxSupportedLanguages>;
static Languages const & GetSupportedLanguages();
@ -90,8 +88,13 @@ public:
AddString(l, utf8s);
}
template <class T>
void ForEach(T && fn) const
// Calls |fn| for each pair of |lang| and |utf8s| stored in this multilang string.
// To avoid excessive calls, |fn| may signal the end of execution via its return value.
template <typename Fn>
typename enable_if<
is_same<typename result_of<Fn(int8_t, std::string)>::type, base::ControlFlow>::value,
void>::type
ForEach(Fn && fn) const
{
size_t i = 0;
size_t const sz = m_s.size();
@ -104,6 +107,18 @@ public:
}
}
// Calls |fn| for each pair of |lang| and |utf8s| stored in this multilang string.
template <typename Fn>
typename enable_if<is_same<typename result_of<Fn(int8_t, std::string)>::type, void>::value,
void>::type
ForEach(Fn && fn) const
{
ForEach([&](int8_t lang, std::string utf8s) {
fn(lang, utf8s);
return base::ControlFlow::Continue;
});
}
bool GetString(int8_t lang, string & utf8s) const;
bool GetString(string const & lang, string & utf8s) const
{
@ -129,6 +144,11 @@ public:
{
utils::ReadString(src, m_s);
}
private:
size_t GetNextIndex(size_t i) const;
std::string m_s;
};
string DebugPrint(StringUtf8Multilang const & s);
std::string DebugPrint(StringUtf8Multilang const & s);

7
feature_list/feature_list.cpp
View file

@ -25,8 +25,6 @@
#include "storage/index.hpp"
#include "storage/storage.hpp"
#include "base/control_flow.hpp"
#include "std/algorithm.hpp"
#include "std/iostream.hpp"
@ -122,10 +120,7 @@ string BuildUniqueId(ms::LatLon const & coords, string const & name)
void AppendNames(FeatureType const & f, vector<string> & columns)
{
vector<string> names(kLangCount);
f.GetNames().ForEach([&names](int8_t code, string const & name) -> base::ControlFlow {
names[code] = string(name);
return base::ControlFlow::Continue;
});
f.GetNames().ForEach([&names](int8_t code, string const & name) { names[code] = name; });
columns.insert(columns.end(), next(names.begin()), names.end());
}

10
generator/dumper.cpp
View file

@ -11,7 +11,6 @@
#include "coding/multilang_utf8_string.hpp"
#include "base/control_flow.hpp"
#include "base/logging.hpp"
#include <algorithm>
@ -124,7 +123,7 @@ namespace feature
public:
TokensContainerT m_stats;
base::ControlFlow operator()(int8_t langCode, string const & name)
void operator()(int8_t langCode, string const & name)
{
CHECK(!name.empty(), ("Feature name is empty"));
@ -133,7 +132,7 @@ namespace feature
MakeBackInsertFunctor(tokens), search::Delimiters());
if (tokens.empty())
return base::ControlFlow::Continue;
return;
for (size_t i = 1; i < tokens.size(); ++i)
{
@ -148,7 +147,6 @@ namespace feature
if (!found.second)
found.first->second.first++;
}
return base::ControlFlow::Continue;
}
void operator()(FeatureType & f, uint32_t)
@ -218,13 +216,12 @@ namespace feature
void DumpFeatureNames(string const & fPath, string const & lang)
{
int8_t const langIndex = StringUtf8Multilang::GetLangIndex(lang);
auto printName = [&](int8_t langCode, string const & name) -> base::ControlFlow {
auto printName = [&](int8_t langCode, string const & name) {
CHECK(!name.empty(), ("Feature name is empty"));
if (langIndex == StringUtf8Multilang::kUnsupportedLanguageCode)
cout << StringUtf8Multilang::GetLangByCode(langCode) << ' ' << name << endl;
else if (langCode == langIndex)
cout << name << endl;
return base::ControlFlow::Continue;
};
feature::ForEachFromDat(fPath, [&](FeatureType & f, uint32_t)
@ -232,5 +229,4 @@ namespace feature
f.ForEachName(printName);
});
}
} // namespace feature

6
generator/restaurants_info/restaurants_info.cpp
View file

@ -104,15 +104,13 @@ void DumpRestaurants(std::vector<FeatureBuilder1> const & features, std::ostream
std::string defaultName;
std::vector<std::string> translations;
multilangName.ForEach(
[&translations, &defaultName](uint8_t const langCode,
std::string const & name) -> base::ControlFlow {
[&translations, &defaultName](uint8_t const langCode, std::string const & name) {
if (langCode == StringUtf8Multilang::kDefaultCode)
{
defaultName = name;
return base::ControlFlow::Continue;
return;
}
translations.push_back(name);
return base::ControlFlow::Continue;
});
auto const center = MercatorBounds::ToLatLon(f.GetKeyPoint());

5
generator/search_index_builder.cpp
View file

@ -29,7 +29,6 @@
#include "coding/writer.hpp"
#include "base/assert.hpp"
#include "base/control_flow.hpp"
#include "base/logging.hpp"
#include "base/scope_guard.hpp"
#include "base/stl_add.hpp"
@ -162,7 +161,7 @@ struct FeatureNameInserter
m_keyValuePairs.emplace_back(key, m_val);
}
base::ControlFlow operator()(signed char lang, string const & name) const
void operator()(signed char lang, string const & name) const
{
strings::UniString const uniName = search::NormalizeAndSimplifyString(name);
@ -201,8 +200,6 @@ struct FeatureNameInserter
for (auto const & token : tokens)
AddToken(lang, token);
}
return base::ControlFlow::Continue;
}
};

11
indexer/editable_map_object.cpp
View file

@ -170,8 +170,6 @@ void RemoveFakesFromName(osm::FakeNames const & fakeNames, StringUtf8Multilang &
auto const it = find(codesToExclude.begin(), codesToExclude.end(), langCode);
if (it == codesToExclude.end())
nameWithoutFakes.AddString(langCode, value);
return base::ControlFlow::Continue;
});
name = nameWithoutFakes;
@ -266,11 +264,10 @@ NamesDataSource EditableMapObject::GetNamesDataSource(StringUtf8Multilang const
++mandatoryCount;
// Push other languages.
source.ForEach([&names, mandatoryCount](int8_t const code,
string const & name) -> base::ControlFlow {
source.ForEach([&names, mandatoryCount](int8_t const code, string const & name) {
// Exclude default name.
if (StringUtf8Multilang::kDefaultCode == code)
return base::ControlFlow::Continue;
return;
auto const mandatoryNamesEnd = names.begin() + mandatoryCount;
// Exclude languages which are already in container (languages with top priority).
@ -280,8 +277,6 @@ NamesDataSource EditableMapObject::GetNamesDataSource(StringUtf8Multilang const
if (mandatoryNamesEnd == it)
names.emplace_back(code, name);
return base::ControlFlow::Continue;
});
return result;
@ -549,8 +544,6 @@ void EditableMapObject::RemoveBlankAndDuplicationsForDefault()
auto const duplicate = langCode != StringUtf8Multilang::kDefaultCode && defaultName == name;
if (!name.empty() && !duplicate)
editedName.AddString(langCode, name);
return base::ControlFlow::Continue;
});
m_name = editedName;

5
indexer/feature.cpp
View file

@ -13,7 +13,6 @@
#include "geometry/distance.hpp"
#include "geometry/robust_orientation.hpp"
#include "base/control_flow.hpp"
#include "base/range_iterator.hpp"
#include "base/stl_helpers.hpp"
@ -128,7 +127,6 @@ editor::XMLFeature FeatureType::ToXML(bool serializeType) const
ForEachName([&feature](uint8_t const & lang, string const & name)
{
feature.SetName(lang, name);
return base::ControlFlow::Continue;
});
string const house = GetHouseNumber();
@ -378,10 +376,9 @@ void FeatureType::SetNames(StringUtf8Multilang const & newNames)
{
m_params.name.Clear();
// Validate passed string to clean up empty names (if any).
newNames.ForEach([this](int8_t langCode, string const & name) -> base::ControlFlow {
newNames.ForEach([this](int8_t langCode, string const & name) {
if (!name.empty())
m_params.name.AddString(langCode, name);
return base::ControlFlow::Continue;
});
if (m_params.name.IsEmpty())

10
indexer/indexer_tests/editable_map_object_test.cpp
View file

@ -4,8 +4,6 @@
#include "indexer/classificator_loader.hpp"
#include "indexer/editable_map_object.hpp"
#include "base/control_flow.hpp"
namespace
{
using osm::EditableMapObject;
@ -29,7 +27,7 @@ string DebugPrint(ExpectedName const & expectedName)
void CheckExpectations(StringUtf8Multilang const & s, vector<ExpectedName> const & expectations)
{
size_t counter = 0;
s.ForEach([&expectations, &counter](int8_t const code, string const & name) -> base::ControlFlow {
s.ForEach([&expectations, &counter](int8_t const code, string const & name) {
auto const it = find_if(expectations.begin(), expectations.end(), [&code](ExpectedName const & item)
{
return GetLangCode(item.m_lang.c_str()) == code;
@ -40,7 +38,6 @@ void CheckExpectations(StringUtf8Multilang const & s, vector<ExpectedName> const
TEST_EQUAL(name, it->m_value, ());
++counter;
return base::ControlFlow::Continue;
});
TEST_EQUAL(counter, expectations.size(), ("Unexpected count of names, expected ", expectations.size(),
@ -581,10 +578,7 @@ UNIT_TEST(EditableMapObject_RemoveBlankNames)
{
auto const getCountOfNames = [](StringUtf8Multilang const & names) {
size_t counter = 0;
names.ForEach([&counter](int8_t const, string const &) -> base::ControlFlow {
++counter;
return base::ControlFlow::Continue;
});
names.ForEach([&counter](int8_t const, string const &) { ++counter; });
return counter;
};

4
search/ranker.cpp
View file

@ -11,7 +11,6 @@
#include "indexer/feature_algo.hpp"
#include "indexer/search_string_utils.hpp"
#include "base/control_flow.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
@ -515,14 +514,13 @@ void Ranker::MakeRankerResults(Geocoder::Params const & geocoderParams,
void Ranker::GetBestMatchName(FeatureType const & f, string & name) const
{
KeywordLangMatcher::Score bestScore;
auto bestNameFinder = [&](int8_t lang, string const & s) -> base::ControlFlow {
auto bestNameFinder = [&](int8_t lang, string const & s) {
auto const score = m_keywordsScorer.CalcScore(lang, s);
if (bestScore < score)
{
bestScore = score;
name = s;
}
return base::ControlFlow::Continue;
};
UNUSED_VALUE(f.ForEachName(bestNameFinder));
}