pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] locales for all objects

Browse source
This commit is contained in:
LaGrunge 2019-07-03 17:34:16 +03:00
parent 92d788c4e2
commit 43c6c88da2
9 changed files with 87 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
coding/string_utf8_multilang.hpp
View file

@ -10,6 +10,7 @@
#include <cstddef>
#include <cstdint>
#include <functional>
#include <string>
#include <utility>
@ -63,6 +64,14 @@ void ReadString(TSource & src, std::string & s)
class StringUtf8Multilang
{
public:
struct Hash
{
std::size_t operator()(StringUtf8Multilang const & s) const noexcept
{
return std::hash<std::string>{}(s.m_s);
}
};
struct Lang
{
/// OSM language code (e.g. for name:en it's "en" part).

14
generator/feature_builder.hpp
View file

@ -150,6 +150,7 @@ public:
FeatureParams const & GetParams() const { return m_params; }
FeatureParams & GetParams() { return m_params; }
std::string GetName(int8_t lang = StringUtf8Multilang::kDefaultCode) const;
StringUtf8Multilang GetMultilangName() const { return m_params.name; }
uint8_t GetRank() const { return m_params.rank; }
bool FormatFullAddress(std::string & res) const;
AddressData const & GetAddressData() const { return m_params.GetAddressData(); }
@ -175,7 +176,8 @@ public:
bool PreSerializeAndRemoveUselessNamesForIntermediate();
void SerializeForIntermediate(Buffer & data) const;
void SerializeBorderForIntermediate(serial::GeometryCodingParams const & params, Buffer & data) const;
void SerializeBorderForIntermediate(serial::GeometryCodingParams const & params,
Buffer & data) const;
void DeserializeFromIntermediate(Buffer & data);
bool PreSerializeAndRemoveUselessNamesForMwm(SupportingData const & data);
@ -201,13 +203,13 @@ public:
void SetCoastCell(int64_t iCell) { m_coastCell = iCell; }
bool IsCoastCell() const { return (m_coastCell != -1); }
protected:
protected:
template <class ToDo>
class ToDoWrapper
{
public:
ToDoWrapper(ToDo && toDo) : m_toDo(std::forward<ToDo>(toDo)) {}
bool operator() (m2::PointD const & p) { return m_toDo(p); }
bool operator()(m2::PointD const & p) { return m_toDo(p); }
void EndRegion() {}
private:
@ -218,9 +220,9 @@ public:
// - point in point-feature
// - origin point of text [future] in line-feature
// - origin point of text or symbol in area-feature
m2::PointD m_center; // Check HEADER_HAS_POINT
m2::PointD m_center; // Check HEADER_HAS_POINT
// List of geometry polygons.
Geometry m_polygons; // Check HEADER_IS_AREA
Geometry m_polygons; // Check HEADER_IS_AREA
m2::RectD m_limitRect;
std::vector<base::GeoObjectId> m_osmIds;
FeatureParams m_params;
@ -264,7 +266,7 @@ void ForEachFromDatRawFormat(std::string const & filename, ToDo && toDo)
/// Parallel process features in .dat file.
template <class ToDo>
void ForEachParallelFromDatRawFormat(size_t threadsCount, std::string const & filename,
ToDo && toDo)
ToDo && toDo)
{
CHECK_GREATER_OR_EQUAL(threadsCount, 1, ());
if (threadsCount == 1)

3
generator/generator_tests/translation_test.cpp
View file

@ -15,8 +15,7 @@ using namespace generator;
// Transliteration tests ---------------------------------------------------------------------------
using Translations = std::vector<std::pair<std::string, std::string>>;
bool TestTransliteration(Translations const & translations,
std::string const & expectedTransliteration,
std::string const & lang)
std::string const & expectedTransliteration, std::string const & lang)
{
StringUtf8Multilang name;
for (auto const & langAndTranslation : translations)

3
generator/geo_objects/geo_objects.cpp
View file

@ -101,7 +101,8 @@ base::JSONPtr MakeGeoObjectValueWithoutAddress(FeatureBuilder const & fb, JsonVa
{
auto jsonWithAddress = json.MakeDeepCopyJson();
auto properties = json_object_get(jsonWithAddress.get(), "properties");
ToJSONObject(*properties, "name", fb.GetName());
Localizator localizator(*properties);
localizator.AddLocale("name", Localizator::EasyObjectWithTranslation(fb.GetMultilangName()));
UpdateCoordinates(fb.GetKeyPoint(), jsonWithAddress);
return jsonWithAddress;
}

9
generator/regions/regions.cpp
View file

@ -102,7 +102,9 @@ private:
ToJSONArray(*coordinates, center.m_lat);
ToJSONObject(*geometry, "coordinates", coordinates);
Localizator localizator;
auto properties = base::NewJSONObject();
Localizator localizator(*properties);
boost::optional<std::string> dref;
for (auto const & p : path)
@ -128,12 +130,7 @@ private:
}
localizator.AddLocale("name", main);
auto properties = base::NewJSONObject();
auto locales = localizator.BuildLocales();
ToJSONObject(*properties, "rank", main.GetRank());
ToJSONObject(*properties, "locales", locales);
if (dref)
ToJSONObject(*properties, "dref", *dref);

38
generator/streets/streets_builder.cpp
View file

@ -1,6 +1,8 @@
#include "generator/streets/streets_builder.hpp"
#include "generator/key_value_storage.hpp"
#include "generator/streets/street_regions_tracing.hpp"
#include "generator/translation.hpp"
#include "indexer/classificator.hpp"
#include "indexer/ftypes_matcher.hpp"
@ -33,9 +35,14 @@ void StreetsBuilder::AssembleStreets(std::string const & pathInStreetsTmpMwm)
void StreetsBuilder::AssembleBindings(std::string const & pathInGeoObjectsTmpMwm)
{
auto const transform = [this](FeatureBuilder & fb, uint64_t /* currPos */) {
auto streetName = fb.GetParams().GetStreet();
std::string streetName = fb.GetParams().GetStreet();
if (!streetName.empty())
AddStreetBinding(std::move(streetName), fb);
{
// TODO(lagrunge): add localizations on street:lang tags
StringUtf8Multilang multilangStreetName;
multilangStreetName.AddString(StringUtf8Multilang::kDefaultCode, streetName);
AddStreetBinding(std::move(multilangStreetName), fb);
}
};
ForEachParallelFromDatRawFormat(m_threadsCount, pathInGeoObjectsTmpMwm, transform);
}
@ -90,7 +97,7 @@ void StreetsBuilder::AddStreetHighway(FeatureBuilder & fb)
for (auto & segment : pathSegments)
{
auto && region = segment.m_region;
auto & street = InsertStreet(region.first, fb.GetName());
auto & street = InsertStreet(region.first, fb.GetMultilangName());
auto const osmId = pathSegments.size() == 1 ? fb.GetMostGenericOsmId() : NextOsmSurrogateId();
street.AddHighwayLine(osmId, std::move(segment.m_path));
}
@ -104,7 +111,7 @@ void StreetsBuilder::AddStreetArea(FeatureBuilder & fb)
std::lock_guard<std::mutex> lock{m_updateMutex};
auto & street = InsertStreet(region->first, fb.GetName());
auto & street = InsertStreet(region->first, fb.GetMultilangName());
auto osmId = fb.GetMostGenericOsmId();
street.AddHighwayArea(osmId, fb.GetOuterGeometry());
}
@ -118,11 +125,11 @@ void StreetsBuilder::AddStreetPoint(FeatureBuilder & fb)
std::lock_guard<std::mutex> lock{m_updateMutex};
auto osmId = fb.GetMostGenericOsmId();
auto & street = InsertStreet(region->first, fb.GetName());
auto & street = InsertStreet(region->first, fb.GetMultilangName());
street.SetPin({fb.GetKeyPoint(), osmId});
}
void StreetsBuilder::AddStreetBinding(std::string && streetName, FeatureBuilder & fb)
void StreetsBuilder::AddStreetBinding(StringUtf8Multilang && streetName, FeatureBuilder & fb)
{
auto const region = FindStreetRegionOwner(fb.GetKeyPoint());
if (!region)
@ -155,26 +162,29 @@ boost::optional<KeyValue> StreetsBuilder::FindStreetRegionOwner(m2::PointD const
return m_regionInfoGetter.FindDeepest(point, isStreetAdministrator);
}
StreetGeometry & StreetsBuilder::InsertStreet(uint64_t regionId, std::string && streetName)
StreetGeometry & StreetsBuilder::InsertStreet(uint64_t regionId, StringUtf8Multilang && streetName)
{
auto & regionStreets = m_regions[regionId];
return regionStreets[std::move(streetName)];
}
base::JSONPtr StreetsBuilder::MakeStreetValue(uint64_t regionId, JsonValue const & regionObject,
std::string const & streetName,
StringUtf8Multilang const & streetName,
m2::RectD const & bbox, m2::PointD const & pinPoint)
{
auto streetObject = base::NewJSONObject();
auto && regionAddress = base::GetJSONObligatoryFieldByPath(regionObject, "properties", "locales",
"default", "address");
auto && regionLocales = base::GetJSONObligatoryFieldByPath(regionObject, "properties", "locales");
auto address = base::JSONPtr{json_deep_copy(const_cast<json_t *>(regionAddress))};
ToJSONObject(*address, "street", streetName);
auto locales = base::JSONPtr{json_deep_copy(const_cast<json_t *>(regionLocales))};
auto properties = base::NewJSONObject();
ToJSONObject(*properties, "address", std::move(address));
ToJSONObject(*properties, "name", streetName);
ToJSONObject(*properties, "locales", std::move(locales));
Localizator localizator(*properties);
auto const & localizee = Localizator::EasyObjectWithTranslation(streetName);
localizator.AddLocale("name", localizee);
localizator.AddLocale("street", localizee, "address");
ToJSONObject(*properties, "dref", KeyValueStorage::SerializeDref(regionId));
ToJSONObject(*streetObject, "properties", std::move(properties));

9
generator/streets/streets_builder.hpp
View file

@ -41,7 +41,8 @@ public:
static bool IsStreet(feature::FeatureBuilder const & fb);
private:
using RegionStreets = std::unordered_map<std::string, StreetGeometry>;
using RegionStreets =
std::unordered_map<StringUtf8Multilang, StreetGeometry, StringUtf8Multilang::Hash>;
void SaveRegionStreetsKv(std::ostream & streamStreetsKv, uint64_t regionId,
RegionStreets const & streets);
@ -50,12 +51,12 @@ private:
void AddStreetHighway(feature::FeatureBuilder & fb);
void AddStreetArea(feature::FeatureBuilder & fb);
void AddStreetPoint(feature::FeatureBuilder & fb);
void AddStreetBinding(std::string && streetName, feature::FeatureBuilder & fb);
void AddStreetBinding(StringUtf8Multilang && streetName, feature::FeatureBuilder & fb);
boost::optional<KeyValue> FindStreetRegionOwner(m2::PointD const & point,
bool needLocality = false);
StreetGeometry & InsertStreet(uint64_t regionId, std::string && streetName);
StreetGeometry & InsertStreet(uint64_t regionId, StringUtf8Multilang && streetName);
base::JSONPtr MakeStreetValue(uint64_t regionId, JsonValue const & regionObject,
std::string const & streetName, m2::RectD const & bbox,
const StringUtf8Multilang & streetName, m2::RectD const & bbox,
m2::PointD const & pinPoint);
base::GeoObjectId NextOsmSurrogateId();

3
generator/translation.cpp
View file

@ -37,7 +37,8 @@ std::string GetTranslatedOrTransliteratedName(StringUtf8Multilang const & name,
return s;
auto const fn = [&s](int8_t code, std::string const & name) {
if (strings::IsASCIIString(name)) {
if (strings::IsASCIIString(name))
{
s = name;
return base::ControlFlow::Break;
}

66
generator/translation.hpp
View file

@ -31,8 +31,28 @@ std::string GetTranslatedOrTransliteratedName(StringUtf8Multilang const & name,
class Localizator
{
public:
class EasyObjectWithTranslation
{
public:
explicit EasyObjectWithTranslation(StringUtf8Multilang const & name) : m_name(name) {}
std::string GetTranslatedOrTransliteratedName(LanguageCode languageCode) const
{
return ::generator::GetTranslatedOrTransliteratedName(m_name, languageCode);
}
std::string GetName(LanguageCode languageCode = StringUtf8Multilang::kDefaultCode) const
{
return ::generator::GetName(m_name, languageCode);
}
private:
StringUtf8Multilang const m_name;
};
explicit Localizator(json_t & node) : m_node(GetOrCreateNode("locales", node)) {}
template <class Object>
void AddLocale(std::string const & label, Object objectWithName,
void AddLocale(std::string const & label, Object const & objectWithName,
std::string const & level = std::string())
{
AddLocale(DefaultLocaleName(), level, objectWithName.GetName(), label);
@ -53,30 +73,20 @@ public:
template <class Verboser>
void AddVerbose(Verboser && verboser, std::string const & level)
{
json_t & locale = GetLocale(DefaultLocaleName());
json_t & node = GetLevel(level, locale);
json_t & locale = GetOrCreateNode(DefaultLocaleName(), m_node);
json_t & node = GetOrCreateNode(level, locale);
verboser(node);
}
base::JSONPtr BuildLocales()
{
auto locales = base::NewJSONObject();
for (auto & localeWithLanguage : m_localesByLanguages)
ToJSONObject(*locales, localeWithLanguage.first, *localeWithLanguage.second);
m_localesByLanguages.clear();
return locales;
}
private:
void AddLocale(std::string const & language, std::string const & level, std::string const & name,
std::string const & label)
{
json_t & locale = GetLocale(language);
json_t & locale = GetOrCreateNode(language, m_node);
if (!level.empty())
{
json_t & levelNode = GetLevel(level, locale);
json_t & levelNode = GetOrCreateNode(level, locale);
ToJSONObject(levelNode, label, name);
}
else
@ -90,29 +100,19 @@ private:
return kDefaultLocaleName;
}
json_t & GetLocale(std::string const & language)
static json_t & GetOrCreateNode(std::string const & nodeName, json_t & root)
{
if (m_localesByLanguages.find(language) == m_localesByLanguages.end())
m_localesByLanguages[language] = json_object();
return *m_localesByLanguages.at(language);
}
static json_t & GetLevel(std::string const & level, json_t & locale)
{
json_t * levelNode = base::GetJSONOptionalField(&locale, level);
if (!levelNode)
json_t * node = base::GetJSONOptionalField(&root, nodeName);
if (!node || base::JSONIsNull(node))
{
levelNode = json_object();
ToJSONObject(locale, level, *levelNode);
node = json_object();
ToJSONObject(root, nodeName, *node);
}
return *levelNode;
return *node;
}
std::vector<std::string> const & LocaleLanguages() const;
using LocalesByLanguages = std::map<std::string, json_t *>;
LocalesByLanguages m_localesByLanguages;
std::vector<std::string> const & LocaleLanguages() const;
json_t & m_node;
};
} // namespace generator