diff --git a/coding/coding_tests/string_utf8_multilang_tests.cpp b/coding/coding_tests/string_utf8_multilang_tests.cpp index c47e220d87..dced8c0cfe 100644 --- a/coding/coding_tests/string_utf8_multilang_tests.cpp +++ b/coding/coding_tests/string_utf8_multilang_tests.cpp @@ -252,3 +252,16 @@ UNIT_TEST(MultilangString_RemoveString) testRemove(strings, {0, 1, 2, 9, 17, 27, 37}); testRemove(strings, {39}); } + +UNIT_TEST(MultilangString_Buffers) +{ + StringUtf8Multilang s; + s.AddString(StringUtf8Multilang::kInternationalCode, "blabla"); + + StringUtf8Multilang const ss = StringUtf8Multilang::FromBuffer(std::string(s.GetBuffer())); + + std::string test; + TEST_EQUAL(ss.CountLangs(), 1, ()); + TEST(ss.GetString(StringUtf8Multilang::kInternationalCode, test), ()); + TEST_EQUAL(test, "blabla", ()); +} diff --git a/coding/string_utf8_multilang.cpp b/coding/string_utf8_multilang.cpp index 1ecc70e94d..bb3829d2cd 100644 --- a/coding/string_utf8_multilang.cpp +++ b/coding/string_utf8_multilang.cpp @@ -12,8 +12,8 @@ namespace // several data releases. // Note that it's not feasible to increase languages number here due to current encoding (6 bit to // store language code). -array const kLanguages = { - {{"default", "Native for each country", {"Any-Latin"}}, +array const kLanguages = {{ + {"default", "Native for each country", {"Any-Latin"}}, {"en", "English", {}}, {"ja", "日本語", {}}, {"fr", "Français", {}}, @@ -76,7 +76,8 @@ array co {"mn", "Mongolian", {"Mongolian-Latin/BGN"}}, {"mk", "Македонски", {"Macedonian-Latin/BGN"}}, {"lv", "Latviešu", {}}, - {"hi", "हिन्दी", {"Any-Latin"}}}}; + {"hi", "हिन्दी", {"Any-Latin"}} +}}; static_assert( kLanguages.size() == StringUtf8Multilang::kMaxSupportedLanguages, @@ -323,3 +324,12 @@ string DebugPrint(StringUtf8Multilang const & s) return result; } + +StringUtf8Multilang StringUtf8Multilang::FromBuffer(std::string && s) +{ + ASSERT(!s.empty(), ()); + StringUtf8Multilang res; + res.m_s = std::move(s); + ASSERT_GREATER(res.CountLangs(), 0, ()); + return res; +} diff --git a/coding/string_utf8_multilang.hpp b/coding/string_utf8_multilang.hpp index 108257a8b3..11a1c90887 100644 --- a/coding/string_utf8_multilang.hpp +++ b/coding/string_utf8_multilang.hpp @@ -181,7 +181,7 @@ public: } } return false; - }; + } bool GetString(int8_t lang, std::string & utf8s) const; bool GetString(std::string const & lang, std::string & utf8s) const @@ -198,6 +198,8 @@ public: int8_t FindString(std::string const & utf8s) const; size_t CountLangs() const; + /// @name Used for serdes. + /// @{ template void Write(TSink & sink) const { @@ -210,6 +212,10 @@ public: utils::ReadString(src, m_s); } + std::string const & GetBuffer() const { return m_s; } + static StringUtf8Multilang FromBuffer(std::string && s); + /// @} + private: TranslationPositions GenerateTranslationPositions() const; std::string GetTranslation(Position const & position) const; diff --git a/generator/generator_tests/osm_type_test.cpp b/generator/generator_tests/osm_type_test.cpp index 1135229fcd..17cc6a512d 100644 --- a/generator/generator_tests/osm_type_test.cpp +++ b/generator/generator_tests/osm_type_test.cpp @@ -1347,6 +1347,26 @@ UNIT_CLASS_TEST(TestWithClassificator, OsmType_Recycling) } } +UNIT_CLASS_TEST(TestWithClassificator, OsmType_Metadata) +{ + { + Tags const tags = { + {"amenity", "restaurant" }, + {"description:ru", "Хорошие настойки"}, + }; + + auto const params = GetFeatureBuilderParams(tags); + + TEST_EQUAL(params.m_types.size(), 1, (params)); + TEST(params.IsTypeExist(GetType({"amenity", "restaurant"})), (params)); + + std::string buffer, desc; + TEST(params.GetMetadata().Get(feature::Metadata::FMD_DESCRIPTION, buffer), ()); + StringUtf8Multilang::FromBuffer(std::move(buffer)).GetString(StringUtf8Multilang::GetLangIndex("ru"), desc); + TEST_EQUAL(desc, "Хорошие настойки", ()); + } +} + UNIT_CLASS_TEST(TestWithClassificator, OsmType_SimpleTypesSmoke) { Tags const oneTypes = { diff --git a/generator/osm2meta.cpp b/generator/osm2meta.cpp index 0eb576cec8..ebda5b4df8 100644 --- a/generator/osm2meta.cpp +++ b/generator/osm2meta.cpp @@ -386,3 +386,89 @@ string MetadataTagProcessorImpl::ValidateAndFormat_duration(string const & v) co return format(hours); } + +MetadataTagProcessor::~MetadataTagProcessor() +{ + if (!m_description.IsEmpty()) + m_params.GetMetadata().Set(feature::Metadata::FMD_DESCRIPTION, m_description.GetBuffer()); +} + +void MetadataTagProcessor::operator()(std::string const & k, std::string const & v) +{ + if (v.empty()) + return; + + using feature::Metadata; + Metadata & md = m_params.GetMetadata(); + + if (strings::StartsWith(k, "description")) + { + // Process description tags. + int8_t lang = StringUtf8Multilang::kDefaultCode; + size_t const i = k.find(':'); + if (i != std::string::npos) + { + int8_t const l = StringUtf8Multilang::GetLangIndex(k.substr(i+1)); + if (l != StringUtf8Multilang::kUnsupportedLanguageCode) + lang = l; + } + + m_description.AddString(lang, v); + return; + } + + Metadata::EType mdType; + if (!Metadata::TypeFromString(k, mdType)) + { + // Specific cases which do not map directly to our metadata types. + if (k == "building:min_level") + { + // Converting this attribute into height only if min_height has not been already set. + if (!md.Has(Metadata::FMD_MIN_HEIGHT)) + md.Set(Metadata::FMD_MIN_HEIGHT, ValidateAndFormat_building_levels(v)); + } + return; + } + + std::string valid; + switch (mdType) + { + case Metadata::FMD_OPEN_HOURS: valid = ValidateAndFormat_opening_hours(v); break; + case Metadata::FMD_FAX_NUMBER: // The same validator as for phone. + case Metadata::FMD_PHONE_NUMBER: valid = ValidateAndFormat_phone(v); break; + case Metadata::FMD_STARS: valid = ValidateAndFormat_stars(v); break; + case Metadata::FMD_OPERATOR: valid = ValidateAndFormat_operator(v); break; + case Metadata::FMD_URL: // The same validator as for website. + case Metadata::FMD_WEBSITE: valid = ValidateAndFormat_url(v); break; + case Metadata::FMD_CONTACT_FACEBOOK: valid = osm::ValidateAndFormat_facebook(v); break; + case Metadata::FMD_CONTACT_INSTAGRAM: valid = osm::ValidateAndFormat_instagram(v); break; + case Metadata::FMD_CONTACT_TWITTER: valid = osm::ValidateAndFormat_twitter(v); break; + case Metadata::FMD_CONTACT_VK: valid = osm::ValidateAndFormat_vk(v); break; + case Metadata::FMD_CONTACT_LINE: valid = osm::ValidateAndFormat_contactLine(v); break; + case Metadata::FMD_INTERNET: valid = ValidateAndFormat_internet(v); break; + case Metadata::FMD_ELE: valid = ValidateAndFormat_ele(v); break; + case Metadata::FMD_TURN_LANES: valid = ValidateAndFormat_turn_lanes(v); break; + case Metadata::FMD_TURN_LANES_FORWARD: valid = ValidateAndFormat_turn_lanes_forward(v); break; + case Metadata::FMD_TURN_LANES_BACKWARD: valid = ValidateAndFormat_turn_lanes_backward(v); break; + case Metadata::FMD_EMAIL: valid = ValidateAndFormat_email(v); break; + case Metadata::FMD_POSTCODE: valid = ValidateAndFormat_postcode(v); break; + case Metadata::FMD_WIKIPEDIA: valid = ValidateAndFormat_wikipedia(v); break; + case Metadata::FMD_FLATS: valid = ValidateAndFormat_flats(v); break; + case Metadata::FMD_MIN_HEIGHT: // The same validator as for height. + case Metadata::FMD_HEIGHT: valid = ValidateAndFormat_height(v); break; + case Metadata::FMD_DENOMINATION: valid = ValidateAndFormat_denomination(v); break; + case Metadata::FMD_BUILDING_LEVELS: valid = ValidateAndFormat_building_levels(v); break; + case Metadata::FMD_LEVEL: valid = ValidateAndFormat_level(v); break; + case Metadata::FMD_AIRPORT_IATA: valid = ValidateAndFormat_airport_iata(v); break; + case Metadata::FMD_DURATION: valid = ValidateAndFormat_duration(v); break; + // Used for old data compatibility only and should not be set: + case Metadata::FMD_CUISINE: + // Metadata types we do not get from OSM. + case Metadata::FMD_BRAND: + case Metadata::FMD_DESCRIPTION: // processed separately + case Metadata::FMD_TEST_ID: + case Metadata::FMD_COUNT: CHECK(false, (mdType, "should not be parsed from OSM.")); + } + + md.Set(mdType, valid); +} diff --git a/generator/osm2meta.hpp b/generator/osm2meta.hpp index 3aedd5296d..9dbdedfbc8 100644 --- a/generator/osm2meta.hpp +++ b/generator/osm2meta.hpp @@ -37,69 +37,14 @@ protected: class MetadataTagProcessor : private MetadataTagProcessorImpl { + StringUtf8Multilang m_description; + public: /// Make base class constructor public. using MetadataTagProcessorImpl::MetadataTagProcessorImpl; - void operator()(std::string const & k, std::string const & v) - { - if (v.empty()) - return; + // Assume that processor is created once, and we can make additional finalization in dtor. + ~MetadataTagProcessor(); - using feature::Metadata; - Metadata & md = m_params.GetMetadata(); - - Metadata::EType mdType; - if (!Metadata::TypeFromString(k, mdType)) - { - // Specific cases which do not map directly to our metadata types. - if (k == "building:min_level") - { - // Converting this attribute into height only if min_height has not been already set. - if (!md.Has(Metadata::FMD_MIN_HEIGHT)) - md.Set(Metadata::FMD_MIN_HEIGHT, ValidateAndFormat_building_levels(v)); - } - return; - } - - std::string valid; - switch (mdType) - { - case Metadata::FMD_OPEN_HOURS: valid = ValidateAndFormat_opening_hours(v); break; - case Metadata::FMD_FAX_NUMBER: // The same validator as for phone. - case Metadata::FMD_PHONE_NUMBER: valid = ValidateAndFormat_phone(v); break; - case Metadata::FMD_STARS: valid = ValidateAndFormat_stars(v); break; - case Metadata::FMD_OPERATOR: valid = ValidateAndFormat_operator(v); break; - case Metadata::FMD_URL: // The same validator as for website. - case Metadata::FMD_WEBSITE: valid = ValidateAndFormat_url(v); break; - case Metadata::FMD_CONTACT_FACEBOOK: valid = osm::ValidateAndFormat_facebook(v); break; - case Metadata::FMD_CONTACT_INSTAGRAM: valid = osm::ValidateAndFormat_instagram(v); break; - case Metadata::FMD_CONTACT_TWITTER: valid = osm::ValidateAndFormat_twitter(v); break; - case Metadata::FMD_CONTACT_VK: valid = osm::ValidateAndFormat_vk(v); break; - case Metadata::FMD_CONTACT_LINE: valid = osm::ValidateAndFormat_contactLine(v); break; - case Metadata::FMD_INTERNET: valid = ValidateAndFormat_internet(v); break; - case Metadata::FMD_ELE: valid = ValidateAndFormat_ele(v); break; - case Metadata::FMD_TURN_LANES: valid = ValidateAndFormat_turn_lanes(v); break; - case Metadata::FMD_TURN_LANES_FORWARD: valid = ValidateAndFormat_turn_lanes_forward(v); break; - case Metadata::FMD_TURN_LANES_BACKWARD: valid = ValidateAndFormat_turn_lanes_backward(v); break; - case Metadata::FMD_EMAIL: valid = ValidateAndFormat_email(v); break; - case Metadata::FMD_POSTCODE: valid = ValidateAndFormat_postcode(v); break; - case Metadata::FMD_WIKIPEDIA: valid = ValidateAndFormat_wikipedia(v); break; - case Metadata::FMD_FLATS: valid = ValidateAndFormat_flats(v); break; - case Metadata::FMD_MIN_HEIGHT: // The same validator as for height. - case Metadata::FMD_HEIGHT: valid = ValidateAndFormat_height(v); break; - case Metadata::FMD_DENOMINATION: valid = ValidateAndFormat_denomination(v); break; - case Metadata::FMD_BUILDING_LEVELS: valid = ValidateAndFormat_building_levels(v); break; - case Metadata::FMD_LEVEL: valid = ValidateAndFormat_level(v); break; - case Metadata::FMD_AIRPORT_IATA: valid = ValidateAndFormat_airport_iata(v); break; - case Metadata::FMD_DURATION: valid = ValidateAndFormat_duration(v); break; - // Used for old data compatibility only and should not be set: - case Metadata::FMD_CUISINE: - // Metadata types we do not get from OSM. - case Metadata::FMD_BRAND: - case Metadata::FMD_TEST_ID: - case Metadata::FMD_COUNT: CHECK(false, (mdType, "should not be parsed from OSM.")); - } - md.Set(mdType, valid); - } + void operator()(std::string const & k, std::string const & v); }; diff --git a/indexer/feature_meta.cpp b/indexer/feature_meta.cpp index 6c8f858a7e..1889818360 100644 --- a/indexer/feature_meta.cpp +++ b/indexer/feature_meta.cpp @@ -189,6 +189,7 @@ string ToString(Metadata::EType type) case Metadata::FMD_EMAIL: return "email"; case Metadata::FMD_POSTCODE: return "addr:postcode"; case Metadata::FMD_WIKIPEDIA: return "wikipedia"; + case Metadata::FMD_DESCRIPTION: return "description"; case Metadata::FMD_FLATS: return "addr:flats"; case Metadata::FMD_HEIGHT: return "height"; case Metadata::FMD_MIN_HEIGHT: return "min_height"; diff --git a/indexer/feature_meta.hpp b/indexer/feature_meta.hpp index 7b5e7458b6..7a5bbe7ac9 100644 --- a/indexer/feature_meta.hpp +++ b/indexer/feature_meta.hpp @@ -127,7 +127,7 @@ public: FMD_EMAIL = 14, FMD_POSTCODE = 15, FMD_WIKIPEDIA = 16, - // FMD_MAXSPEED used to be 17 but now it is stored in a section of its own. + FMD_DESCRIPTION = 17, FMD_FLATS = 18, FMD_HEIGHT = 19, FMD_MIN_HEIGHT = 20, diff --git a/indexer/map_object.hpp b/indexer/map_object.hpp index 6ca806d78c..1a703a2370 100644 --- a/indexer/map_object.hpp +++ b/indexer/map_object.hpp @@ -188,6 +188,7 @@ std::vector MetadataToProps(std::vector const & metadata) case Metadata::FMD_AIRPORT_IATA: case Metadata::FMD_BRAND: case Metadata::FMD_DURATION: + case Metadata::FMD_DESCRIPTION: case Metadata::FMD_COUNT: break; // Please add new cases when compiler issues an "unhandled switch case" warning here.