From 8e8d56c944377c83519af73f97459d4bade51aff Mon Sep 17 00:00:00 2001 From: tatiana-yan Date: Tue, 19 Nov 2019 16:09:54 +0300 Subject: [PATCH] [indexer] Do not save postcodes to metadata. --- editor/editor_config.cpp | 2 -- editor/editor_tests/xml_feature_test.cpp | 5 ++++ editor/osm_editor.cpp | 3 +++ editor/xml_feature.cpp | 17 ++++++++++++ editor/xml_feature.hpp | 3 +++ generator/osm2meta.cpp | 5 ---- generator/osm2meta.hpp | 6 ++--- generator/osm2type.cpp | 5 ++-- indexer/editable_map_object.cpp | 10 ++----- indexer/editable_map_object.hpp | 2 +- indexer/feature.cpp | 33 ++++++++++++++++++++++++ indexer/feature.hpp | 10 ++++++- indexer/feature_meta.cpp | 2 -- indexer/feature_meta.hpp | 2 +- indexer/map_object.cpp | 3 +++ indexer/map_object.hpp | 2 ++ indexer/shared_load_info.cpp | 8 ++++++ indexer/shared_load_info.hpp | 3 +++ search/retrieval.cpp | 2 +- 19 files changed, 96 insertions(+), 27 deletions(-) diff --git a/editor/editor_config.cpp b/editor/editor_config.cpp index f82018bb27..1b06591294 100644 --- a/editor/editor_config.cpp +++ b/editor/editor_config.cpp @@ -28,7 +28,6 @@ static unordered_map const kNamesToFMD = { // {"", feature::Metadata::FMD_TURN_LANES_FORWARD}, // {"", feature::Metadata::FMD_TURN_LANES_BACKWARD}, {"email", feature::Metadata::FMD_EMAIL}, - {"postcode", feature::Metadata::FMD_POSTCODE}, {"wikipedia", feature::Metadata::FMD_WIKIPEDIA}, // {"", feature::Metadata::FMD_MAXSPEED}, {"flats", feature::Metadata::FMD_FLATS}, @@ -127,7 +126,6 @@ bool EditorConfig::GetTypeDescription(vector classificatorTypes, { outDesc.m_address = isBuilding = true; outDesc.m_editableFields.push_back(feature::Metadata::FMD_BUILDING_LEVELS); - outDesc.m_editableFields.push_back(feature::Metadata::FMD_POSTCODE); classificatorTypes.erase(it); break; } diff --git a/editor/editor_tests/xml_feature_test.cpp b/editor/editor_tests/xml_feature_test.cpp index b87127d7e1..29b8b89f9b 100644 --- a/editor/editor_tests/xml_feature_test.cpp +++ b/editor/editor_tests/xml_feature_test.cpp @@ -61,6 +61,7 @@ UNIT_TEST(XMLFeature_Setters) feature.SetName("int_name", "Gorky Park"); feature.SetHouse("10"); + feature.SetPostcode("127001"); feature.SetTagValue("opening_hours", "Mo-Fr 08:15-17:30"); feature.SetTagValue("amenity", "atm"); @@ -74,6 +75,7 @@ UNIT_TEST(XMLFeature_Setters) + @@ -167,6 +169,7 @@ auto const kTestNode = R"( + @@ -186,6 +189,7 @@ UNIT_TEST(XMLFeature_FromXml) TEST(!feature.HasKey("FooBarBaz"), ()); TEST_EQUAL(feature.GetHouse(), "10", ()); + TEST_EQUAL(feature.GetPostcode(), "127001", ()); TEST_EQUAL(feature.GetCenter(), ms::LatLon(55.7978998, 37.4745280), ()); TEST_EQUAL(feature.GetName(), "Gorki Park", ()); TEST_EQUAL(feature.GetName("default"), "Gorki Park", ()); @@ -361,6 +365,7 @@ UNIT_TEST(XMLFeature_FromXMLAndBackToXML) + )"; diff --git a/editor/osm_editor.cpp b/editor/osm_editor.cpp index a8b0ad6179..a26552eea8 100644 --- a/editor/osm_editor.cpp +++ b/editor/osm_editor.cpp @@ -130,6 +130,9 @@ bool AreObjectsEqualButStreet(osm::EditableMapObject const & lhs, if (lhs.GetHouseNumber() != rhs.GetHouseNumber()) return false; + if (lhs.GetPostcode() != rhs.GetPostcode()) + return false; + if (!lhs.GetMetadata().Equals(rhs.GetMetadata())) return false; diff --git a/editor/xml_feature.cpp b/editor/xml_feature.cpp index de59d5321b..8e31cd7223 100644 --- a/editor/xml_feature.cpp +++ b/editor/xml_feature.cpp @@ -26,6 +26,7 @@ constexpr char const * kUploadTimestamp = "upload_timestamp"; constexpr char const * kUploadStatus = "upload_status"; constexpr char const * kUploadError = "upload_error"; constexpr char const * kHouseNumber = "addr:housenumber"; +constexpr char const * kPostcode = "addr:postcode"; constexpr char const * kUnknownType = "unknown"; constexpr char const * kNodeType = "node"; @@ -260,6 +261,10 @@ string XMLFeature::GetHouse() const { return GetTagValue(kHouseNumber); } void XMLFeature::SetHouse(string const & house) { SetTagValue(kHouseNumber, house); } +string XMLFeature::GetPostcode() const { return GetTagValue(kPostcode); } + +void XMLFeature::SetPostcode(string const & postcode) { SetTagValue(kPostcode, postcode); } + time_t XMLFeature::GetModificationTime() const { return base::StringToTimestamp(GetRootNode().attribute(kTimestamp).value()); @@ -393,6 +398,10 @@ void ApplyPatch(XMLFeature const & xml, osm::EditableMapObject & object) if (!house.empty()) object.SetHouseNumber(house); + auto const postcode = xml.GetPostcode(); + if (!postcode.empty()) + object.SetPostcode(postcode); + xml.ForEachTag([&object](string const & k, string const & v) { if (!object.UpdateMetadataValue(k, v)) LOG(LWARNING, ("Patch feature has unknown tags", k, v)); @@ -421,6 +430,10 @@ XMLFeature ToXML(osm::EditableMapObject const & object, bool serializeType) if (!house.empty()) toFeature.SetHouse(house); + auto const postcode = object.GetPostcode(); + if (!postcode.empty()) + toFeature.SetPostcode(postcode); + if (serializeType) { feature::TypesHolder th = object.GetTypes(); @@ -474,6 +487,10 @@ bool FromXML(XMLFeature const & xml, osm::EditableMapObject & object) if (!house.empty()) object.SetHouseNumber(house); + auto const postcode = xml.GetPostcode(); + if (!postcode.empty()) + object.SetPostcode(postcode); + feature::TypesHolder types; xml.ForEachTag([&object, &types](string const & k, string const & v) { if (object.UpdateMetadataValue(k, v)) diff --git a/editor/xml_feature.hpp b/editor/xml_feature.hpp index ce8e565464..d1e0922487 100644 --- a/editor/xml_feature.hpp +++ b/editor/xml_feature.hpp @@ -127,6 +127,9 @@ public: std::string GetHouse() const; void SetHouse(std::string const & house); + std::string GetPostcode() const; + void SetPostcode(std::string const & postcode); + /// Our and OSM modification time are equal. time_t GetModificationTime() const; void SetModificationTime(time_t const time); diff --git a/generator/osm2meta.cpp b/generator/osm2meta.cpp index b3adeab8d7..227c3db0c4 100644 --- a/generator/osm2meta.cpp +++ b/generator/osm2meta.cpp @@ -143,11 +143,6 @@ string MetadataTagProcessorImpl::ValidateAndFormat_email(string const & v) const return v; } -string MetadataTagProcessorImpl::ValidateAndFormat_postcode(string const & v) const -{ - return v; -} - string MetadataTagProcessorImpl::ValidateAndFormat_flats(string const & v) const { return v; diff --git a/generator/osm2meta.hpp b/generator/osm2meta.hpp index cd215389c2..f71d7107b5 100644 --- a/generator/osm2meta.hpp +++ b/generator/osm2meta.hpp @@ -23,7 +23,6 @@ struct MetadataTagProcessorImpl std::string ValidateAndFormat_turn_lanes_forward(std::string const & v) const; std::string ValidateAndFormat_turn_lanes_backward(std::string const & v) const; std::string ValidateAndFormat_email(std::string const & v) const; - std::string ValidateAndFormat_postcode(std::string const & v) const; std::string ValidateAndFormat_flats(std::string const & v) const; std::string ValidateAndFormat_internet(std::string v) const; std::string ValidateAndFormat_height(std::string const & v) const; @@ -84,7 +83,6 @@ public: case Metadata::FMD_TURN_LANES_FORWARD: valid = ValidateAndFormat_turn_lanes_forward(v); break; case Metadata::FMD_TURN_LANES_BACKWARD: valid = ValidateAndFormat_turn_lanes_backward(v); break; case Metadata::FMD_EMAIL: valid = ValidateAndFormat_email(v); break; - case Metadata::FMD_POSTCODE: valid = ValidateAndFormat_postcode(v); break; case Metadata::FMD_WIKIPEDIA: valid = ValidateAndFormat_wikipedia(v); break; case Metadata::FMD_FLATS: valid = ValidateAndFormat_flats(v); break; case Metadata::FMD_MIN_HEIGHT: // The same validator as for height. @@ -96,13 +94,15 @@ public: case Metadata::FMD_LEVEL: valid = ValidateAndFormat_level(v); break; case Metadata::FMD_AIRPORT_IATA: valid = ValidateAndFormat_airport_iata(v); break; case Metadata::FMD_DURATION: valid = ValidateAndFormat_duration(v); break; + // Used for old data compatibility only and should not be set: + case Metadata::FMD_POSTCODE: CHECK(false, (mdType, "used for compatibility, should not be set.")); // Metadata types we do not get from OSM. case Metadata::FMD_SPONSORED_ID: case Metadata::FMD_PRICE_RATE: case Metadata::FMD_RATING: case Metadata::FMD_BRAND: case Metadata::FMD_TEST_ID: - case Metadata::FMD_COUNT: CHECK(false, (mdType, "should not be parsed from OSM")); + case Metadata::FMD_COUNT: CHECK(false, (mdType, "should not be parsed from OSM.")); } md.Set(mdType, valid); return false; diff --git a/generator/osm2type.cpp b/generator/osm2type.cpp index 5650148115..95ab3253be 100644 --- a/generator/osm2type.cpp +++ b/generator/osm2type.cpp @@ -870,9 +870,8 @@ void GetNameAndType(OsmElement * p, FeatureParams & params, function const & EditableMapObject::GetNearbyStreets() const { return m_nearbyStreets; } -string EditableMapObject::GetPostcode() const -{ - return m_metadata.Get(feature::Metadata::FMD_POSTCODE); -} +string EditableMapObject::GetPostcode() const { return m_postcode; } string EditableMapObject::GetWikipedia() const { @@ -453,10 +450,7 @@ bool EditableMapObject::UpdateMetadataValue(string const & key, string const & v return true; } -void EditableMapObject::SetPostcode(string const & postcode) -{ - m_metadata.Set(feature::Metadata::FMD_POSTCODE, postcode); -} +void EditableMapObject::SetPostcode(string const & postcode) { m_postcode = postcode; } void EditableMapObject::SetPhone(string const & phone) { diff --git a/indexer/editable_map_object.hpp b/indexer/editable_map_object.hpp index 0945e7ab42..010dceed99 100644 --- a/indexer/editable_map_object.hpp +++ b/indexer/editable_map_object.hpp @@ -30,7 +30,7 @@ struct EditableProperties } bool m_name = false; - /// If true, enables editing of house number, street address and post code. + /// If true, enables editing of house number, street address and postcode. bool m_address = false; std::vector m_metadata; bool IsEditable() const { return m_name || m_address || !m_metadata.empty(); } diff --git a/indexer/feature.cpp b/indexer/feature.cpp index 8642d968ef..c9d335743f 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -6,6 +6,7 @@ #include "indexer/feature_utils.hpp" #include "indexer/feature_visibility.hpp" #include "indexer/map_object.hpp" +#include "indexer/postcodes.hpp" #include "indexer/scales.hpp" #include "indexer/shared_load_info.hpp" @@ -233,6 +234,9 @@ FeatureType::FeatureType(osm::MapObject const & emo) m_params.house.Set(house); m_parsed.m_common = true; + m_postcode = emo.GetPostcode(); + m_parsed.m_postcode = true; + m_metadata = emo.GetMetadata(); m_parsed.m_metadata = true; @@ -548,6 +552,29 @@ void FeatureType::ParseMetadata() m_parsed.m_metadata = true; } +void FeatureType::ParsePostcode() +{ + if (m_parsed.m_postcode) + return; + + auto postcodesReader = m_loadInfo->GetPostcodesReader(); + if (postcodesReader) + { + auto postcodes = indexer::Postcodes::Load(*postcodesReader->GetPtr()); + CHECK(postcodes, ()); + auto const havePostcode = postcodes->Get(m_id.m_index, m_postcode); + CHECK(!havePostcode || !m_postcode.empty(), (havePostcode, m_postcode)); + } + else + { + // Old data compatibility. + ParseMetadata(); + m_postcode = m_metadata.Get(feature::Metadata::FMD_POSTCODE); + } + + m_parsed.m_postcode = true; +} + StringUtf8Multilang const & FeatureType::GetNames() { ParseCommon(); @@ -765,6 +792,12 @@ string FeatureType::GetRoadNumber() return m_params.ref; } +string const & FeatureType::GetPostcode() +{ + ParsePostcode(); + return m_postcode; +} + feature::Metadata & FeatureType::GetMetadata() { ParseMetadata(); diff --git a/indexer/feature.hpp b/indexer/feature.hpp index 016cb684a3..76d1d01cbf 100644 --- a/indexer/feature.hpp +++ b/indexer/feature.hpp @@ -155,6 +155,8 @@ public: uint64_t GetPopulation(); std::string GetRoadNumber(); + std::string const & GetPostcode(); + feature::Metadata & GetMetadata(); /// @name Statistic functions. @@ -193,8 +195,12 @@ private: bool m_points = false; bool m_triangles = false; bool m_metadata = false; + bool m_postcode = false; - void Reset() { m_types = m_common = m_header2 = m_points = m_triangles = m_metadata = false; } + void Reset() + { + m_types = m_common = m_header2 = m_points = m_triangles = m_metadata = m_postcode = false; + } }; struct Offsets @@ -216,6 +222,7 @@ private: void ParseCommon(); void ParseHeader2(); void ParseMetadata(); + void ParsePostcode(); void ParseGeometryAndTriangles(int scale); uint8_t m_header = 0; @@ -223,6 +230,7 @@ private: FeatureID m_id; FeatureParamsBase m_params; + std::string m_postcode; m2::PointD m_center; m2::RectD m_limitRect; diff --git a/indexer/feature_meta.cpp b/indexer/feature_meta.cpp index d8ce86c4b0..31c2a5f08c 100644 --- a/indexer/feature_meta.cpp +++ b/indexer/feature_meta.cpp @@ -74,8 +74,6 @@ bool Metadata::TypeFromString(string const & k, Metadata::EType & outType) outType = Metadata::FMD_TURN_LANES_BACKWARD; else if (k == "email" || k == "contact:email") outType = Metadata::FMD_EMAIL; - else if (k == "addr:postcode") - outType = Metadata::FMD_POSTCODE; else if (k == "wikipedia") outType = Metadata::FMD_WIKIPEDIA; else if (k == "addr:flats") diff --git a/indexer/feature_meta.hpp b/indexer/feature_meta.hpp index 6380f5373f..2f4183277e 100644 --- a/indexer/feature_meta.hpp +++ b/indexer/feature_meta.hpp @@ -120,7 +120,7 @@ public: FMD_TURN_LANES_FORWARD = 12, FMD_TURN_LANES_BACKWARD = 13, FMD_EMAIL = 14, - FMD_POSTCODE = 15, + FMD_POSTCODE = 15, // Used for old data compatibility only. Should be empty for new mwms. FMD_WIKIPEDIA = 16, // FMD_MAXSPEED used to be 17 but now it is stored in a section of its own. FMD_FLATS = 18, diff --git a/indexer/map_object.cpp b/indexer/map_object.cpp index 1b9e249d40..2d9094d9bc 100644 --- a/indexer/map_object.cpp +++ b/indexer/map_object.cpp @@ -75,6 +75,7 @@ void MapObject::SetFromFeatureType(FeatureType & ft) m_types = feature::TypesHolder(ft); m_metadata = ft.GetMetadata(); m_houseNumber = ft.GetHouseNumber(); + m_postcode = ft.GetPostcode(); m_featureID = ft.GetID(); m_geomType = ft.GetGeomType(); if (m_geomType == feature::GeomType::Area) @@ -113,6 +114,8 @@ StringUtf8Multilang const & MapObject::GetNameMultilang() const string const & MapObject::GetHouseNumber() const { return m_houseNumber; } +string const & MapObject::GetPostcode() const { return m_postcode; } + string MapObject::GetLocalizedType() const { ASSERT(!m_types.Empty(), ()); diff --git a/indexer/map_object.hpp b/indexer/map_object.hpp index c67cd1ad26..c029bd1488 100644 --- a/indexer/map_object.hpp +++ b/indexer/map_object.hpp @@ -68,6 +68,7 @@ public: StringUtf8Multilang const & GetNameMultilang() const; std::string const & GetHouseNumber() const; + std::string const & GetPostcode() const; /// @name Metadata fields. //@{ @@ -118,6 +119,7 @@ protected: StringUtf8Multilang m_name; std::string m_houseNumber; + std::string m_postcode; feature::TypesHolder m_types; feature::Metadata m_metadata; diff --git a/indexer/shared_load_info.cpp b/indexer/shared_load_info.cpp index 6278d8fb51..e532f8032a 100644 --- a/indexer/shared_load_info.cpp +++ b/indexer/shared_load_info.cpp @@ -41,4 +41,12 @@ SharedLoadInfo::Reader SharedLoadInfo::GetTrianglesReader(int ind) const { return m_cont.GetReader(GetTagForIndex(TRIANGLE_FILE_TAG, ind)); } + +boost::optional SharedLoadInfo::GetPostcodesReader() const +{ + if (!m_cont.IsExist(POSTCODES_FILE_TAG)) + return {}; + + return m_cont.GetReader(POSTCODES_FILE_TAG); +} } // namespace feature diff --git a/indexer/shared_load_info.hpp b/indexer/shared_load_info.hpp index 8318f9f2da..a7b64b217c 100644 --- a/indexer/shared_load_info.hpp +++ b/indexer/shared_load_info.hpp @@ -7,6 +7,8 @@ #include "base/macros.hpp" +#include + namespace feature { // This info is created once per FeaturesVector. @@ -24,6 +26,7 @@ public: Reader GetAltitudeReader() const; Reader GetGeometryReader(int ind) const; Reader GetTrianglesReader(int ind) const; + boost::optional GetPostcodesReader() const; version::Format GetMWMFormat() const { return m_header.GetFormat(); } diff --git a/search/retrieval.cpp b/search/retrieval.cpp index 40c75f9388..a5beb386b7 100644 --- a/search/retrieval.cpp +++ b/search/retrieval.cpp @@ -225,7 +225,7 @@ pair MatchFeatureByNameAndType(EditableMapObject const & emo, bool MatchFeatureByPostcode(EditableMapObject const & emo, TokenSlice const & slice) { - string const postcode = emo.GetMetadata().Get(feature::Metadata::FMD_POSTCODE); + string const postcode = emo.GetPostcode(); vector tokens; NormalizeAndTokenizeString(postcode, tokens, Delimiters()); if (slice.Size() > tokens.size())